diff --git a/Src/Base/AMReX_Orientation.H b/Src/Base/AMReX_Orientation.H index 064344cafd4..de9c54a1b6c 100644 --- a/Src/Base/AMReX_Orientation.H +++ b/Src/Base/AMReX_Orientation.H @@ -75,7 +75,7 @@ public: * according to the above ordering. */ AMREX_GPU_HOST_DEVICE - operator int () const noexcept { return val; } + constexpr operator int () const noexcept { return val; } //! Return opposite orientation. AMREX_GPU_HOST_DEVICE Orientation flip () const noexcept @@ -97,6 +97,30 @@ public: //! Read from an istream. friend std::istream& operator>> (std::istream& os, Orientation& o); + //! Int value of the x-lo-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int xlo () noexcept { return 0; } + + //! Int value of the x-hi-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int xhi () noexcept { return AMREX_SPACEDIM; } + + //! Int value of the y-lo-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int ylo () noexcept { return 1; } + + //! Int value of the y-hi-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int yhi () noexcept { return 1+AMREX_SPACEDIM; } + + //! Int value of the z-lo-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int zlo () noexcept { return 2; } + + //! Int value of the z-hi-face + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static constexpr int zhi () noexcept { return 2+AMREX_SPACEDIM; } + private: //! Used internally. AMREX_GPU_HOST_DEVICE diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.H index a522d5aa927..1ed29a84801 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.H @@ -105,7 +105,8 @@ public: // for cuda void applyBCTensor (int amrlev, int mglev, MultiFab& vel, BCMode bc_mode, StateMode s_mode, const MLMGBndry* bndry) const; - void compCrossTerms(int amrlev, int mglev, MultiFab const& mf) const; + void compCrossTerms(int amrlev, int mglev, MultiFab const& mf, + const MLMGBndry* bndry) const; }; } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp index 590e062a3a1..87bb78da730 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp @@ -226,7 +226,7 @@ MLEBTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode MultiFab const& kapebmf = m_eb_kappa[amrlev][mglev]; Real bscalar = m_b_scalar; - compCrossTerms(amrlev, mglev, in); + compCrossTerms(amrlev, mglev, in, bndry); MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.EnableTiling().SetDynamic(true); @@ -289,15 +289,23 @@ MLEBTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode } void -MLEBTensorOp::compCrossTerms(int amrlev, int mglev, MultiFab const& mf) const +MLEBTensorOp::compCrossTerms(int amrlev, int mglev, MultiFab const& mf, + const MLMGBndry* bndry) const { auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; auto area = (factory) ? factory->getAreaFrac() : Array{AMREX_D_DECL(nullptr,nullptr,nullptr)}; + const auto& bcondloc = *m_bcondloc[amrlev][mglev]; + + Array4 foo; + const Geometry& geom = m_geom[amrlev][mglev]; const auto dxinv = geom.InvCellSizeArray(); + const Box& domain = geom.growPeriodicDomain(1); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); Array const& etamf = m_b_coeffs[amrlev][mglev]; Array const& kapmf = m_kappa[amrlev][mglev]; @@ -346,56 +354,143 @@ MLEBTensorOp::compCrossTerms(int amrlev, int mglev, MultiFab const& mf) const } ); } else { - AMREX_D_TERM(Array4 const fxfab = fluxmf[0].array(mfi);, - Array4 const fyfab = fluxmf[1].array(mfi);, - Array4 const fzfab = fluxmf[2].array(mfi);); - Array4 const vfab = mf.const_array(mfi); - AMREX_D_TERM(Array4 const etaxfab = etamf[0].const_array(mfi);, - Array4 const etayfab = etamf[1].const_array(mfi);, - Array4 const etazfab = etamf[2].const_array(mfi);); - AMREX_D_TERM(Array4 const kapxfab = kapmf[0].const_array(mfi);, - Array4 const kapyfab = kapmf[1].const_array(mfi);, - Array4 const kapzfab = kapmf[2].const_array(mfi);); - - if (fabtyp == FabType::regular) - { - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); - } - , ybx, tybx, - { - mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); - } - , zbx, tzbx, - { - mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); + AMREX_D_TERM(Array4 const fxfab = fluxmf[0].array(mfi);, + Array4 const fyfab = fluxmf[1].array(mfi);, + Array4 const fzfab = fluxmf[2].array(mfi);); + Array4 const vfab = mf.const_array(mfi); + AMREX_D_TERM(Array4 const etaxfab = etamf[0].const_array(mfi);, + Array4 const etayfab = etamf[1].const_array(mfi);, + Array4 const etazfab = etamf[2].const_array(mfi);); + AMREX_D_TERM(Array4 const kapxfab = kapmf[0].const_array(mfi);, + Array4 const kapyfab = kapmf[1].const_array(mfi);, + Array4 const kapzfab = kapmf[2].const_array(mfi);); + + if (fabtyp == FabType::regular) + { + if (domain.strictly_contains(bx)) { + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); + } + ); + } else { + const auto & bdcv = bcondloc.bndryConds(mfi); + + Array2D bct; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + } + } + + const auto& bvxlo = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::low )].array(mfi) : foo; + const auto& bvylo = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::low )].array(mfi) : foo; + const auto& bvxhi = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::high)].array(mfi) : foo; + const auto& bvyhi = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; +#if (AMREX_SPACEDIM == 3) + const auto& bvzlo = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::low )].array(mfi) : foo; + const auto& bvzhi = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; +#endif + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv, + bvxlo, bvxhi, bct, dlo, dhi); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv, + bvylo, bvyhi, bct, dlo, dhi); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv, + bvzlo, bvzhi, bct, dlo, dhi); + } + ); } - ); - } - else - { - AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, - Array4 const& apy = area[1]->const_array(mfi);, - Array4 const& apz = area[2]->const_array(mfi);); - Array4 const& flag = flags->const_array(mfi); + } + else + { + AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, + Array4 const& apy = area[1]->const_array(mfi);, + Array4 const& apz = area[2]->const_array(mfi);); + Array4 const& flag = flags->const_array(mfi); + + if (domain.strictly_contains(bx)) { + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mlebtensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,apx,flag,dxinv); + } + , ybx, tybx, + { + mlebtensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,apy,flag,dxinv); + } + , zbx, tzbx, + { + mlebtensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,apz,flag,dxinv); + } + ); + } else { + const auto & bdcv = bcondloc.bndryConds(mfi); + + Array2D bct; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + } + } + + const auto& bvxlo = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::low )].array(mfi) : foo; + const auto& bvylo = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::low )].array(mfi) : foo; + const auto& bvxhi = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::high)].array(mfi) : foo; + const auto& bvyhi = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; +#if (AMREX_SPACEDIM == 3) + const auto& bvzlo = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::low )].array(mfi) : foo; + const auto& bvzhi = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; +#endif - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mlebtensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,apx,flag,dxinv); - } - , ybx, tybx, - { - mlebtensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,apy,flag,dxinv); - } - , zbx, tzbx, - { - mlebtensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,apz,flag,dxinv); - } - ); - } + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mlebtensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,apx,flag,dxinv, bvxlo, bvxhi, bct, dlo, dhi); + } + , ybx, tybx, + { + mlebtensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,apy,flag,dxinv, bvylo, bvyhi, bct, dlo, dhi); + } + , zbx, tzbx, + { + mlebtensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,apz,flag,dxinv, bvzlo, bvzhi, bct, dlo, dhi); + } + ); + } + } } } @@ -411,7 +506,7 @@ MLEBTensorOp::compFlux (int amrlev, const Array& fluxe BL_PROFILE("MLEBTensorOp::compFlux()"); if ( !(loc==Location::FaceCenter || loc==Location::FaceCentroid) ) - amrex::Abort("MLEBTensorOp::compFlux() unknown location for fluxes."); + amrex::Abort("MLEBTensorOp::compFlux() unknown location for fluxes."); const int mglev = 0; const int ncomp = getNComp(); @@ -429,7 +524,7 @@ MLEBTensorOp::compFlux (int amrlev, const Array& fluxe Array& fluxmf = m_tauflux[amrlev][mglev]; Real bscalar = m_b_scalar; - compCrossTerms(amrlev, mglev, sol); + compCrossTerms(amrlev, mglev, sol, m_bndry_sol[amrlev].get()); MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.EnableTiling().SetDynamic(true); @@ -515,104 +610,11 @@ MLEBTensorOp::compFlux (int amrlev, const Array& fluxe } void -MLEBTensorOp::compVelGrad (int amrlev, const Array& fluxes, - MultiFab& sol, Location loc) const +MLEBTensorOp::compVelGrad (int /*amrlev*/, + const Array& /*fluxes*/, + MultiFab& /*sol*/, Location /*loc*/) const { - BL_PROFILE("MLEBTensorOp::compVelGrad()"); - - if ( !(loc==Location::FaceCenter || loc==Location::FaceCentroid) ) - amrex::Abort("MLEBTensorOp::compVelGrad() unknown location for VelGradients."); - - const int mglev = 0; - - applyBCTensor(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, m_bndry_sol[amrlev].get()); - - auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); - const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; - - const Geometry& geom = m_geom[amrlev][mglev]; - const auto dxinv = geom.InvCellSizeArray(); - - const int dim_fluxes = AMREX_SPACEDIM*AMREX_SPACEDIM; - - MFItInfo mfi_info; - if (Gpu::notInLaunchRegion()) mfi_info.EnableTiling().SetDynamic(true); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - { - Array fluxfab_tmp; - for (MFIter mfi(sol, mfi_info); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); - - auto fabtyp = (flags) ? (*flags)[mfi].getType(bx) : FabType::regular; - if (fabtyp == FabType::covered) continue; - - if (fabtyp == FabType::regular) - { - - Array4 const vfab = sol.const_array(mfi); - AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, - Box const ybx = mfi.nodaltilebox(1);, - Box const zbx = mfi.nodaltilebox(2);); - AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,dim_fluxes);, - fluxfab_tmp[1].resize(ybx,dim_fluxes);, - fluxfab_tmp[2].resize(zbx,dim_fluxes);); - AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();, - Elixir fyeli = fluxfab_tmp[1].elixir();, - Elixir fzeli = fluxfab_tmp[2].elixir();); - AMREX_D_TERM(Array4 const fxfab = fluxfab_tmp[0].array();, - Array4 const fyfab = fluxfab_tmp[1].array();, - Array4 const fzfab = fluxfab_tmp[2].array();); - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mltensor_vel_grads_fx(txbx,fxfab,vfab,dxinv); - } - , ybx, tybx, - { - mltensor_vel_grads_fy(tybx,fyfab,vfab,dxinv); - } - , zbx, tzbx, - { - mltensor_vel_grads_fz(tzbx,fzfab,vfab,dxinv); - } - ); - -// The derivatives are put in the array with the following order: -// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 -// in 2D: dU/dx, dV/dx, dU/dy, dV/dy -// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz - - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& nbx = mfi.nodaltilebox(idim); - Array4 dst = fluxes[idim]->array(mfi); - Array4 src = fluxfab_tmp[idim].const_array(); - AMREX_HOST_DEVICE_PARALLEL_FOR_4D (nbx, dim_fluxes, i, j, k, n, - { - dst(i,j,k,n) = src(i,j,k,n); - }); - } - - - } -// else if ( loc==Location::FaceCenter ) -// { -// -// amrex::Abort("compVelGrad not yet implemented for cut-cells "); -// -// } - else // loc==Location::FaceCentroid - { - - amrex::Abort("compVelGrad not yet implemented for cut-cells "); - - } - - } - } + amrex::Abort("compVelGrad not yet implemented for EB."); } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp_bc.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp_bc.cpp index c9c6eb232bb..98beecf01df 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp_bc.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp_bc.cpp @@ -13,11 +13,12 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto& bcondloc = *m_bcondloc[amrlev][mglev]; const auto& maskvals = m_maskvals[amrlev][mglev]; - FArrayBox foofab(Box::TheUnitBox(),3); - const auto& foo = foofab.array(); + Array4 foo; const auto dxinv = m_geom[amrlev][mglev].InvCellSizeArray(); const Box& domain = m_geom[amrlev][mglev].growPeriodicDomain(1); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; @@ -39,14 +40,13 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto & bdlv = bcondloc.bndryLocs(mfi); const auto & bdcv = bcondloc.bndryConds(mfi); - GpuArray bct; - GpuArray bcl; - for (OrientationIter face; face; ++face) { - Orientation ori = face(); - const int iface = ori; - for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { - bct[iface*AMREX_SPACEDIM+icomp] = bdcv[icomp][ori]; - bcl[iface*AMREX_SPACEDIM+icomp] = bdlv[icomp][ori]; + Array2D bct; + Array2D bcl; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + bcl(ori,icomp) = bdlv[icomp][ori]; } } @@ -72,7 +72,7 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, mxlo, mylo, mxhi, myhi, bvxlo, bvylo, bvxhi, bvyhi, bct, bcl, inhomog, imaxorder, - dxinv, domain); + dxinv, dlo, dhi); }); #else const auto& mzlo = maskvals[Orientation(2,Orientation::low )].array(mfi); @@ -83,14 +83,37 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto& bvzhi = (bndry != nullptr) ? (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; - AMREX_HOST_DEVICE_FOR_1D ( 12, iedge, +#ifdef AMREX_USE_GPU + if (Gpu::inLaunchRegion()) { + amrex::launch(12, 64, Gpu::gpuStream(), +#ifdef AMREX_USE_DPCPP + [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) + { + int bid = item.get_group_linear_id(); + int tid = item.get_local_linear_id(); + int bdim = item.get_local_range(0); +#else + [=] AMREX_GPU_DEVICE () + { + int bid = blockIdx.x; + int tid = threadIdx.x; + int bdim = blockDim.x; +#endif + mltensor_fill_edges(bid, tid, bdim, vbx, velfab, + mxlo, mylo, mzlo, mxhi, myhi, mzhi, + bvxlo, bvylo, bvzlo, bvxhi, bvyhi, bvzhi, + bct, bcl, inhomog, imaxorder, + dxinv, dlo, dhi); + }); + } else +#endif { - mltensor_fill_edges(iedge, vbx, velfab, + mltensor_fill_edges(vbx, velfab, mxlo, mylo, mzlo, mxhi, myhi, mzhi, bvxlo, bvylo, bvzlo, bvxhi, bvyhi, bvzhi, bct, bcl, inhomog, imaxorder, - dxinv, domain); - }); + dxinv, dlo, dhi); + } AMREX_HOST_DEVICE_FOR_1D ( 8, icorner, { @@ -98,13 +121,12 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, mxlo, mylo, mzlo, mxhi, myhi, mzhi, bvxlo, bvylo, bvzlo, bvxhi, bvyhi, bvzhi, bct, bcl, inhomog, imaxorder, - dxinv, domain); + dxinv, dlo, dhi); }); + #endif } } - - // Notet that it is incorrect to call EnforcePeriodicity on vel. } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H index 165497d1a20..d93ea3a5d1a 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H @@ -6,10 +6,95 @@ namespace amrex { -namespace { - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - Real mlebtensor_weight (int d) { - return (d==2) ? 0.5 : ((d==1) ? 1.0 : 0.0); +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + Array4 const& etax, + Array4 const& kapx, + Array4 const& apx, + Array4 const& flag, + GpuArray const& dxinv) noexcept +{ + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = 2./3.; + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + if (apx(i,j,0) == 0.0) + { + fx(i,j,0,0) = 0.0; + fx(i,j,0,1) = 0.0; + } + else + { + int jhip = j + flag(i ,j,0).isConnected(0, 1,0); + int jhim = j - flag(i ,j,0).isConnected(0,-1,0); + int jlop = j + flag(i-1,j,0).isConnected(0, 1,0); + int jlom = j - flag(i-1,j,0).isConnected(0,-1,0); + Real whi = mlebtensor_weight(jhip-jhim); + Real wlo = mlebtensor_weight(jlop-jlom); + Real dudy = mlebtensor_dy_on_xface(i,j,k,0,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dvdy = mlebtensor_dy_on_xface(i,j,k,1,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); + Real divu = dvdy; + Real xif = kapx(i,j,0); + Real mun = Real(0.75)*(etax(i,j,0,0)-xif);// restore the original eta + Real mut = etax(i,j,0,1); + fx(i,j,0,0) = -mun*(-twoThirds*divu) - xif*divu; + fx(i,j,0,1) = -mut*dudy; + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + Array4 const& etay, + Array4 const& kapy, + Array4 const& apy, + Array4 const& flag, + GpuArray const& dxinv) noexcept +{ + const Real dxi = dxinv[0]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = 2./3.; + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + if (apy(i,j,0) == 0.0) + { + fy(i,j,0,0) = 0.0; + fy(i,j,0,1) = 0.0; + } + else + { + int ihip = i + flag(i,j ,0).isConnected( 1,0,0); + int ihim = i - flag(i,j ,0).isConnected(-1,0,0); + int ilop = i + flag(i,j-1,0).isConnected( 1,0,0); + int ilom = i - flag(i,j-1,0).isConnected(-1,0,0); + Real whi = mlebtensor_weight(ihip-ihim); + Real wlo = mlebtensor_weight(ilop-ilom); + Real dudx = mlebtensor_dx_on_yface(i,j,k,0,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dvdx = mlebtensor_dx_on_yface(i,j,k,1,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); + Real divu = dudx; + Real xif = kapy(i,j,0); + Real mun = Real(0.75)*(etay(i,j,0,1)-xif);// restore the original eta + Real mut = etay(i,j,0,0); + fy(i,j,0,0) = -mut*dvdx; + fy(i,j,0,1) = -mun*(-twoThirds*divu) - xif*divu; + } + } } } @@ -20,13 +105,20 @@ void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, Array4 const& kapx, Array4 const& apx, Array4 const& flag, - GpuArray const& dxinv) noexcept + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept { const Real dyi = dxinv[1]; const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); constexpr Real twoThirds = 2./3.; + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { @@ -43,13 +135,15 @@ void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, int jlom = j - flag(i-1,j,0).isConnected(0,-1,0); Real whi = mlebtensor_weight(jhip-jhim); Real wlo = mlebtensor_weight(jlop-jlom); - Real dudy = (0.5*dyi) * ((vel(i ,jhip,0,0)-vel(i ,jhim,0,0))*whi - +(vel(i-1,jlop,0,0)-vel(i-1,jlom,0,0))*wlo); - Real dvdy = (0.5*dyi) * ((vel(i ,jhip,0,1)-vel(i ,jhim,0,1))*whi - +(vel(i-1,jlop,0,1)-vel(i-1,jlom,0,1))*wlo); + Real dudy = mlebtensor_dy_on_xface(i,j,k,0,vel,dyi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dvdy = mlebtensor_dy_on_xface(i,j,k,1,vel,dyi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); Real divu = dvdy; Real xif = kapx(i,j,0); - Real mun = 0.75*(etax(i,j,0,0)-xif); // restore the original eta + Real mun = Real(0.75)*(etax(i,j,0,0)-xif);// restore the original eta Real mut = etax(i,j,0,1); fx(i,j,0,0) = -mun*(-twoThirds*divu) - xif*divu; fx(i,j,0,1) = -mut*dudy; @@ -65,13 +159,20 @@ void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, Array4 const& kapy, Array4 const& apy, Array4 const& flag, - GpuArray const& dxinv) noexcept + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept { const Real dxi = dxinv[0]; const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); constexpr Real twoThirds = 2./3.; + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { @@ -88,15 +189,16 @@ void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, int ilom = i - flag(i,j-1,0).isConnected(-1,0,0); Real whi = mlebtensor_weight(ihip-ihim); Real wlo = mlebtensor_weight(ilop-ilom); - Real dudx = (0.5*dxi) * ((vel(ihip,j ,0,0)-vel(ihim,j ,0,0))*whi - +(vel(ilop,j-1,0,0)-vel(ilom,j-1,0,0))*wlo); - Real dvdx = (0.5*dxi) * ((vel(ihip,j ,0,1)-vel(ihim,j ,0,1))*whi - +(vel(ilop,j-1,0,1)-vel(ilom,j-1,0,1))*wlo); - + Real dudx = mlebtensor_dx_on_yface(i,j,k,0,vel,dxi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dvdx = mlebtensor_dx_on_yface(i,j,k,1,vel,dxi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); Real divu = dudx; Real xif = kapy(i,j,0); - Real mun = 0.75*(etay(i,j,0,1)-xif); // restore the original eta - Real mut = etay(i,j,0,0); + Real mun = Real(0.75)*(etay(i,j,0,1)-xif);// restore the original eta + Real mut = etay(i,j,0,0); fy(i,j,0,0) = -mut*dvdx; fy(i,j,0,1) = -mun*(-twoThirds*divu) - xif*divu; } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H index 3c26566e7ac..2651addee2c 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H @@ -6,11 +6,44 @@ namespace amrex { -namespace { - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - Real mlebtensor_weight (int d) { - return (d==2) ? 0.5 : ((d==1) ? 1.0 : 0.0); - } +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dz_on_xface (int i, int j, int, int n, + Array4 const& vel, Real dzi, + Real whi, Real wlo, + int khip, int khim, int klop, int klom) noexcept +{ + return Real(0.5)*dzi * ((vel(i ,j,khip,n)-vel(i ,j,khim,n))*whi + + (vel(i-1,j,klop,n)-vel(i-1,j,klom,n))*wlo); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dz_on_yface (int i, int j, int, int n, + Array4 const& vel, Real dzi, + Real whi, Real wlo, + int khip, int khim, int klop, int klom) noexcept +{ + return Real(0.5)*dzi * ((vel(i,j ,khip,n)-vel(i,j ,khim,n))*whi + + (vel(i,j-1,klop,n)-vel(i,j-1,klom,n))*wlo); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dx_on_zface (int, int j, int k, int n, + Array4 const& vel, Real dxi, + Real whi, Real wlo, + int ihip, int ihim, int ilop, int ilom) noexcept +{ + return Real(0.5)*dxi * ((vel(ihip,j,k ,n)-vel(ihim,j,k ,n))*whi + + (vel(ilop,j,k-1,n)-vel(ilom,j,k-1,n))*wlo); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dy_on_zface (int i, int, int k, int n, + Array4 const& vel, Real dyi, + Real whi, Real wlo, + int jhip, int jhim, int jlop, int jlom) noexcept +{ + return Real(0.5)*dyi * ((vel(i,jhip,k ,n)-vel(i,jhim,k ,n))*whi + + (vel(i,jlop,k-1,n)-vel(i,jlom,k-1,n))*wlo); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -46,26 +79,24 @@ void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, int jlom = j - flag(i-1,j,k).isConnected(0,-1,0); Real whi = mlebtensor_weight(jhip-jhim); Real wlo = mlebtensor_weight(jlop-jlom); - Real dudy = (0.5*dyi) * ((vel(i ,jhip,k,0)-vel(i ,jhim,k,0))*whi - +(vel(i-1,jlop,k,0)-vel(i-1,jlom,k,0))*wlo); - Real dvdy = (0.5*dyi) * ((vel(i ,jhip,k,1)-vel(i ,jhim,k,1))*whi - +(vel(i-1,jlop,k,1)-vel(i-1,jlom,k,1))*wlo); - + Real dudy = mlebtensor_dy_on_xface(i,j,k,0,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dvdy = mlebtensor_dy_on_xface(i,j,k,1,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); int khip = k + flag(i ,j,k).isConnected(0,0, 1); int khim = k - flag(i ,j,k).isConnected(0,0,-1); int klop = k + flag(i-1,j,k).isConnected(0,0, 1); int klom = k - flag(i-1,j,k).isConnected(0,0,-1); whi = mlebtensor_weight(khip-khim); wlo = mlebtensor_weight(klop-klom); - Real dudz = (0.5*dzi) * ((vel(i ,j,khip,0)-vel(i ,j,khim,0))*whi - +(vel(i-1,j,klop,0)-vel(i-1,j,klom,0))*wlo); - Real dwdz = (0.5*dzi) * ((vel(i ,j,khip,2)-vel(i ,j,khim,2))*whi - +(vel(i-1,j,klop,2)-vel(i-1,j,klom,2))*wlo); - + Real dudz = mlebtensor_dz_on_xface(i,j,k,0,vel,dzi, + whi,wlo,khip,khim,klop,klom); + Real dwdz = mlebtensor_dz_on_xface(i,j,k,2,vel,dzi, + whi,wlo,khip,khim,klop,klom); Real divu = dvdy + dwdz; Real xif = kapx(i,j,k); - Real mun = 0.75*(etax(i,j,k,0)-xif); // restore the original eta - Real mut = etax(i,j,k,1); + Real mun = Real(0.75)*(etax(i,j,k,0)-xif);// restore the original eta + Real mut = etax(i,j,k,1); fx(i,j,k,0) = -mun*(-twoThirds*divu) - xif*divu; fx(i,j,k,1) = -mut*dudy; fx(i,j,k,2) = -mut*dudz; @@ -108,26 +139,24 @@ void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, int ilom = i - flag(i,j-1,k).isConnected(-1,0,0); Real whi = mlebtensor_weight(ihip-ihim); Real wlo = mlebtensor_weight(ilop-ilom); - Real dudx = (0.5*dxi) * ((vel(ihip,j ,k,0)-vel(ihim,j ,k,0))*whi - +(vel(ilop,j-1,k,0)-vel(ilom,j-1,k,0))*wlo); - Real dvdx = (0.5*dxi) * ((vel(ihip,j ,k,1)-vel(ihim,j ,k,1))*whi - +(vel(ilop,j-1,k,1)-vel(ilom,j-1,k,1))*wlo); - + Real dudx = mlebtensor_dx_on_yface(i,j,k,0,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dvdx = mlebtensor_dx_on_yface(i,j,k,1,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); int khip = k + flag(i,j ,k).isConnected(0,0, 1); int khim = k - flag(i,j ,k).isConnected(0,0,-1); int klop = k + flag(i,j-1,k).isConnected(0,0, 1); int klom = k - flag(i,j-1,k).isConnected(0,0,-1); whi = mlebtensor_weight(khip-khim); wlo = mlebtensor_weight(klop-klom); - Real dvdz = (0.5*dzi) * ((vel(i,j ,khip,1)-vel(i,j ,khim,1))*whi - +(vel(i,j-1,klop,1)-vel(i,j-1,klom,1))*wlo); - Real dwdz = (0.5*dzi) * ((vel(i,j ,khip,2)-vel(i,j ,khim,2))*whi - +(vel(i,j-1,klop,2)-vel(i,j-1,klom,2))*wlo); - + Real dvdz = mlebtensor_dz_on_yface(i,j,k,1,vel,dzi, + whi,wlo,khip,khim,klop,klom); + Real dwdz = mlebtensor_dz_on_yface(i,j,k,2,vel,dzi, + whi,wlo,khip,khim,klop,klom); Real divu = dudx + dwdz; Real xif = kapy(i,j,k); - Real mun = 0.75*(etay(i,j,k,1)-xif); // restore the original eta - Real mut = etay(i,j,k,0); + Real mun = Real(0.75)*(etay(i,j,k,1)-xif);// restore the original eta + Real mut = etay(i,j,k,0); fy(i,j,k,0) = -mut*dvdx; fy(i,j,k,1) = -mun*(-twoThirds*divu) - xif*divu; fy(i,j,k,2) = -mut*dvdz; @@ -170,27 +199,457 @@ void mlebtensor_cross_terms_fz (Box const& box, Array4 const& fz, int ilom = i - flag(i,j,k-1).isConnected(-1,0,0); Real whi = mlebtensor_weight(ihip-ihim); Real wlo = mlebtensor_weight(ilop-ilom); + Real dudx = mlebtensor_dx_on_zface(i,j,k,0,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dwdx = mlebtensor_dx_on_zface(i,j,k,2,vel,dxi, + whi,wlo,ihip,ihim,ilop,ilom); + int jhip = j + flag(i,j,k ).isConnected(0, 1,0); + int jhim = j - flag(i,j,k ).isConnected(0,-1,0); + int jlop = j + flag(i,j,k-1).isConnected(0, 1,0); + int jlom = j - flag(i,j,k-1).isConnected(0,-1,0); + whi = mlebtensor_weight(jhip-jhim); + wlo = mlebtensor_weight(jlop-jlom); + Real dvdy = mlebtensor_dy_on_zface(i,j,k,1,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dwdy = mlebtensor_dy_on_zface(i,j,k,2,vel,dyi, + whi,wlo,jhip,jhim,jlop,jlom); + Real divu = dudx + dvdy; + Real xif = kapz(i,j,k); + Real mun = Real(0.75)*(etaz(i,j,k,2)-xif);// restore the original eta + Real mut = etaz(i,j,k,0); + + fz(i,j,k,0) = -mut*dwdx; + fz(i,j,k,1) = -mut*dwdy; + fz(i,j,k,2) = -mun*(-twoThirds*divu) - xif*divu; + } + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dz_on_xface (int i, int j, int k, int n, + Array4 const& vel, Real dzi, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int khip, int khim, int klop, int klom) noexcept +{ + Real ddz; + if (i == dlo.x) { + if (bct(Orientation::xlo(),n) == AMREX_LO_DIRICHLET && bvxlo) { + if (k == dlo.z) { + ddz = (bvxlo(i-1,j,k ,n) * Real(-1.5) + + bvxlo(i-1,j,k+1,n) * Real(2.) + + bvxlo(i-1,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvxlo(i-1,j,k ,n) * Real(-1.5) + + bvxlo(i-1,j,k-1,n) * Real(2.) + + bvxlo(i-1,j,k-2,n) * Real(-0.5)) * dzi; + } else { + ddz = whi*dzi*(bvxlo(i-1,j,khip,n)-bvxlo(i-1,j,khim,n)); + } + } else if (bct(Orientation::xlo(),n) == AMREX_LO_NEUMANN) { + ddz = whi*dzi*(vel(i,j,khip,n)-vel(i,j,khim,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); + } + } else if (i == dhi.x+1) { + if (bct(Orientation::xhi(),n) == AMREX_LO_DIRICHLET && bvxhi) { + if (k == dlo.z) { + ddz = (bvxhi(i,j,k ,n) * Real(-1.5) + + bvxhi(i,j,k+1,n) * Real(2.) + + bvxhi(i,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvxhi(i,j,k ,n) * Real(-1.5) + + bvxhi(i,j,k-1,n) * Real(2.) + + bvxhi(i,j,k-2,n) * Real(-0.5)) * dzi; + } else { + ddz = wlo*dzi*(bvxhi(i,j,klop,n)-bvxhi(i,j,klom,n)); + } + } else if (bct(Orientation::xhi(),n) == AMREX_LO_NEUMANN) { + ddz = wlo*dzi*(vel(i-1,j,klop,n)-vel(i-1,j,klom,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); + } + } else { + ddz = mlebtensor_dz_on_xface(i,j,k,n,vel,dzi,whi,wlo,khip,khim,klop,klom); + } + return ddz; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dz_on_yface (int i, int j, int k, int n, + Array4 const& vel, Real dzi, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int khip, int khim, int klop, int klom) noexcept +{ + Real ddz; + if (j == dlo.y) { + if (bct(Orientation::ylo(),n) == AMREX_LO_DIRICHLET && bvylo) { + if (k == dlo.z) { + ddz = (bvylo(i,j-1,k ,n) * Real(-1.5) + + bvylo(i,j-1,k+1,n) * Real(2.) + + bvylo(i,j-1,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvylo(i,j-1,k ,n) * Real(-1.5) + + bvylo(i,j-1,k-1,n) * Real(2.) + + bvylo(i,j-1,k-2,n) * Real(-0.5)) * dzi; + } else { + ddz = whi*dzi*(bvylo(i,j-1,khip,n)-bvylo(i,j-1,khim,n)); + } + } else if (bct(Orientation::ylo(),n) == AMREX_LO_NEUMANN) { + ddz = whi*dzi*(vel(i,j,khip,n)-vel(i,j,khim,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); + } + } else if (j == dhi.y+1) { + if (bct(Orientation::yhi(),n) == AMREX_LO_DIRICHLET && bvyhi) { + if (k == dlo.z) { + ddz = (bvyhi(i,j,k ,n) * Real(-1.5) + + bvyhi(i,j,k+1,n) * Real(2.) + + bvyhi(i,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvyhi(i,j,k ,n) * Real(-1.5) + + bvyhi(i,j,k-1,n) * Real(2.) + + bvyhi(i,j,k-2,n) * Real(-0.5)) * dzi; + } else { + ddz = wlo*dzi*(bvyhi(i,j,klop,n)-bvyhi(i,j,klom,n)); + } + } else if (bct(Orientation::yhi(),n) == AMREX_LO_NEUMANN) { + ddz = wlo*dzi*(vel(i,j-1,klop,n)-vel(i,j-1,klom,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); + } + } else { + ddz = mlebtensor_dz_on_yface(i,j,k,n,vel,dzi,whi,wlo,khip,khim,klop,klom); + } + return ddz; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dx_on_zface (int i, int j, int k, int n, + Array4 const& vel, Real dxi, + Array4 const& bvzlo, + Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int ihip, int ihim, int ilop, int ilom) noexcept +{ + Real ddx; + if (k == dlo.z) { + if (bct(Orientation::zlo(),n) == AMREX_LO_DIRICHLET && bvzlo) { + if (i == dlo.x) { + ddx = (bvzlo(i ,j,k-1,n) * Real(-1.5) + + bvzlo(i+1,j,k-1,n) * Real(2.) + + bvzlo(i+2,j,k-1,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvzlo(i ,j,k-1,n) * Real(-1.5) + + bvzlo(i-1,j,k-1,n) * Real(2.) + + bvzlo(i-2,j,k-1,n) * Real(-0.5)) * dxi; + } else { + ddx = whi*dxi*(bvzlo(ihip,j,k-1,n)-bvzlo(ihim,j,k-1,n)); + } + } else if (bct(Orientation::zlo(),n) == AMREX_LO_NEUMANN) { + ddx = whi*dxi*(vel(ihip,j,k,n)-vel(ihim,j,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else if (k == dhi.z+1) { + if (bct(Orientation::zhi(),n) == AMREX_LO_DIRICHLET && bvzhi) { + if (i == dlo.x) { + ddx = (bvzhi(i ,j,k,n) * Real(-1.5) + + bvzhi(i+1,j,k,n) * Real(2.) + + bvzhi(i+2,j,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvzhi(i ,j,k,n) * Real(-1.5) + + bvzhi(i-1,j,k,n) * Real(2.) + + bvzhi(i-2,j,k,n) * Real(-0.5)) * dxi; + } else { + ddx = wlo*dxi*(bvzhi(ilop,j,k,n)-bvzhi(ilom,j,k,n)); + } + } else if (bct(Orientation::zhi(),n) == AMREX_LO_NEUMANN) { + ddx = wlo*dxi*(vel(ilop,j,k-1,n)-vel(ilom,j,k-1,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else { + ddx = mlebtensor_dx_on_zface(i,j,k,n,vel,dxi,whi,wlo,ihip,ihim,ilop,ilom); + + } + return ddx; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dy_on_zface (int i, int j, int k, int n, + Array4 const& vel, Real dyi, + Array4 const& bvzlo, + Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int jhip, int jhim, int jlop, int jlom) noexcept +{ + Real ddy; + if (k == dlo.z) { + if (bct(Orientation::zlo(),n) == AMREX_LO_DIRICHLET && bvzlo) { + if (j == dlo.y) { + ddy = (bvzlo(i,j ,k-1,n) * Real(-1.5) + + bvzlo(i,j+1,k-1,n) * Real(2.) + + bvzlo(i,j+2,k-1,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvzlo(i,j ,k-1,n) * Real(-1.5) + + bvzlo(i,j-1,k-1,n) * Real(2.) + + bvzlo(i,j-2,k-1,n) * Real(-0.5)) * dyi; + } else { + ddy = whi*dyi*(bvzlo(i,jhip,k-1,n)-bvzlo(i,jhim,k-1,n)); + } + } else if (bct(Orientation::zlo(),n) == AMREX_LO_NEUMANN) { + ddy = whi*dyi*(vel(i,jhip,k,n)-vel(i,jhim,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else if (k == dhi.z+1) { + if (bct(Orientation::zhi(),n) == AMREX_LO_DIRICHLET && bvzhi) { + if (j == dlo.y) { + ddy = (bvzhi(i,j ,k,n) * Real(-1.5) + + bvzhi(i,j+1,k,n) * Real(2.) + + bvzhi(i,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvzhi(i,j ,k,n) * Real(-1.5) + + bvzhi(i,j-1,k,n) * Real(2.) + + bvzhi(i,j-2,k,n) * Real(-0.5)) * dyi; + } else { + ddy = wlo*dyi*(bvzhi(i,jlop,k,n)-bvzhi(i,jlom,k,n)); + } + } else if (bct(Orientation::zhi(),n) == AMREX_LO_NEUMANN) { + ddy = wlo*dyi*(vel(i,jlop,k-1,n)-vel(i,jlom,k-1,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else { + ddy = mlebtensor_dy_on_zface(i,j,k,n,vel,dyi,whi,wlo,jhip,jhim,jlop,jlom); + } + return ddy; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + Array4 const& etax, + Array4 const& kapx, + Array4 const& apx, + Array4 const& flag, + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept + +{ + const Real dyi = dxinv[1]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = 2./3.; + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + if (apx(i,j,k) == 0.0) + { + fx(i,j,k,0) = 0.0; + fx(i,j,k,1) = 0.0; + fx(i,j,k,2) = 0.0; + } + else + { + int jhip = j + flag(i ,j,k).isConnected(0, 1,0); + int jhim = j - flag(i ,j,k).isConnected(0,-1,0); + int jlop = j + flag(i-1,j,k).isConnected(0, 1,0); + int jlom = j - flag(i-1,j,k).isConnected(0,-1,0); + Real whi = mlebtensor_weight(jhip-jhim); + Real wlo = mlebtensor_weight(jlop-jlom); + Real dudy = mlebtensor_dy_on_xface(i,j,k,0,vel,dyi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dvdy = mlebtensor_dy_on_xface(i,j,k,1,vel,dyi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); + int khip = k + flag(i ,j,k).isConnected(0,0, 1); + int khim = k - flag(i ,j,k).isConnected(0,0,-1); + int klop = k + flag(i-1,j,k).isConnected(0,0, 1); + int klom = k - flag(i-1,j,k).isConnected(0,0,-1); + whi = mlebtensor_weight(khip-khim); + wlo = mlebtensor_weight(klop-klom); + Real dudz = mlebtensor_dz_on_xface(i,j,k,0,vel,dzi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,khip,khim,klop,klom); + Real dwdz = mlebtensor_dz_on_xface(i,j,k,2,vel,dzi, + bvxlo,bvxhi,bct,dlo,dhi, + whi,wlo,khip,khim,klop,klom); + Real divu = dvdy + dwdz; + Real xif = kapx(i,j,k); + Real mun = Real(0.75)*(etax(i,j,k,0)-xif);// restore the original eta + Real mut = etax(i,j,k,1); + fx(i,j,k,0) = -mun*(-twoThirds*divu) - xif*divu; + fx(i,j,k,1) = -mut*dudy; + fx(i,j,k,2) = -mut*dudz; + } + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + Array4 const& etay, + Array4 const& kapy, + Array4 const& apy, + Array4 const& flag, + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = 2./3.; + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + if (apy(i,j,k) == 0.0) + { + fy(i,j,k,0) = 0.0; + fy(i,j,k,1) = 0.0; + fy(i,j,k,2) = 0.0; + } + else + { + int ihip = i + flag(i,j ,k).isConnected( 1,0,0); + int ihim = i - flag(i,j ,k).isConnected(-1,0,0); + int ilop = i + flag(i,j-1,k).isConnected( 1,0,0); + int ilom = i - flag(i,j-1,k).isConnected(-1,0,0); + Real whi = mlebtensor_weight(ihip-ihim); + Real wlo = mlebtensor_weight(ilop-ilom); + Real dudx = mlebtensor_dx_on_yface(i,j,k,0,vel,dxi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dvdx = mlebtensor_dx_on_yface(i,j,k,1,vel,dxi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); + int khip = k + flag(i,j ,k).isConnected(0,0, 1); + int khim = k - flag(i,j ,k).isConnected(0,0,-1); + int klop = k + flag(i,j-1,k).isConnected(0,0, 1); + int klom = k - flag(i,j-1,k).isConnected(0,0,-1); + whi = mlebtensor_weight(khip-khim); + wlo = mlebtensor_weight(klop-klom); + Real dvdz = mlebtensor_dz_on_yface(i,j,k,1,vel,dzi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,khip,khim,klop,klom); + Real dwdz = mlebtensor_dz_on_yface(i,j,k,2,vel,dzi, + bvylo,bvyhi,bct,dlo,dhi, + whi,wlo,khip,khim,klop,klom); + Real divu = dudx + dwdz; + Real xif = kapy(i,j,k); + Real mun = Real(0.75)*(etay(i,j,k,1)-xif);// restore the original eta + Real mut = etay(i,j,k,0); + fy(i,j,k,0) = -mut*dvdx; + fy(i,j,k,1) = -mun*(-twoThirds*divu) - xif*divu; + fy(i,j,k,2) = -mut*dvdz; + } + } + } + } +} - Real dudx = (0.5*dxi) * ((vel(ihip,j,k ,0)-vel(ihim,j,k ,0))*whi - +(vel(ilop,j,k-1,0)-vel(ilom,j,k-1,0))*wlo); - Real dwdx = (0.5*dxi) * ((vel(ihip,j,k ,2)-vel(ihim,j,k ,2))*whi - +(vel(ilop,j,k-1,2)-vel(ilom,j,k-1,2))*wlo); +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mlebtensor_cross_terms_fz (Box const& box, Array4 const& fz, + Array4 const& vel, + Array4 const& etaz, + Array4 const& kapz, + Array4 const& apz, + Array4 const& flag, + GpuArray const& dxinv, + Array4 const& bvzlo, + Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = 2./3.; + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + if (apz(i,j,k) == 0.0) + { + fz(i,j,k,0) = 0.0; + fz(i,j,k,1) = 0.0; + fz(i,j,k,2) = 0.0; + } + else + { + int ihip = i + flag(i,j,k ).isConnected( 1,0,0); + int ihim = i - flag(i,j,k ).isConnected(-1,0,0); + int ilop = i + flag(i,j,k-1).isConnected( 1,0,0); + int ilom = i - flag(i,j,k-1).isConnected(-1,0,0); + Real whi = mlebtensor_weight(ihip-ihim); + Real wlo = mlebtensor_weight(ilop-ilom); + Real dudx = mlebtensor_dx_on_zface(i,j,k,0,vel,dxi, + bvzlo,bvzhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); + Real dwdx = mlebtensor_dx_on_zface(i,j,k,2,vel,dxi, + bvzlo,bvzhi,bct,dlo,dhi, + whi,wlo,ihip,ihim,ilop,ilom); int jhip = j + flag(i,j,k ).isConnected(0, 1,0); int jhim = j - flag(i,j,k ).isConnected(0,-1,0); int jlop = j + flag(i,j,k-1).isConnected(0, 1,0); int jlom = j - flag(i,j,k-1).isConnected(0,-1,0); whi = mlebtensor_weight(jhip-jhim); wlo = mlebtensor_weight(jlop-jlom); - Real dvdy = (0.5*dyi) * ((vel(i,jhip,k ,1)-vel(i,jhim,k ,1))*whi - +(vel(i,jlop,k-1,1)-vel(i,jlom,k-1,1))*wlo); - Real dwdy = (0.5*dyi) * ((vel(i,jhip,k ,2)-vel(i,jhim,k ,2))*whi - +(vel(i,jlop,k-1,2)-vel(i,jlom,k-1,2))*wlo); - + Real dvdy = mlebtensor_dy_on_zface(i,j,k,1,vel,dyi, + bvzlo,bvzhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); + Real dwdy = mlebtensor_dy_on_zface(i,j,k,2,vel,dyi, + bvzlo,bvzhi,bct,dlo,dhi, + whi,wlo,jhip,jhim,jlop,jlom); Real divu = dudx + dvdy; Real xif = kapz(i,j,k); - Real mun = 0.75*(etaz(i,j,k,2)-xif); // restore the original eta - Real mut = etaz(i,j,k,0); + Real mun = Real(0.75)*(etaz(i,j,k,2)-xif);// restore the original eta + Real mut = etaz(i,j,k,0); fz(i,j,k,0) = -mut*dwdx; fz(i,j,k,1) = -mut*dwdy; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_K.H index c814b3b8e41..8abdde8a7c0 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_K.H @@ -4,6 +4,145 @@ #include +namespace amrex { + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_weight (int d) { + return (d==2) ? 0.5 : ((d==1) ? 1.0 : 0.0); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dy_on_xface (int i, int, int k, int n, + Array4 const& vel, Real dyi, + Real whi, Real wlo, + int jhip, int jhim, int jlop, int jlom) noexcept +{ + return Real(0.5)*dyi * ((vel(i ,jhip,k,n)-vel(i ,jhim,k,n))*whi + + (vel(i-1,jlop,k,n)-vel(i-1,jlom,k,n))*wlo); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dx_on_yface (int, int j, int k, int n, + Array4 const& vel, Real dxi, + Real whi, Real wlo, + int ihip, int ihim, int ilop, int ilom) noexcept +{ + return Real(0.5)*dxi * ((vel(ihip,j ,k,n)-vel(ihim,j ,k,n))*whi + + (vel(ilop,j-1,k,n)-vel(ilom,j-1,k,n))*wlo); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dy_on_xface (int i, int j, int k, int n, + Array4 const& vel, Real dyi, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int jhip, int jhim, int jlop, int jlom) noexcept +{ + Real ddy; + if (i == dlo.x) { + if (bct(Orientation::xlo(),n) == AMREX_LO_DIRICHLET && bvxlo) { + if (j == dlo.y) { + ddy = (bvxlo(i-1,j ,k,n) * Real(-1.5) + + bvxlo(i-1,j+1,k,n) * Real(2.) + + bvxlo(i-1,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvxlo(i-1,j ,k,n) * Real(-1.5) + + bvxlo(i-1,j-1,k,n) * Real(2.) + + bvxlo(i-1,j-2,k,n) * Real(-0.5)) * dyi; + } else { + ddy = whi*dyi*(bvxlo(i-1,jhip,k,n)-bvxlo(i-1,jhim,k,n)); + } + } else if (bct(Orientation::xlo(),n) == AMREX_LO_NEUMANN) { + ddy = whi*dyi*(vel(i,jhip,k,n)-vel(i,jhim,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else if (i == dhi.x+1) { + if (bct(Orientation::xhi(),n) == AMREX_LO_DIRICHLET && bvxhi) { + if (j == dlo.y) { + ddy = (bvxhi(i,j ,k,n) * Real(-1.5) + + bvxhi(i,j+1,k,n) * Real(2.) + + bvxhi(i,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvxhi(i,j ,k,n) * Real(-1.5) + + bvxhi(i,j-1,k,n) * Real(2.) + + bvxhi(i,j-2,k,n) * Real(-0.5)) * dyi; + } else { + ddy = wlo*dyi*(bvxhi(i,jlop,k,n)-bvxhi(i,jlom,k,n)); + } + } else if (bct(Orientation::xhi(),n) == AMREX_LO_NEUMANN) { + ddy = wlo*dyi*(vel(i-1,jlop,k,n)-vel(i-1,jlom,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else { + ddy = mlebtensor_dy_on_xface(i,j,k,n,vel,dyi,whi,wlo,jhip,jhim,jlop,jlom); + } + return ddy; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mlebtensor_dx_on_yface (int i, int j, int k, int n, + Array4 const& vel, Real dxi, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi, + Real whi, Real wlo, + int ihip, int ihim, int ilop, int ilom) noexcept +{ + Real ddx; + if (j == dlo.y) { + if (bct(Orientation::ylo(),n) == AMREX_LO_DIRICHLET && bvylo) { + if (i == dlo.x) { + ddx = (bvylo(i ,j-1,k,n) * Real(-1.5) + + bvylo(i+1,j-1,k,n) * Real(2.) + + bvylo(i+2,j-1,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvylo(i ,j-1,k,n) * Real(-1.5) + + bvylo(i-1,j-1,k,n) * Real(2.) + + bvylo(i-2,j-1,k,n) * Real(-0.5)) * dxi; + } else { + ddx = whi*dxi*(bvylo(ihip,j-1,k,n)-bvylo(ihim,j-1,k,n)); + } + } else if (bct(Orientation::ylo(),n) == AMREX_LO_NEUMANN) { + ddx = whi*dxi*(vel(ihip,j,k,n)-vel(ihim,j,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else if (j == dhi.y+1) { + if (bct(Orientation::yhi(),n) == AMREX_LO_DIRICHLET && bvyhi) { + if (i == dlo.x) { + ddx = (bvyhi(i ,j,k,n) * Real(-1.5) + + bvyhi(i+1,j,k,n) * Real(2.) + + bvyhi(i+2,j,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvyhi(i ,j,k,n) * Real(-1.5) + + bvyhi(i-1,j,k,n) * Real(2.) + + bvyhi(i-2,j,k,n) * Real(-0.5)) * dxi; + } else { + ddx = wlo*dxi*(bvyhi(ilop,j,k,n)-bvyhi(ilom,j,k,n)); + } + } else if (bct(Orientation::yhi(),n) == AMREX_LO_NEUMANN) { + ddx = wlo*dxi*(vel(ilop,j-1,k,n)-vel(ilom,j-1,k,n)); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else { + ddx = mlebtensor_dx_on_yface(i,j,k,n,vel,dxi,whi,wlo,ihip,ihim,ilop,ilom); + } + return ddx; +} + +} + #if (AMREX_SPACEDIM == 1) #elif (AMREX_SPACEDIM == 2) #include diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp index d4e77f312dc..0750ffdd969 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp @@ -210,9 +210,16 @@ MLTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode bc if (mglev >= m_kappa[amrlev].size()) return; - applyBCTensor(amrlev, mglev, in, bc_mode, s_mode, bndry ); + applyBCTensor(amrlev, mglev, in, bc_mode, s_mode, bndry); + + const auto& bcondloc = *m_bcondloc[amrlev][mglev]; + + Array4 foo; const auto dxinv = m_geom[amrlev][mglev].InvCellSizeArray(); + const Box& domain = m_geom[amrlev][mglev].growPeriodicDomain(1); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); Array const& etamf = m_b_coeffs[amrlev][mglev]; Array const& kapmf = m_kappa[amrlev][mglev]; @@ -247,20 +254,65 @@ MLTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode bc Array4 const fyfab = fluxfab_tmp[1].array();, Array4 const fzfab = fluxfab_tmp[2].array();); - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); - } - , ybx, tybx, - { - mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); - } - , zbx, tzbx, - { - mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); - } - ); + if (domain.strictly_contains(bx)) { + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); + } + ); + } else { + const auto & bdcv = bcondloc.bndryConds(mfi); + + Array2D bct; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + } + } + + const auto& bvxlo = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::low )].array(mfi) : foo; + const auto& bvylo = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::low )].array(mfi) : foo; + const auto& bvxhi = (bndry != nullptr) ? + (*bndry)[Orientation(0,Orientation::high)].array(mfi) : foo; + const auto& bvyhi = (bndry != nullptr) ? + (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; +#if (AMREX_SPACEDIM == 3) + const auto& bvzlo = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::low )].array(mfi) : foo; + const auto& bvzhi = (bndry != nullptr) ? + (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; +#endif + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv, + bvxlo, bvxhi, bct, dlo, dhi); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv, + bvylo, bvyhi, bct, dlo, dhi); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv, + bvzlo, bvzhi, bct, dlo, dhi); + } + ); + } if (m_overset_mask[amrlev][mglev]) { const auto& osm = m_overset_mask[amrlev][mglev]->array(mfi); @@ -288,18 +340,18 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, #if (AMREX_SPACEDIM == 1) amrex::ignore_unused(amrlev,mglev,vel,bc_mode,bndry); #else + const int inhomog = bc_mode == BCMode::Inhomogeneous; const int imaxorder = maxorder; const auto& bcondloc = *m_bcondloc[amrlev][mglev]; const auto& maskvals = m_maskvals[amrlev][mglev]; - FArrayBox foofab(Box::TheUnitBox(),3); - const auto& foo = foofab.array(); + Array4 foo; const auto dxinv = m_geom[amrlev][mglev].InvCellSizeArray(); const Box& domain = m_geom[amrlev][mglev].growPeriodicDomain(1); - - // Domain and coarse-fine boundaries are handled below. + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.SetDynamic(true); @@ -315,14 +367,13 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto & bdlv = bcondloc.bndryLocs(mfi); const auto & bdcv = bcondloc.bndryConds(mfi); - GpuArray bct; - GpuArray bcl; - for (OrientationIter face; face; ++face) { - Orientation ori = face(); - const int iface = ori; - for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { - bct[iface*AMREX_SPACEDIM+icomp] = bdcv[icomp][ori]; - bcl[iface*AMREX_SPACEDIM+icomp] = bdlv[icomp][ori]; + Array2D bct; + Array2D bcl; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + bcl(ori,icomp) = bdlv[icomp][ori]; } } @@ -341,14 +392,13 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; #if (AMREX_SPACEDIM == 2) - AMREX_HOST_DEVICE_FOR_1D ( 4, icorner, { mltensor_fill_corners(icorner, vbx, velfab, mxlo, mylo, mxhi, myhi, bvxlo, bvylo, bvxhi, bvyhi, bct, bcl, inhomog, imaxorder, - dxinv, domain); + dxinv, dlo, dhi); }); #else const auto& mzlo = maskvals[Orientation(2,Orientation::low )].array(mfi); @@ -360,18 +410,40 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; // only edge vals used in 3D stencil - AMREX_HOST_DEVICE_FOR_1D ( 12, iedge, +#ifdef AMREX_USE_GPU + if (Gpu::inLaunchRegion()) { + amrex::launch(12, 64, Gpu::gpuStream(), +#ifdef AMREX_USE_DPCPP + [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) + { + int bid = item.get_group_linear_id(); + int tid = item.get_local_linear_id(); + int bdim = item.get_local_range(0); +#else + [=] AMREX_GPU_DEVICE () + { + int bid = blockIdx.x; + int tid = threadIdx.x; + int bdim = blockDim.x; +#endif + mltensor_fill_edges(bid, tid, bdim, vbx, velfab, + mxlo, mylo, mzlo, mxhi, myhi, mzhi, + bvxlo, bvylo, bvzlo, bvxhi, bvyhi, bvzhi, + bct, bcl, inhomog, imaxorder, + dxinv, dlo, dhi); + }); + } else +#endif { - mltensor_fill_edges(iedge, vbx, velfab, + mltensor_fill_edges(vbx, velfab, mxlo, mylo, mzlo, mxhi, myhi, mzhi, bvxlo, bvylo, bvzlo, bvxhi, bvyhi, bvzhi, bct, bcl, inhomog, imaxorder, - dxinv, domain); - }); + dxinv, dlo, dhi); + } #endif } - // Notet that it is incorrect to call EnforcePeriodicity on vel. #endif } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp index 705f38052d1..d395ecdac13 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp @@ -16,9 +16,15 @@ MLTensorOp::compFlux (int amrlev, const Array& fluxes, const int ncomp = getNComp(); MLABecLaplacian::compFlux(amrlev, fluxes, sol, loc); - applyBCTensor(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, m_bndry_sol[amrlev].get()); + MLMGBndry const* bndry = m_bndry_sol[amrlev].get(); + applyBCTensor(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, bndry); + + const auto& bcondloc = *m_bcondloc[amrlev][mglev]; const auto dxinv = m_geom[amrlev][mglev].InvCellSizeArray(); + const Box& domain = m_geom[amrlev][mglev].growPeriodicDomain(1); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); Array const& etamf = m_b_coeffs[amrlev][mglev]; Array const& kapmf = m_kappa[amrlev][mglev]; @@ -52,20 +58,59 @@ MLTensorOp::compFlux (int amrlev, const Array& fluxes, Array4 const fyfab = fluxfab_tmp[1].array();, Array4 const fzfab = fluxfab_tmp[2].array();); - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); - } - , ybx, tybx, - { - mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); - } - , zbx, tzbx, - { - mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); - } - ); + if (domain.strictly_contains(mfi.tilebox())) { + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); + } + ); + } else { + const auto & bdcv = bcondloc.bndryConds(mfi); + + Array2D bct; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + } + } + + const auto& bvxlo = (*bndry)[Orientation(0,Orientation::low )].array(mfi); + const auto& bvylo = (*bndry)[Orientation(1,Orientation::low )].array(mfi); + const auto& bvxhi = (*bndry)[Orientation(0,Orientation::high)].array(mfi); + const auto& bvyhi = (*bndry)[Orientation(1,Orientation::high)].array(mfi); +#if (AMREX_SPACEDIM == 3) + const auto& bvzlo = (*bndry)[Orientation(2,Orientation::low )].array(mfi); + const auto& bvzhi = (*bndry)[Orientation(2,Orientation::high)].array(mfi); +#endif + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv, + bvxlo, bvxhi, bct, dlo, dhi); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv, + bvylo, bvyhi, bct, dlo, dhi); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv, + bvzlo, bvzhi, bct, dlo, dhi); + } + ); + } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const Box& nbx = mfi.nodaltilebox(idim); @@ -95,33 +140,36 @@ MLTensorOp::compVelGrad (int amrlev, const Array& flux const int mglev = 0; - applyBCTensor(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, m_bndry_sol[amrlev].get()); + MLMGBndry const* bndry = m_bndry_sol[amrlev].get(); + applyBC(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, bndry); + applyBCTensor(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, bndry); + + const auto& bcondloc = *m_bcondloc[amrlev][mglev]; const auto dxinv = m_geom[amrlev][mglev].InvCellSizeArray(); - const int dim_fluxes = AMREX_SPACEDIM*AMREX_SPACEDIM; + const Box& domain = m_geom[amrlev][mglev].growPeriodicDomain(1); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif + for (MFIter mfi(sol, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Array fluxfab_tmp; + Array4 const vfab = sol.const_array(mfi); + AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, + Box const ybx = mfi.nodaltilebox(1);, + Box const zbx = mfi.nodaltilebox(2);) + AMREX_D_TERM(Array4 const fxfab = fluxes[0]->array(mfi);, + Array4 const fyfab = fluxes[1]->array(mfi);, + Array4 const fzfab = fluxes[2]->array(mfi);) - for (MFIter mfi(sol, TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const vfab = sol.const_array(mfi); - AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, - Box const ybx = mfi.nodaltilebox(1);, - Box const zbx = mfi.nodaltilebox(2);); - AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,dim_fluxes);, - fluxfab_tmp[1].resize(ybx,dim_fluxes);, - fluxfab_tmp[2].resize(zbx,dim_fluxes);); - AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();, - Elixir fyeli = fluxfab_tmp[1].elixir();, - Elixir fzeli = fluxfab_tmp[2].elixir();); - AMREX_D_TERM(Array4 const fxfab = fluxfab_tmp[0].array();, - Array4 const fyfab = fluxfab_tmp[1].array();, - Array4 const fzfab = fluxfab_tmp[2].array();); +// The derivatives are put in the array with the following order: +// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 +// in 2D: dU/dx, dV/dx, dU/dy, dV/dy +// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz + if (domain.strictly_contains(mfi.tilebox())) { AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM ( xbx, txbx, { @@ -136,23 +184,39 @@ MLTensorOp::compVelGrad (int amrlev, const Array& flux mltensor_vel_grads_fz(tzbx,fzfab,vfab,dxinv); } ); - -// The derivatives are put in the array with the following order: -// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 -// in 2D: dU/dx, dV/dx, dU/dy, dV/dy -// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz - - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& nbx = mfi.nodaltilebox(idim); - Array4 dst = fluxes[idim]->array(mfi); - Array4 src = fluxfab_tmp[idim].const_array(); - AMREX_HOST_DEVICE_PARALLEL_FOR_4D (nbx, dim_fluxes, i, j, k, n, - { - dst(i,j,k,n) = src(i,j,k,n); - }); + } else { + const auto & bdcv = bcondloc.bndryConds(mfi); + + Array2D bct; + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + for (OrientationIter face; face; ++face) { + Orientation ori = face(); + bct(ori,icomp) = bdcv[icomp][ori]; + } } + const auto& bvxlo = (*bndry)[Orientation(0,Orientation::low )].array(mfi); + const auto& bvylo = (*bndry)[Orientation(1,Orientation::low )].array(mfi); + const auto& bvxhi = (*bndry)[Orientation(0,Orientation::high)].array(mfi); + const auto& bvyhi = (*bndry)[Orientation(1,Orientation::high)].array(mfi); +#if (AMREX_SPACEDIM == 3) + const auto& bvzlo = (*bndry)[Orientation(2,Orientation::low )].array(mfi); + const auto& bvzhi = (*bndry)[Orientation(2,Orientation::high)].array(mfi); +#endif + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_vel_grads_fx(txbx,fxfab,vfab,dxinv,bvxlo,bvxhi,bct,dlo,dhi); + } + , ybx, tybx, + { + mltensor_vel_grads_fy(tybx,fyfab,vfab,dxinv,bvylo,bvyhi,bct,dlo,dhi); + } + , zbx, tzbx, + { + mltensor_vel_grads_fz(tzbx,fzfab,vfab,dxinv,bvzlo,bvzhi,bct,dlo,dhi); + } + ); } } #endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensor_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLTensor_2D_K.H index 8f10f08ec58..a40fa4611a8 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensor_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensor_2D_K.H @@ -17,110 +17,168 @@ void mltensor_fill_corners (int icorner, Box const& vbox, // vbox: the valid box Array4 const& bcvalylo, Array4 const& bcvalxhi, Array4 const& bcvalyhi, - GpuArray const& bct, - GpuArray const& bcl, + Array2D const& bct, + Array2D const& bcl, int inhomog, int maxorder, - GpuArray const& dxinv, Box const& domain) noexcept + GpuArray const& dxinv, + Dim3 const& dlo, Dim3 const& dhi) noexcept { - constexpr int oxlo = 0; - constexpr int oylo = 1; - constexpr int oxhi = 2; - constexpr int oyhi = 3; - constexpr int xdir = 0; - constexpr int ydir = 1; + constexpr int k = 0; const auto blen = amrex::length(vbox); const auto vlo = amrex::lbound(vbox); const auto vhi = amrex::ubound(vbox); - const auto dlo = amrex::lbound(domain); - const auto dhi = amrex::ubound(domain); - for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { - switch (icorner) { - case 0: { - // xlo & ylo - if (mxlo(vlo.x-1,vlo.y-1,0) != BndryData::covered) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),ydir,1); - if (vlo.x == dlo.x && vlo.y == dlo.y) { - vel(vlo.x-1,vlo.y-1,0,icomp) = vel(vlo.x-1,vlo.y,0,icomp) - + vel(vlo.x,vlo.y-1,0,icomp) - vel(vlo.x,vlo.y,0,icomp); - } else if (vlo.x == dlo.x || mylo(vlo.x,vlo.y-1,0) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); + if (icorner == 0) { // xlo & ylo + int const i = vlo.x-1; + int const j = vlo.y-1; + if (mxlo(i,j,k) != BndryData::covered && (dlo.x != vlo.x || dlo.y != vlo.y)) { + bool x_interior = mylo(i+1,j ,k) == BndryData::covered; // i+1,j is a valid cell inside domain + bool x_exterior = mylo(i+1,j ,k) == BndryData::not_covered; // i+1,j is a ghost cell inside domain + bool y_interior = mxlo(i ,j+1,k) == BndryData::covered; + bool y_exterior = mxlo(i ,j+1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || dlo.x == vlo.x) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || dlo.y == vlo.y) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); } } - break; } - case 1: { - // xhi & ylo - if (mxhi(vhi.x+1,vlo.y-1,0) != BndryData::covered) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),ydir,1); - if (vhi.x == dhi.x && vlo.y == dlo.y) { - vel(vhi.x+1,vlo.y-1,0,icomp) = vel(vhi.x+1,vlo.y,0,icomp) - + vel(vhi.x,vlo.y-1,0,icomp) - vel(vhi.x,vlo.y,0,icomp); - } else if (vhi.x == dhi.x || mylo(vhi.x,vlo.y-1,0) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); + } else if (icorner == 1) { // xhi & ylo + int const i = vhi.x+1; + int const j = vlo.y-1; + if (mxhi(i,j,k) != BndryData::covered && (dhi.x != vhi.x || dlo.y != vlo.y)) { + bool x_interior = mylo(i-1,j ,k) == BndryData::covered; + bool x_exterior = mylo(i-1,j ,k) == BndryData::not_covered; + bool y_interior = mxhi(i ,j+1,k) == BndryData::covered; + bool y_exterior = mxhi(i ,j+1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || dhi.x == vhi.x) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || dlo.y == vlo.y) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); } } - break; } - case 2: { - // xlo & yhi - if (mxlo(vlo.x-1,vhi.y+1,0) != BndryData::covered) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),ydir,1); - if (vlo.x == dlo.x && vhi.y == dhi.y) { - vel(vlo.x-1,vhi.y+1,0,icomp) = vel(vlo.x-1,vhi.y,0,icomp) - + vel(vlo.x,vhi.y+1,0,icomp) - vel(vlo.x,vhi.y,0,icomp); - } else if (vlo.x == dlo.x || myhi(vlo.x,vhi.y+1,0) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); + } else if (icorner == 2) { // xlo & yhi + int const i = vlo.x-1; + int const j = vhi.y+1; + if (mxlo(i,j,k) != BndryData::covered && (dlo.x != vlo.x || dhi.y != vhi.y)) { + bool x_interior = myhi(i+1,j ,k) == BndryData::covered; + bool x_exterior = myhi(i+1,j ,k) == BndryData::not_covered; + bool y_interior = mxlo(i ,j-1,k) == BndryData::covered; + bool y_exterior = mxlo(i ,j-1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || dlo.x == vlo.x) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || dhi.y == vhi.y) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); } } - break; } - case 3: { - // xhi & yhi - if (mxhi(vhi.x+1,vhi.y+1,0) != BndryData::covered) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),ydir,1); - if (vhi.x == dhi.x && vhi.y == dhi.y) { - vel(vhi.x+1,vhi.y+1,0,icomp) = vel(vhi.x+1,vhi.y,0,icomp) - + vel(vhi.x,vhi.y+1,0,icomp) - vel(vhi.x,vhi.y,0,icomp); - } else if (vhi.x == dhi.x || myhi(vhi.x,vhi.y+1,0) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); + } else if (icorner == 3) { // xhi & yhi + int const i = vhi.x+1; + int const j = vhi.y+1; + if (mxhi(i,j,k) != BndryData::covered && (dhi.x != vhi.x || dhi.y != vhi.y)) { + bool x_interior = myhi(i-1,j ,k) == BndryData::covered; + bool x_exterior = myhi(i-1,j ,k) == BndryData::not_covered; + bool y_interior = mxhi(i ,j-1,k) == BndryData::covered; + bool y_exterior = mxhi(i ,j-1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || dhi.x == vhi.x) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || dhi.y == vhi.y) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); } } - break; - } - default: {} } } } @@ -137,11 +195,12 @@ void mltensor_cross_terms_fx (Box const& box, Array4 const& fx, const auto hi = amrex::ubound(box); constexpr Real twoThirds = Real(2./3.); + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudy = (vel(i,j+1,0,0)+vel(i-1,j+1,0,0)-vel(i,j-1,0,0)-vel(i-1,j-1,0,0))*(Real(0.25)*dyi); - Real dvdy = (vel(i,j+1,0,1)+vel(i-1,j+1,0,1)-vel(i,j-1,0,1)-vel(i-1,j-1,0,1))*(Real(0.25)*dyi); + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi); Real divu = dvdy; Real xif = kapx(i,j,0); Real mun = Real(0.75)*(etax(i,j,0,0)-xif); // restore the original eta @@ -164,11 +223,80 @@ void mltensor_cross_terms_fy (Box const& box, Array4 const& fy, const auto hi = amrex::ubound(box); constexpr Real twoThirds = Real(2./3.); + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,0,0)+vel(i+1,j-1,0,0)-vel(i-1,j,0,0)-vel(i-1,j-1,0,0))*(Real(0.25)*dxi); - Real dvdx = (vel(i+1,j,0,1)+vel(i+1,j-1,0,1)-vel(i-1,j,0,1)-vel(i-1,j-1,0,1))*(Real(0.25)*dxi); + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi); + Real divu = dudx; + Real xif = kapy(i,j,0); + Real mun = Real(0.75)*(etay(i,j,0,1)-xif); // restore the original eta + Real mut = etay(i,j,0,0); + fy(i,j,0,0) = -mut*dvdx; + fy(i,j,0,1) = -mun*(-twoThirds*divu) - xif*divu; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_cross_terms_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + Array4 const& etax, + Array4 const& kapx, + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = Real(2./3.); + + // Three BC types: reflect odd, neumann, and dirichlet + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real divu = dvdy; + Real xif = kapx(i,j,0); + Real mun = Real(0.75)*(etax(i,j,0,0)-xif); // restore the original eta + Real mut = etax(i,j,0,1); + fx(i,j,0,0) = -mun*(-twoThirds*divu) - xif*divu; + fx(i,j,0,1) = -mut*dudy; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_cross_terms_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + Array4 const& etay, + Array4 const& kapy, + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = Real(2./3.); + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); Real divu = dudx; Real xif = kapy(i,j,0); Real mun = Real(0.75)*(etay(i,j,0,1)-xif); // restore the original eta @@ -241,13 +369,14 @@ void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real dudx = (vel(i,j,0,0) - vel(i-1,j,0,0))*dxi; Real dvdx = (vel(i,j,0,1) - vel(i-1,j,0,1))*dxi; - Real dudy = (vel(i,j+1,0,0)+vel(i-1,j+1,0,0)-vel(i,j-1,0,0)-vel(i-1,j-1,0,0))*(Real(0.25)*dyi); - Real dvdy = (vel(i,j+1,0,1)+vel(i-1,j+1,0,1)-vel(i,j-1,0,1)-vel(i-1,j-1,0,1))*(Real(0.25)*dyi); + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi); fx(i,j,0,0) = dudx; fx(i,j,0,1) = dvdx; fx(i,j,0,2) = dudy; @@ -266,11 +395,74 @@ void mltensor_vel_grads_fy (Box const& box, Array4 const& fy, const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); + int k = 0; for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,0,0)+vel(i+1,j-1,0,0)-vel(i-1,j,0,0)-vel(i-1,j-1,0,0))*(Real(0.25)*dxi); - Real dvdx = (vel(i+1,j,0,1)+vel(i+1,j-1,0,1)-vel(i-1,j,0,1)-vel(i-1,j-1,0,1))*(Real(0.25)*dxi); + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi); + Real dudy = (vel(i,j,0,0) - vel(i,j-1,0,0))*dyi; + Real dvdy = (vel(i,j,0,1) - vel(i,j-1,0,1))*dyi; + fy(i,j,0,0) = dudx; + fy(i,j,0,1) = dvdx; + fy(i,j,0,2) = dudy; + fy(i,j,0,3) = dvdy; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = (vel(i,j,0,0) - vel(i-1,j,0,0))*dxi; + Real dvdx = (vel(i,j,0,1) - vel(i-1,j,0,1))*dxi; + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + fx(i,j,0,0) = dudx; + fx(i,j,0,1) = dvdx; + fx(i,j,0,2) = dudy; + fx(i,j,0,3) = dvdy; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_vel_grads_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + + int k = 0; + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); Real dudy = (vel(i,j,0,0) - vel(i,j-1,0,0))*dyi; Real dvdy = (vel(i,j,0,1) - vel(i,j-1,0,1))*dyi; fy(i,j,0,0) = dudx; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H index a4a4c7df9ef..a5de05a385e 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H @@ -6,6 +6,643 @@ namespace amrex { +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xlo_ylo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxlo, + Array4 const& mylo, + Array4 const& bcvalxlo, + Array4 const& bcvalylo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xlo_domain, bool ylo_domain) noexcept +{ + if (mxlo(i,j,k) != BndryData::covered && (!xlo_domain || !ylo_domain)) { + bool x_interior = mylo(i+1,j ,k) == BndryData::covered; + bool x_exterior = mylo(i+1,j ,k) == BndryData::not_covered; + bool y_interior = mxlo(i ,j+1,k) == BndryData::covered; + bool y_exterior = mxlo(i ,j+1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || ylo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xhi_ylo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxhi, + Array4 const& mylo, + Array4 const& bcvalxhi, + Array4 const& bcvalylo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xhi_domain, bool ylo_domain) noexcept +{ + if (mxhi(i,j,k) != BndryData::covered && (!xhi_domain || !ylo_domain)) { + bool x_interior = mylo(i-1,j ,k) == BndryData::covered; + bool x_exterior = mylo(i-1,j ,k) == BndryData::not_covered; + bool y_interior = mxhi(i ,j+1,k) == BndryData::covered; + bool y_exterior = mxhi(i ,j+1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || ylo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xlo_yhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxlo, + Array4 const& myhi, + Array4 const& bcvalxlo, + Array4 const& bcvalyhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xlo_domain, bool yhi_domain) noexcept +{ + if (mxlo(i,j,k) != BndryData::covered && (!xlo_domain || !yhi_domain)) { + bool x_interior = myhi(i+1,j ,k) == BndryData::covered; + bool x_exterior = myhi(i+1,j ,k) == BndryData::not_covered; + bool y_interior = mxlo(i ,j-1,k) == BndryData::covered; + bool y_exterior = mxlo(i ,j-1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || yhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xhi_yhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxhi, + Array4 const& myhi, + Array4 const& bcvalxhi, + Array4 const& bcvalyhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xhi_domain, bool yhi_domain) noexcept +{ + if (mxhi(i,j,k) != BndryData::covered && (!xhi_domain || !yhi_domain)) { + bool x_interior = myhi(i-1,j ,k) == BndryData::covered; + bool x_exterior = myhi(i-1,j ,k) == BndryData::not_covered; + bool y_interior = mxhi(i ,j-1,k) == BndryData::covered; + bool y_exterior = mxhi(i ,j-1,k) == BndryData::not_covered; + if ((x_interior && y_interior) || (x_exterior && y_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (y_interior || yhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xlo_zlo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxlo, + Array4 const& mzlo, + Array4 const& bcvalxlo, + Array4 const& bcvalzlo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xlo_domain, bool zlo_domain) noexcept +{ + if (mxlo(i,j,k) != BndryData::covered && (!xlo_domain || !zlo_domain)) { + bool x_interior = mzlo(i+1,j,k ) == BndryData::covered; + bool x_exterior = mzlo(i+1,j,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j,k+1) == BndryData::covered; + bool z_exterior = mxlo(i ,j,k+1) == BndryData::not_covered; + if ((x_interior && z_interior) || (x_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (z_interior || zlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xhi_zlo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxhi, + Array4 const& mzlo, + Array4 const& bcvalxhi, + Array4 const& bcvalzlo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xhi_domain, bool zlo_domain) noexcept +{ + if (mxhi(i,j,k) != BndryData::covered && (!xhi_domain || !zlo_domain)) { + bool x_interior = mzlo(i-1,j,k ) == BndryData::covered; + bool x_exterior = mzlo(i-1,j,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j,k+1) == BndryData::covered; + bool z_exterior = mxhi(i ,j,k+1) == BndryData::not_covered; + if ((x_interior && z_interior) || (x_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (z_interior || zlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xlo_zhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxlo, + Array4 const& mzhi, + Array4 const& bcvalxlo, + Array4 const& bcvalzhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xlo_domain, bool zhi_domain) noexcept +{ + if (mxlo(i,j,k) != BndryData::covered && (!xlo_domain || !zhi_domain)) { + bool x_interior = mzhi(i+1,j,k ) == BndryData::covered; + bool x_exterior = mzhi(i+1,j,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j,k-1) == BndryData::covered; + bool z_exterior = mxlo(i ,j,k-1) == BndryData::not_covered; + if ((x_interior && z_interior) || (x_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } + } else if (z_interior || zhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_xhi_zhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mxhi, + Array4 const& mzhi, + Array4 const& bcvalxhi, + Array4 const& bcvalzhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool xhi_domain, bool zhi_domain) noexcept +{ + if (mxhi(i,j,k) != BndryData::covered && (!xhi_domain || !zhi_domain)) { + bool x_interior = mzhi(i-1,j,k ) == BndryData::covered; + bool x_exterior = mzhi(i-1,j,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j,k-1) == BndryData::covered; + bool z_exterior = mxhi(i ,j,k-1) == BndryData::not_covered; + if ((x_interior && z_interior) || (x_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (x_interior || xhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } + } else if (z_interior || zhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_ylo_zlo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mylo, + Array4 const& mzlo, + Array4 const& bcvalylo, + Array4 const& bcvalzlo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool ylo_domain, bool zlo_domain) noexcept +{ + if (mylo(i,j,k) != BndryData::covered && (!ylo_domain || !zlo_domain)) { + bool y_interior = mzlo(i,j+1,k ) == BndryData::covered; + bool y_exterior = mzlo(i,j+1,k ) == BndryData::not_covered; + bool z_interior = mylo(i,j ,k+1) == BndryData::covered; + bool z_exterior = mylo(i,j ,k+1) == BndryData::not_covered; + if ((y_interior && z_interior) || (y_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (y_interior || ylo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } + } else if (z_interior || zlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_yhi_zlo (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& myhi, + Array4 const& mzlo, + Array4 const& bcvalyhi, + Array4 const& bcvalzlo, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool yhi_domain, bool zlo_domain) noexcept +{ + if (myhi(i,j,k) != BndryData::covered && (!yhi_domain || !zlo_domain)) { + bool y_interior = mzlo(i,j-1,k ) == BndryData::covered; + bool y_exterior = mzlo(i,j-1,k ) == BndryData::not_covered; + bool z_interior = myhi(i,j ,k+1) == BndryData::covered; + bool z_exterior = myhi(i,j ,k+1) == BndryData::not_covered; + if ((y_interior && z_interior) || (y_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (y_interior || yhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } + } else if (z_interior || zlo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_ylo_zhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& mylo, + Array4 const& mzhi, + Array4 const& bcvalylo, + Array4 const& bcvalzhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool ylo_domain, bool zhi_domain) noexcept +{ + if (mylo(i,j,k) != BndryData::covered && (!ylo_domain || !zhi_domain)) { + bool y_interior = mzhi(i,j+1,k ) == BndryData::covered; + bool y_exterior = mzhi(i,j+1,k ) == BndryData::not_covered; + bool z_interior = mylo(i,j ,k-1) == BndryData::covered; + bool z_exterior = mylo(i,j ,k-1) == BndryData::not_covered; + if ((y_interior && z_interior) || (y_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (y_interior || ylo_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } + } else if (z_interior || zhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges_yhi_zhi (int const i, int const j, int const k, Dim3 const& blen, + Array4 const& vel, + Array4 const& myhi, + Array4 const& mzhi, + Array4 const& bcvalyhi, + Array4 const& bcvalzhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + bool yhi_domain, bool zhi_domain) noexcept +{ + if (myhi(i,j,k) != BndryData::covered && (!yhi_domain || !zhi_domain)) { + bool y_interior = mzhi(i,j-1,k ) == BndryData::covered; + bool y_exterior = mzhi(i,j-1,k ) == BndryData::not_covered; + bool z_interior = myhi(i,j ,k-1) == BndryData::covered; + bool z_exterior = myhi(i,j ,k-1) == BndryData::not_covered; + if ((y_interior && z_interior) || (y_exterior && z_exterior)) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } + } else if (y_interior || yhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } + } else if (z_interior || zhi_domain) { + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + } + } + } +} + +#ifdef AMREX_USE_EB AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mltensor_fill_corners (int icorner, Box const& vbox, // vbox: the valid box Array4 const& vel, @@ -21,495 +658,680 @@ void mltensor_fill_corners (int icorner, Box const& vbox, // vbox: the valid box Array4 const& bcvalxhi, Array4 const& bcvalyhi, Array4 const& bcvalzhi, - GpuArray const& bct, - GpuArray const& bcl, + Array2D const& bct, + Array2D const& bcl, int inhomog, int maxorder, - GpuArray const& dxinv, Box const& domain) noexcept + GpuArray const& dxinv, + Dim3 const& dlo, Dim3 const& dhi) noexcept { - constexpr int oxlo = 0; - constexpr int oylo = 1; - constexpr int ozlo = 2; - constexpr int oxhi = 3; - constexpr int oyhi = 4; - constexpr int ozhi = 5; - constexpr int xdir = 0; - constexpr int ydir = 1; - constexpr int zdir = 2; const auto blen = amrex::length(vbox); const auto vlo = amrex::lbound(vbox); const auto vhi = amrex::ubound(vbox); - const auto dlo = amrex::lbound(domain); - const auto dhi = amrex::ubound(domain); + bool xlo_domain = (vlo.x == dlo.x); + bool ylo_domain = (vlo.y == dlo.y); + bool zlo_domain = (vlo.z == dlo.z); + bool xhi_domain = (vhi.x == dhi.x); + bool yhi_domain = (vhi.y == dhi.y); + bool zhi_domain = (vhi.z == dhi.z); + for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { switch (icorner) { case 0: { // xlo & ylo & zlo - Box bx = amrex::adjCellLo(amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),ydir,1),zdir,1); - if (vlo.x == dlo.x && vlo.y == dlo.y && vlo.z == dlo.z) { - vel (vlo.x-1,vlo.y-1,vlo.z-1,icomp) - = vel(vlo.x-1,vlo.y ,vlo.z ,icomp) - + vel(vlo.x ,vlo.y-1,vlo.z ,icomp) - + vel(vlo.x ,vlo.y ,vlo.z-1,icomp) - - vel(vlo.x ,vlo.y ,vlo.z ,icomp) * Real(2.0); - } else if (vlo.x == dlo.x && vlo.y == dlo.y) { - vel (vlo.x-1,vlo.y-1,vlo.z-1,icomp) - = vel(vlo.x-1,vlo.y ,vlo.z-1,icomp) - + vel(vlo.x ,vlo.y-1,vlo.z-1,icomp) - - vel(vlo.x ,vlo.y ,vlo.z-1,icomp); - } else if (vlo.x == dlo.x && vlo.z == dlo.z) { - vel (vlo.x-1,vlo.y-1,vlo.z-1,icomp) - = vel(vlo.x-1,vlo.y-1,vlo.z ,icomp) - + vel(vlo.x ,vlo.y-1,vlo.z-1,icomp) - - vel(vlo.x ,vlo.y-1,vlo.z ,icomp); - } else if (vlo.y == dlo.y && vlo.z == dlo.z) { - vel (vlo.x-1,vlo.y-1,vlo.z-1,icomp) - = vel(vlo.x-1,vlo.y-1,vlo.z ,icomp) - + vel(vlo.x-1,vlo.y ,vlo.z-1,icomp) - - vel(vlo.x-1,vlo.y ,vlo.z ,icomp); - } else if (vlo.x == dlo.x) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vlo.y-1,vlo.z-1) != BndryData::covered) { - if (mylo(vlo.x,vlo.y-1,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vlo.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + int i = vlo.x-1; + int j = vlo.y-1; + int k = vlo.z-1; + if (mxlo(i,j,k) != BndryData::covered && + (!xlo_domain || !ylo_domain || !zlo_domain)) { + bool x_interior = mylo(i+1,j ,k ) == BndryData::covered; + bool x_exterior = mylo(i+1,j ,k ) == BndryData::not_covered; + bool y_interior = mxlo(i ,j+1,k ) == BndryData::covered; + bool y_exterior = mxlo(i ,j+1,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j ,k+1) == BndryData::covered; + bool z_exterior = mxlo(i ,j ,k+1) == BndryData::not_covered; + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); } } break; } case 1: { // xhi & ylo & zlo - Box bx = amrex::adjCellLo(amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),ydir,1),zdir,1); - if (vhi.x == dhi.x && vlo.y == dlo.y && vlo.z == dlo.z) { - vel (vhi.x+1,vlo.y-1,vlo.z-1,icomp) - = vel(vhi.x+1,vlo.y ,vlo.z ,icomp) - + vel(vhi.x ,vlo.y-1,vlo.z ,icomp) - + vel(vhi.x ,vlo.y ,vlo.z-1,icomp) - - vel(vhi.x ,vlo.y ,vlo.z ,icomp) * Real(2.0); - } else if (vhi.x == dhi.x && vlo.y == dlo.y) { - vel (vhi.x+1,vlo.y-1,vlo.z-1,icomp) - = vel(vhi.x+1,vlo.y ,vlo.z-1,icomp) - + vel(vhi.x ,vlo.y-1,vlo.z-1,icomp) - - vel(vhi.x ,vlo.y ,vlo.z-1,icomp); - } else if (vhi.x == dhi.x && vlo.z == dlo.z) { - vel (vhi.x+1,vlo.y-1,vlo.z-1,icomp) - = vel(vhi.x+1,vlo.y-1,vlo.z ,icomp) - + vel(vhi.x ,vlo.y-1,vlo.z-1,icomp) - - vel(vhi.x ,vlo.y-1,vlo.z ,icomp); - } else if (vlo.y == dlo.y && vlo.z == dlo.z) { - vel (vhi.x+1,vlo.y-1,vlo.z-1,icomp) - = vel(vhi.x+1,vlo.y-1,vlo.z ,icomp) - + vel(vhi.x+1,vlo.y ,vlo.z-1,icomp) - - vel(vhi.x+1,vlo.y ,vlo.z ,icomp); - } else if (vhi.x == dhi.x) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vlo.y-1,vlo.z-1) != BndryData::covered) { - if (mylo(vhi.x,vlo.y-1,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vlo.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + int i = vhi.x+1; + int j = vlo.y-1; + int k = vlo.z-1; + bool x_interior = mylo(i-1,j ,k ) == BndryData::covered; + bool x_exterior = mylo(i-1,j ,k ) == BndryData::not_covered; + bool y_interior = mxhi(i ,j+1,k ) == BndryData::covered; + bool y_exterior = mxhi(i ,j+1,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j ,k+1) == BndryData::covered; + bool z_exterior = mxhi(i ,j ,k+1) == BndryData::not_covered; + if (mxhi(i,j,k) != BndryData::covered && + (!xhi_domain || !ylo_domain || !zlo_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); } } break; } case 2: { // xlo & yhi & zlo - Box bx = amrex::adjCellLo(amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),ydir,1),zdir,1); - if (vlo.x == dlo.x && vhi.y == dhi.y && vlo.z == dlo.z) { - vel (vlo.x-1,vhi.y+1,vlo.z-1,icomp) - = vel(vlo.x-1,vhi.y ,vlo.z ,icomp) - + vel(vlo.x ,vhi.y+1,vlo.z ,icomp) - + vel(vlo.x ,vhi.y ,vlo.z-1,icomp) - - vel(vlo.x ,vhi.y ,vlo.z ,icomp) * Real(2.0); - } else if (vlo.x == dlo.x && vhi.y == dhi.y) { - vel (vlo.x-1,vhi.y+1,vlo.z-1,icomp) - = vel(vlo.x-1,vhi.y ,vlo.z-1,icomp) - + vel(vlo.x ,vhi.y+1,vlo.z-1,icomp) - - vel(vlo.x ,vhi.y ,vlo.z-1,icomp); - } else if (vlo.x == dlo.x && vlo.z == dlo.z) { - vel (vlo.x-1,vhi.y+1,vlo.z-1,icomp) - = vel(vlo.x-1,vhi.y+1,vlo.z ,icomp) - + vel(vlo.x ,vhi.y+1,vlo.z-1,icomp) - - vel(vlo.x ,vhi.y+1,vlo.z ,icomp); - } else if (vhi.y == dhi.y && vlo.z == dlo.z) { - vel (vlo.x-1,vhi.y+1,vlo.z-1,icomp) - = vel(vlo.x-1,vhi.y+1,vlo.z ,icomp) - + vel(vlo.x-1,vhi.y ,vlo.z-1,icomp) - - vel(vlo.x-1,vhi.y ,vlo.z ,icomp); - } else if (vlo.x == dlo.x) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vhi.y+1,vlo.z-1) != BndryData::covered) { - if (myhi(vlo.x,vhi.y+1,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vhi.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + int i = vlo.x-1; + int j = vhi.y+1; + int k = vlo.z-1; + bool x_interior = myhi(i+1,j ,k ) == BndryData::covered; + bool x_exterior = myhi(i+1,j ,k ) == BndryData::not_covered; + bool y_interior = mxlo(i ,j-1,k ) == BndryData::covered; + bool y_exterior = mxlo(i ,j-1,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j ,k+1) == BndryData::covered; + bool z_exterior = mxlo(i ,j ,k+1) == BndryData::not_covered; + if (mxlo(i,j,k) != BndryData::covered && + (!xlo_domain || !yhi_domain || !zlo_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); } } break; } case 3: { // xhi & yhi & zlo - Box bx = amrex::adjCellLo(amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),ydir,1),zdir,1); - if (vhi.x == dhi.x && vhi.y == dhi.y && vlo.z == dlo.z) { - vel (vhi.x+1,vhi.y+1,vlo.z-1,icomp) - = vel(vhi.x+1,vhi.y ,vlo.z ,icomp) - + vel(vhi.x ,vhi.y+1,vlo.z ,icomp) - + vel(vhi.x ,vhi.y ,vlo.z-1,icomp) - - vel(vhi.x ,vhi.y ,vlo.z ,icomp) * Real(2.0); - } else if (vhi.x == dhi.x && vhi.y == dhi.y) { - vel (vhi.x+1,vhi.y+1,vlo.z-1,icomp) - = vel(vhi.x+1,vhi.y ,vlo.z-1,icomp) - + vel(vhi.x ,vhi.y+1,vlo.z-1,icomp) - - vel(vhi.x ,vhi.y ,vlo.z-1,icomp); - } else if (vhi.x == dhi.x && vlo.z == dlo.z) { - vel (vhi.x+1,vhi.y+1,vlo.z-1,icomp) - = vel(vhi.x+1,vhi.y+1,vlo.z ,icomp) - + vel(vhi.x ,vhi.y+1,vlo.z-1,icomp) - - vel(vhi.x ,vhi.y+1,vlo.z ,icomp); - } else if (vhi.y == dhi.y && vlo.z == dlo.z) { - vel (vhi.x+1,vhi.y+1,vlo.z-1,icomp) - = vel(vhi.x+1,vhi.y+1,vlo.z ,icomp) - + vel(vhi.x+1,vhi.y ,vlo.z-1,icomp) - - vel(vhi.x+1,vhi.y ,vlo.z ,icomp); - } else if (vhi.x == dhi.x) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vhi.y+1,vlo.z-1) != BndryData::covered) { - if (myhi(vhi.x,vhi.y+1,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vhi.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + int i = vhi.x+1; + int j = vhi.y+1; + int k = vlo.z-1; + bool x_interior = myhi(i-1,j ,k ) == BndryData::covered; + bool x_exterior = myhi(i-1,j ,k ) == BndryData::not_covered; + bool y_interior = mxhi(i ,j-1,k ) == BndryData::covered; + bool y_exterior = mxhi(i ,j-1,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j ,k+1) == BndryData::covered; + bool z_exterior = mxhi(i ,j ,k+1) == BndryData::not_covered; + if (mxhi(i,j,k) != BndryData::covered && + (!xhi_domain || !yhi_domain || !zlo_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::low, i,j,k, blen.z, vel, mzlo, + bct(Orientation::zlo(), icomp), + bcl(Orientation::zlo(), icomp), + bcvalzlo, maxorder, dxinv[2], inhomog, icomp); } } break; } case 4: { // xlo & ylo & zhi - Box bx = amrex::adjCellHi(amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),ydir,1),zdir,1); - if (vlo.x == dlo.x && vlo.y == dlo.y && vhi.z == dhi.z) { - vel (vlo.x-1, vlo.y-1, vhi.z+1,icomp) - = vel(vlo.x-1, vlo.y , vhi.z ,icomp) - + vel(vlo.x , vlo.y-1, vhi.z ,icomp) - + vel(vlo.x , vlo.y , vhi.z+1,icomp) - - vel(vlo.x , vlo.y , vhi.z ,icomp) * Real(2.0); - } else if (vlo.x == dlo.x && vlo.y == dlo.y) { - vel (vlo.x-1, vlo.y-1, vhi.z+1,icomp) - = vel(vlo.x-1, vlo.y , vhi.z+1,icomp) - + vel(vlo.x , vlo.y-1, vhi.z+1,icomp) - - vel(vlo.x , vlo.y , vhi.z+1,icomp); - } else if (vlo.x == dlo.x && vhi.z == dhi.z) { - vel (vlo.x-1, vlo.y-1, vhi.z+1,icomp) - = vel(vlo.x-1, vlo.y-1, vhi.z ,icomp) - + vel(vlo.x , vlo.y-1, vhi.z+1,icomp) - - vel(vlo.x , vlo.y-1, vhi.z ,icomp); - } else if (vlo.y == dlo.y && vhi.z == dhi.z) { - vel (vlo.x-1, vlo.y-1, vhi.z+1,icomp) - = vel(vlo.x-1, vlo.y-1, vhi.z ,icomp) - + vel(vlo.x-1, vlo.y , vhi.z+1,icomp) - - vel(vlo.x-1, vlo.y , vhi.z ,icomp); - } else if (vlo.x == dlo.x) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vlo.y-1,vhi.z+1) != BndryData::covered) { - if (mylo(vlo.x,vlo.y-1,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vlo.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + int i = vlo.x-1; + int j = vlo.y-1; + int k = vhi.z+1; + bool x_interior = mylo(i+1,j ,k ) == BndryData::covered; + bool x_exterior = mylo(i+1,j ,k ) == BndryData::not_covered; + bool y_interior = mxlo(i ,j+1,k ) == BndryData::covered; + bool y_exterior = mxlo(i ,j+1,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j ,k-1) == BndryData::covered; + bool z_exterior = mxlo(i ,j ,k-1) == BndryData::not_covered; + if (mxlo(i,j,k) != BndryData::covered && + (!xlo_domain || !ylo_domain || !zhi_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); } } break; } case 5: { // xhi & ylo & zhi - Box bx = amrex::adjCellHi(amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),ydir,1),zdir,1); - if (vhi.x == dhi.x && vlo.y == dlo.y && vhi.z == dhi.z) { - vel (vhi.x+1,vlo.y-1,vhi.z+1,icomp) - = vel(vhi.x+1,vlo.y ,vhi.z ,icomp) - + vel(vhi.x ,vlo.y-1,vhi.z ,icomp) - + vel(vhi.x ,vlo.y ,vhi.z+1,icomp) - - vel(vhi.x ,vlo.y ,vhi.z ,icomp) * Real(2.0); - } else if (vhi.x == dhi.x && vlo.y == dlo.y) { - vel (vhi.x+1,vlo.y-1,vhi.z+1,icomp) - = vel(vhi.x+1,vlo.y ,vhi.z+1,icomp) - + vel(vhi.x ,vlo.y-1,vhi.z+1,icomp) - - vel(vhi.x ,vlo.y ,vhi.z+1,icomp); - } else if (vhi.x == dhi.x && vhi.z == dhi.z) { - vel (vhi.x+1,vlo.y-1,vhi.z+1,icomp) - = vel(vhi.x+1,vlo.y-1,vhi.z ,icomp) - + vel(vhi.x ,vlo.y-1,vhi.z+1,icomp) - - vel(vhi.x ,vlo.y-1,vhi.z ,icomp); - } else if (vlo.y == dlo.y && vhi.z == dhi.z) { - vel (vhi.x+1,vlo.y-1,vhi.z+1,icomp) - = vel(vhi.x+1,vlo.y-1,vhi.z ,icomp) - + vel(vhi.x+1,vlo.y ,vhi.z+1,icomp) - - vel(vhi.x+1,vlo.y ,vhi.z ,icomp); - } else if (vhi.x == dhi.x) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vlo.y-1,vhi.z+1) != BndryData::covered) { - if (mylo(vhi.x,vlo.y-1,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vlo.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + int i = vhi.x+1; + int j = vlo.y-1; + int k = vhi.z+1; + bool x_interior = mylo(i-1,j ,k ) == BndryData::covered; + bool x_exterior = mylo(i-1,j ,k ) == BndryData::not_covered; + bool y_interior = mxhi(i ,j+1,k ) == BndryData::covered; + bool y_exterior = mxhi(i ,j+1,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j ,k-1) == BndryData::covered; + bool z_exterior = mxhi(i ,j ,k-1) == BndryData::not_covered; + if (mxhi(i,j,k) != BndryData::covered && + (!xhi_domain || !ylo_domain || !zhi_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::low, i,j,k, blen.y, vel, mylo, + bct(Orientation::ylo(), icomp), + bcl(Orientation::ylo(), icomp), + bcvalylo, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); } } break; } case 6: { // xlo & yhi & zhi - Box bx = amrex::adjCellHi(amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),ydir,1),zdir,1); - if (vlo.x == dlo.x && vhi.y == dhi.y && vhi.z == dhi.z) { - vel (vlo.x-1,vhi.y+1,vhi.z+1,icomp) - = vel(vlo.x-1,vhi.y ,vhi.z ,icomp) - + vel(vlo.x ,vhi.y+1,vhi.z ,icomp) - + vel(vlo.x ,vhi.y ,vhi.z+1,icomp) - - vel(vlo.x ,vhi.y ,vhi.z ,icomp) * Real(2.0); - } else if (vlo.x == dlo.x && vhi.y == dhi.y) { - vel (vlo.x-1,vhi.y+1,vhi.z+1,icomp) - = vel(vlo.x-1,vhi.y ,vhi.z+1,icomp) - + vel(vlo.x ,vhi.y+1,vhi.z+1,icomp) - - vel(vlo.x ,vhi.y ,vhi.z+1,icomp); - } else if (vlo.x == dlo.x && vhi.z == dhi.z) { - vel (vlo.x-1,vhi.y+1,vhi.z+1,icomp) - = vel(vlo.x-1,vhi.y+1,vhi.z ,icomp) - + vel(vlo.x ,vhi.y+1,vhi.z+1,icomp) - - vel(vlo.x ,vhi.y+1,vhi.z ,icomp); - } else if (vhi.y == dhi.y && vhi.z == dhi.z) { - vel (vlo.x-1,vhi.y+1,vhi.z+1,icomp) - = vel(vlo.x-1,vhi.y+1,vhi.z ,icomp) - + vel(vlo.x-1,vhi.y ,vhi.z+1,icomp) - - vel(vlo.x-1,vhi.y ,vhi.z ,icomp); - } else if (vlo.x == dlo.x) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vhi.y+1,vhi.z+1) != BndryData::covered) { - if (myhi(vlo.x,vhi.y+1,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxlo(vlo.x-1,vhi.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + int i = vlo.x-1; + int j = vhi.y+1; + int k = vhi.z+1; + bool x_interior = myhi(i+1,j ,k ) == BndryData::covered; + bool x_exterior = myhi(i+1,j ,k ) == BndryData::not_covered; + bool y_interior = mxlo(i ,j-1,k ) == BndryData::covered; + bool y_exterior = mxlo(i ,j-1,k ) == BndryData::not_covered; + bool z_interior = mxlo(i ,j ,k-1) == BndryData::covered; + bool z_exterior = mxlo(i ,j ,k-1) == BndryData::not_covered; + if (mxlo(i,j,k) != BndryData::covered && + (!xlo_domain || !yhi_domain || !zhi_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::low, i,j,k, blen.x, vel, mxlo, + bct(Orientation::xlo(), icomp), + bcl(Orientation::xlo(), icomp), + bcvalxlo, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); } } break; } case 7: { // xhi & yhi & zhi - Box bx = amrex::adjCellHi(amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),ydir,1),zdir,1); - if (vhi.x == dhi.x && vhi.y == dhi.y && vhi.z == dhi.z) { - vel (vhi.x+1,vhi.y+1,vhi.z+1,icomp) - = vel(vhi.x+1,vhi.y ,vhi.z ,icomp) - + vel(vhi.x ,vhi.y+1,vhi.z ,icomp) - + vel(vhi.x ,vhi.y ,vhi.z+1,icomp) - - vel(vhi.x ,vhi.y ,vhi.z ,icomp) * Real(2.0); - } else if (vhi.x == dhi.x && vhi.y == dhi.y) { - vel (vhi.x+1,vhi.y+1,vhi.z+1,icomp) - = vel(vhi.x+1,vhi.y ,vhi.z+1,icomp) - + vel(vhi.x ,vhi.y+1,vhi.z+1,icomp) - - vel(vhi.x ,vhi.y ,vhi.z+1,icomp); - } else if (vhi.x == dhi.x && vhi.z == dhi.z) { - vel (vhi.x+1,vhi.y+1,vhi.z+1,icomp) - = vel(vhi.x+1,vhi.y+1,vhi.z ,icomp) - + vel(vhi.x ,vhi.y+1,vhi.z+1,icomp) - - vel(vhi.x ,vhi.y+1,vhi.z ,icomp); - } else if (vhi.y == dhi.y && vhi.z == dhi.z) { - vel (vhi.x+1,vhi.y+1,vhi.z+1,icomp) - = vel(vhi.x+1,vhi.y+1,vhi.z ,icomp) - + vel(vhi.x+1,vhi.y ,vhi.z+1,icomp) - - vel(vhi.x+1,vhi.y ,vhi.z ,icomp); - } else if (vhi.x == dhi.x) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vhi.y+1,vhi.z+1) != BndryData::covered) { - if (myhi(vhi.x,vhi.y+1,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (mxhi(vhi.x+1,vhi.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + int i = vhi.x+1; + int j = vhi.y+1; + int k = vhi.z+1; + bool x_interior = myhi(i-1,j ,k ) == BndryData::covered; + bool x_exterior = myhi(i-1,j ,k ) == BndryData::not_covered; + bool y_interior = mxhi(i ,j-1,k ) == BndryData::covered; + bool y_exterior = mxhi(i ,j-1,k ) == BndryData::not_covered; + bool z_interior = mxhi(i ,j ,k-1) == BndryData::covered; + bool z_exterior = mxhi(i ,j ,k-1) == BndryData::not_covered; + if (mxhi(i,j,k) != BndryData::covered && + (!xhi_domain || !yhi_domain || !zhi_domain)) { + if ((x_interior && y_interior && z_interior) || + (x_exterior && y_exterior && z_exterior)) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + tmp += vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = Real(1./3.)*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && y_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior && z_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (y_interior && z_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + Real tmp = vel(i,j,k,icomp); + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); + vel(i,j,k,icomp) = 0.5_rt*(tmp+vel(i,j,k,icomp)); + } else if (x_interior) { + mllinop_apply_bc_x(Orientation::high, i,j,k, blen.x, vel, mxhi, + bct(Orientation::xhi(), icomp), + bcl(Orientation::xhi(), icomp), + bcvalxhi, maxorder, dxinv[0], inhomog, icomp); + } else if (y_interior) { + mllinop_apply_bc_y(Orientation::high, i,j,k, blen.y, vel, myhi, + bct(Orientation::yhi(), icomp), + bcl(Orientation::yhi(), icomp), + bcvalyhi, maxorder, dxinv[1], inhomog, icomp); + } else if (z_interior) { + mllinop_apply_bc_z(Orientation::high, i,j,k, blen.z, vel, mzhi, + bct(Orientation::zhi(), icomp), + bcl(Orientation::zhi(), icomp), + bcvalzhi, maxorder, dxinv[2], inhomog, icomp); } } break; @@ -518,9 +1340,10 @@ void mltensor_fill_corners (int icorner, Box const& vbox, // vbox: the valid box } } } +#endif -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void mltensor_fill_edges (int iedge, Box const& vbox, // vbox: the valid box +inline +void mltensor_fill_edges (Box const& vbox, // vbox: the valid box Array4 const& vel, Array4 const& mxlo, Array4 const& mylo, @@ -534,522 +1357,486 @@ void mltensor_fill_edges (int iedge, Box const& vbox, // vbox: the valid box Array4 const& bcvalxhi, Array4 const& bcvalyhi, Array4 const& bcvalzhi, - GpuArray const& bct, - GpuArray const& bcl, + Array2D const& bct, + Array2D const& bcl, int inhomog, int maxorder, - GpuArray const& dxinv, Box const& domain) noexcept + GpuArray const& dxinv, + Dim3 const& dlo, Dim3 const& dhi) noexcept + { - constexpr int oxlo = 0; - constexpr int oylo = 1; - constexpr int ozlo = 2; - constexpr int oxhi = 3; - constexpr int oyhi = 4; - constexpr int ozhi = 5; - constexpr int xdir = 0; - constexpr int ydir = 1; - constexpr int zdir = 2; const auto blen = amrex::length(vbox); const auto vlo = amrex::lbound(vbox); const auto vhi = amrex::ubound(vbox); - const auto dlo = amrex::lbound(domain); - const auto dhi = amrex::ubound(domain); - for (int icomp = 0; icomp < AMREX_SPACEDIM; ++icomp) { - switch (iedge) { - case 0: { - // xlo & ylo - if (vlo.x == dlo.x && vlo.y == dlo.y) { - for (int k = vlo.z; k <= vhi.z; ++k) { - vel (vlo.x-1,vlo.y-1,k,icomp) - = vel(vlo.x ,vlo.y-1,k,icomp) - + vel(vlo.x-1,vlo.y ,k,icomp) - - vel(vlo.x ,vlo.y ,k,icomp); - } - } else if (vlo.x == dlo.x) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - for (int k = vlo.z; k <= vhi.z; ++k) { - if (mxlo(vlo.x-1,vlo.y-1,k) != BndryData::covered) { - Box bx(IntVect(vlo.x-1,vlo.y-1,k),IntVect(vlo.x-1,vlo.y-1,k)); - if (mylo(vlo.x,vlo.y-1,k) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } - } - } - } - break; + bool xlo_domain = (vlo.x == dlo.x); + bool ylo_domain = (vlo.y == dlo.y); + bool zlo_domain = (vlo.z == dlo.z); + bool xhi_domain = (vhi.x == dhi.x); + bool yhi_domain = (vhi.y == dhi.y); + bool zhi_domain = (vhi.z == dhi.z); + + for (int k = vlo.z; k <= vhi.z; ++k) { + mltensor_fill_edges_xlo_ylo(vlo.x-1, vlo.y-1, k, blen, vel, mxlo, mylo, bcvalxlo, bcvalylo, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, ylo_domain); + mltensor_fill_edges_xhi_ylo(vhi.x+1, vlo.y-1, k, blen, vel, mxhi, mylo, bcvalxhi, bcvalylo, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, ylo_domain); + mltensor_fill_edges_xlo_yhi(vlo.x-1, vhi.y+1, k, blen, vel, mxlo, myhi, bcvalxlo, bcvalyhi, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, yhi_domain); + mltensor_fill_edges_xhi_yhi(vhi.x+1, vhi.y+1, k, blen, vel, mxhi, myhi, bcvalxhi, bcvalyhi, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, yhi_domain); + } + + for (int j = vlo.y; j <= vhi.y; ++j) { + mltensor_fill_edges_xlo_zlo(vlo.x-1, j, vlo.z-1, blen, vel, mxlo, mzlo, bcvalxlo, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, zlo_domain); + mltensor_fill_edges_xhi_zlo(vhi.x+1, j, vlo.z-1, blen, vel, mxhi, mzlo, bcvalxhi, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, zlo_domain); + mltensor_fill_edges_xlo_zhi(vlo.x-1, j, vhi.z+1, blen, vel, mxlo, mzhi, bcvalxlo, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, zhi_domain); + mltensor_fill_edges_xhi_zhi(vhi.x+1, j, vhi.z+1, blen, vel, mxhi, mzhi, bcvalxhi, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, zhi_domain); + } + + for (int i = vlo.x; i <= vhi.x; ++i) { + mltensor_fill_edges_ylo_zlo(i, vlo.y-1, vlo.z-1, blen, vel, mylo, mzlo, bcvalylo, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, ylo_domain, zlo_domain); + mltensor_fill_edges_yhi_zlo(i, vhi.y+1, vlo.z-1, blen, vel, myhi, mzlo, bcvalyhi, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, yhi_domain, zlo_domain); + mltensor_fill_edges_ylo_zhi(i, vlo.y-1, vhi.z+1, blen, vel, mylo, mzhi, bcvalylo, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, ylo_domain, zhi_domain); + mltensor_fill_edges_yhi_zhi(i, vhi.y+1, vhi.z+1, blen, vel, myhi, mzhi, bcvalyhi, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, yhi_domain, zhi_domain); + } +} + +#ifdef AMREX_USE_GPU +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void mltensor_fill_edges (int const bid, int const tid, int const bdim, + Box const& vbox, // vbox: the valid box + Array4 const& vel, + Array4 const& mxlo, + Array4 const& mylo, + Array4 const& mzlo, + Array4 const& mxhi, + Array4 const& myhi, + Array4 const& mzhi, + Array4 const& bcvalxlo, + Array4 const& bcvalylo, + Array4 const& bcvalzlo, + Array4 const& bcvalxhi, + Array4 const& bcvalyhi, + Array4 const& bcvalzhi, + Array2D const& bct, + Array2D const& bcl, + int inhomog, int maxorder, + GpuArray const& dxinv, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const auto blen = amrex::length(vbox); + const auto vlo = amrex::lbound(vbox); + const auto vhi = amrex::ubound(vbox); + bool xlo_domain = (vlo.x == dlo.x); + bool ylo_domain = (vlo.y == dlo.y); + bool zlo_domain = (vlo.z == dlo.z); + bool xhi_domain = (vhi.x == dhi.x); + bool yhi_domain = (vhi.y == dhi.y); + bool zhi_domain = (vhi.z == dhi.z); + if (bid == 0) { + for (int k = vlo.z + tid; k <= vhi.z; k += bdim) { + mltensor_fill_edges_xlo_ylo(vlo.x-1, vlo.y-1, k, blen, vel, mxlo, mylo, bcvalxlo, bcvalylo, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, ylo_domain); } - case 1: { - // xhi & ylo - if (vhi.x == dhi.x && vlo.y == dlo.y) { - for (int k = vlo.z; k <= vhi.z; ++k) { - vel (vhi.x+1,vlo.y-1,k,icomp) - = vel(vhi.x ,vlo.y-1,k,icomp) - + vel(vhi.x+1,vlo.y ,k,icomp) - - vel(vhi.x ,vlo.y ,k,icomp); - } - } else if (vhi.x == dhi.x) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.y == dlo.y) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - for (int k = vlo.z; k <= vhi.z; ++k) { - if (mxhi(vhi.x+1,vlo.y-1,k) != BndryData::covered) { - Box bx(IntVect(vhi.x+1,vlo.y-1,k),IntVect(vhi.x+1,vlo.y-1,k)); - if (mylo(vhi.x,vlo.y-1,k) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } - } - } + } else if (bid == 1) { + for (int k = vlo.z + tid; k <= vhi.z; k += bdim) { + mltensor_fill_edges_xhi_ylo(vhi.x+1, vlo.y-1, k, blen, vel, mxhi, mylo, bcvalxhi, bcvalylo, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, ylo_domain); + } + } else if (bid == 2) { + for (int k = vlo.z + tid; k <= vhi.z; k += bdim) { + mltensor_fill_edges_xlo_yhi(vlo.x-1, vhi.y+1, k, blen, vel, mxlo, myhi, bcvalxlo, bcvalyhi, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, yhi_domain); + } + } else if (bid == 3) { + for (int k = vlo.z + tid; k <= vhi.z; k += bdim) { + mltensor_fill_edges_xhi_yhi(vhi.x+1, vhi.y+1, k, blen, vel, mxhi, myhi, bcvalxhi, bcvalyhi, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, yhi_domain); + } + } else if (bid == 4) { + for (int j = vlo.y + tid; j <= vhi.y; j += bdim) { + mltensor_fill_edges_xlo_zlo(vlo.x-1, j, vlo.z-1, blen, vel, mxlo, mzlo, bcvalxlo, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, zlo_domain); + } + } else if (bid == 5) { + for (int j = vlo.y + tid; j <= vhi.y; j += bdim) { + mltensor_fill_edges_xhi_zlo(vhi.x+1, j, vlo.z-1, blen, vel, mxhi, mzlo, bcvalxhi, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, zlo_domain); + } + } else if (bid == 6) { + for (int j = vlo.y + tid; j <= vhi.y; j += bdim) { + mltensor_fill_edges_xlo_zhi(vlo.x-1, j, vhi.z+1, blen, vel, mxlo, mzhi, bcvalxlo, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, xlo_domain, zhi_domain); + } + } else if (bid == 7) { + for (int j = vlo.y + tid; j <= vhi.y; j += bdim) { + mltensor_fill_edges_xhi_zhi(vhi.x+1, j, vhi.z+1, blen, vel, mxhi, mzhi, bcvalxhi, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, xhi_domain, zhi_domain); + } + } else if (bid == 8) { + for (int i = vlo.x + tid; i <= vhi.x; i += bdim) { + mltensor_fill_edges_ylo_zlo(i, vlo.y-1, vlo.z-1, blen, vel, mylo, mzlo, bcvalylo, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, ylo_domain, zlo_domain); + } + } else if (bid == 9) { + for (int i = vlo.x + tid; i <= vhi.x; i += bdim) { + mltensor_fill_edges_yhi_zlo(i, vhi.y+1, vlo.z-1, blen, vel, myhi, mzlo, bcvalyhi, bcvalzlo, + bct, bcl, inhomog, maxorder, dxinv, yhi_domain, zlo_domain); + } + } else if (bid == 10) { + for (int i = vlo.x + tid; i <= vhi.x; i += bdim) { + mltensor_fill_edges_ylo_zhi(i, vlo.y-1, vhi.z+1, blen, vel, mylo, mzhi, bcvalylo, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, ylo_domain, zhi_domain); + } + } else if (bid == 11) { + for (int i = vlo.x + tid; i <= vhi.x; i += bdim) { + mltensor_fill_edges_yhi_zhi(i, vhi.y+1, vhi.z+1, blen, vel, myhi, mzhi, bcvalyhi, bcvalzhi, + bct, bcl, inhomog, maxorder, dxinv, yhi_domain, zhi_domain); + } + } +} +#endif + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dz_on_xface (int i, int j, int k, int n, Array4 const& vel, Real dzi) noexcept +{ + return (vel(i,j,k+1,n)+vel(i-1,j,k+1,n)-vel(i,j,k-1,n)-vel(i-1,j,k-1,n))*(Real(0.25)*dzi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dz_on_yface (int i, int j, int k, int n, Array4 const& vel, Real dzi) noexcept +{ + return (vel(i,j,k+1,n)+vel(i,j-1,k+1,n)-vel(i,j,k-1,n)-vel(i,j-1,k-1,n))*(Real(0.25)*dzi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dx_on_zface (int i, int j, int k, int n, Array4 const& vel, Real dxi) noexcept +{ + return (vel(i+1,j,k,n)+vel(i+1,j,k-1,n)-vel(i-1,j,k,n)-vel(i-1,j,k-1,n))*(Real(0.25)*dxi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dy_on_zface (int i, int j, int k, int n, Array4 const& vel, Real dyi) noexcept +{ + return (vel(i,j+1,k,n)+vel(i,j+1,k-1,n)-vel(i,j-1,k,n)-vel(i,j-1,k-1,n))*(Real(0.25)*dyi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_cross_terms_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + Array4 const& etax, + Array4 const& kapx, + GpuArray const& dxinv) noexcept +{ + const Real dyi = dxinv[1]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = Real(2./3.); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi); + Real dudz = mltensor_dz_on_xface(i,j,k,0,vel,dzi); + Real dwdz = mltensor_dz_on_xface(i,j,k,2,vel,dzi); + Real divu = dvdy + dwdz; + Real xif = kapx(i,j,k); + Real mun = Real(0.75)*(etax(i,j,k,0)-xif); // restore the original eta + Real mut = etax(i,j,k,1); + fx(i,j,k,0) = -mun*(-twoThirds*divu) - xif*divu; + fx(i,j,k,1) = -mut*(dudy); + fx(i,j,k,2) = -mut*(dudz); } - break; } - case 2: { - // xlo & yhi - if (vlo.x == dlo.x && vhi.y == dhi.y) { - for (int k = vlo.z; k <= vhi.z; ++k) { - vel (vlo.x-1,vhi.y+1,k,icomp) - = vel(vlo.x ,vhi.y+1,k,icomp) - + vel(vlo.x-1,vhi.y ,k,icomp) - - vel(vlo.x ,vhi.y ,k,icomp); - } - } else if (vlo.x == dlo.x) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - for (int k = vlo.z; k <= vhi.z; ++k) { - if (mxlo(vlo.x-1,vhi.y+1,k) != BndryData::covered) { - Box bx(IntVect(vlo.x-1,vhi.y+1,k),IntVect(vlo.x-1,vhi.y+1,k)); - if (myhi(vlo.x,vhi.y+1,k) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } - } - } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_cross_terms_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + Array4 const& etay, + Array4 const& kapy, + GpuArray const& dxinv) noexcept +{ + const Real dxi = dxinv[0]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = Real(2./3.); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi); + Real dvdz = mltensor_dz_on_yface(i,j,k,1,vel,dzi); + Real dwdz = mltensor_dz_on_yface(i,j,k,2,vel,dzi); + Real divu = dudx + dwdz; + Real xif = kapy(i,j,k); + Real mun = Real(0.75)*(etay(i,j,k,1)-xif); // restore the original eta + Real mut = etay(i,j,k,0); + fy(i,j,k,0) = -mut*(dvdx); + fy(i,j,k,1) = -mun*(-twoThirds*divu) - xif*divu; + fy(i,j,k,2) = -mut*(dvdz); } - break; } - case 3: { - // xhi & yhi - if (vhi.x == dhi.x && vhi.y == dhi.y) { - for (int k = vlo.z; k <= vhi.z; ++k) { - vel (vhi.x+1,vhi.y+1,k,icomp) - = vel(vhi.x ,vhi.y+1,k,icomp) - + vel(vhi.x+1,vhi.y ,k,icomp) - - vel(vhi.x ,vhi.y ,k,icomp); - } - } else if (vhi.x == dhi.x) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.y == dhi.y) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),ydir,1); - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - for (int k = vlo.z; k <= vhi.z; ++k) { - if (mxhi(vhi.x+1,vhi.y+1,k) != BndryData::covered) { - Box bx(IntVect(vhi.x+1,vhi.y+1,k),IntVect(vhi.x+1,vhi.y+1,k)); - if (myhi(vhi.x,vhi.y+1,k) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } - } - } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_cross_terms_fz (Box const& box, Array4 const& fz, + Array4 const& vel, + Array4 const& etaz, + Array4 const& kapz, + GpuArray const& dxinv) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + constexpr Real twoThirds = Real(2./3.); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + AMREX_PRAGMA_SIMD + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_zface(i,j,k,0,vel,dxi); + Real dwdx = mltensor_dx_on_zface(i,j,k,2,vel,dxi); + Real dvdy = mltensor_dy_on_zface(i,j,k,1,vel,dyi); + Real dwdy = mltensor_dy_on_zface(i,j,k,2,vel,dyi); + Real divu = dudx + dvdy; + Real xif = kapz(i,j,k); + Real mun = Real(0.75)*(etaz(i,j,k,2)-xif); // restore the original eta + Real mut = etaz(i,j,k,0); + fz(i,j,k,0) = -mut*(dwdx); + fz(i,j,k,1) = -mut*(dwdy); + fz(i,j,k,2) = -mun*(-twoThirds*divu) - xif*divu; } - break; } - case 4: { - // xlo & zlo - if (vlo.x == dlo.x && vlo.z == dlo.z) { - for (int j = vlo.y; j <= vhi.y; ++j) { - vel (vlo.x-1,j,vlo.z-1,icomp) - = vel(vlo.x ,j,vlo.z-1,icomp) - + vel(vlo.x-1,j,vlo.z ,icomp) - - vel(vlo.x ,j,vlo.z ,icomp); - } - } else if (vlo.x == dlo.x) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dz_on_xface (int i, int j, int k, int n, Array4 const& vel, Real dzi, + Array4 const& bvxlo, Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddz; + if (i == dlo.x) { + if (bct(Orientation::xlo(),n) == AMREX_LO_DIRICHLET && bvxlo) { + if (k == dlo.z) { + ddz = (bvxlo(i-1,j,k ,n) * Real(-1.5) + + bvxlo(i-1,j,k+1,n) * Real(2.) + + bvxlo(i-1,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvxlo(i-1,j,k ,n) * Real(-1.5) + + bvxlo(i-1,j,k-1,n) * Real(2.) + + bvxlo(i-1,j,k-2,n) * Real(-0.5)) * dzi; } else { - for (int j = vlo.y; j <= vhi.y; ++j) { - if (mxlo(vlo.x-1,j,vlo.z-1) != BndryData::covered) { - Box bx(IntVect(vlo.x-1,j,vlo.z-1),IntVect(vlo.x-1,j,vlo.z-1)); - if (mzlo(vlo.x,j,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddz = (bvxlo(i-1,j,k+1,n)-bvxlo(i-1,j,k-1,n))*(Real(0.5)*dzi); } - break; + } else if (bct(Orientation::xlo(),n) == AMREX_LO_NEUMANN) { + ddz = (vel(i,j,k+1,n)-vel(i,j,k-1,n))*(Real(0.5)*dzi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); } - case 5: { - // xhi & zlo - if (vhi.x == dhi.x && vlo.z == dlo.z) { - for (int j = vlo.y; j <= vhi.y; ++j) { - vel (vhi.x+1,j,vlo.z-1,icomp) - = vel(vhi.x ,j,vlo.z-1,icomp) - + vel(vhi.x+1,j,vlo.z ,icomp) - - vel(vhi.x ,j,vlo.z ,icomp); - } - } else if (vhi.x == dhi.x) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + } else if (i == dhi.x+1) { + if (bct(Orientation::xhi(),n) == AMREX_LO_DIRICHLET && bvxhi) { + if (k == dlo.z) { + ddz = (bvxhi(i,j,k ,n) * Real(-1.5) + + bvxhi(i,j,k+1,n) * Real(2.) + + bvxhi(i,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvxhi(i,j,k ,n) * Real(-1.5) + + bvxhi(i,j,k-1,n) * Real(2.) + + bvxhi(i,j,k-2,n) * Real(-0.5)) * dzi; } else { - for (int j = vlo.y; j <= vhi.y; ++j) { - if (mxhi(vhi.x+1,j,vlo.z-1) != BndryData::covered) { - Box bx(IntVect(vhi.x+1,j,vlo.z-1),IntVect(vhi.x+1,j,vlo.z-1)); - if (mzlo(vhi.x,j,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddz = (bvxhi(i,j,k+1,n)-bvxhi(i,j,k-1,n))*(Real(0.5)*dzi); } - break; + } else if (bct(Orientation::xhi(),n) == AMREX_LO_NEUMANN) { + ddz = (vel(i-1,j,k+1,n)-vel(i-1,j,k-1,n))*(Real(0.5)*dzi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); } - case 6: { - // xlo & zhi - if (vlo.x == dlo.x && vhi.z == dhi.z) { - for (int j = vlo.y; j <= vhi.y; ++j) { - vel (vlo.x-1,j,vhi.z+1,icomp) - = vel(vlo.x ,j,vhi.z+1,icomp) - + vel(vlo.x-1,j,vhi.z ,icomp) - - vel(vlo.x ,j,vhi.z ,icomp); - } - } else if (vlo.x == dlo.x) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + } else { + ddz = mltensor_dz_on_xface(i,j,k,n,vel,dzi); + } + return ddz; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dz_on_yface (int i, int j, int k, int n, Array4 const& vel, Real dzi, + Array4 const& bvylo, Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddz; + if (j == dlo.y) { + if (bct(Orientation::ylo(),n) == AMREX_LO_DIRICHLET && bvylo) { + if (k == dlo.z) { + ddz = (bvylo(i,j-1,k ,n) * Real(-1.5) + + bvylo(i,j-1,k+1,n) * Real(2.) + + bvylo(i,j-1,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvylo(i,j-1,k ,n) * Real(-1.5) + + bvylo(i,j-1,k-1,n) * Real(2.) + + bvylo(i,j-1,k-2,n) * Real(-0.5)) * dzi; } else { - for (int j = vlo.y; j <= vhi.y; ++j) { - if (mxlo(vlo.x-1,j,vhi.z+1) != BndryData::covered) { - Box bx(IntVect(vlo.x-1,j,vhi.z+1),IntVect(vlo.x-1,j,vhi.z+1)); - if (mzhi(vlo.x,j,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxlo; - mllinop_apply_bc_x(Orientation::low, bx, blen.x, - vel, mxlo, bct[offset+icomp], bcl[offset+icomp], - bcvalxlo, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddz = (bvylo(i,j-1,k+1,n)-bvylo(i,j-1,k-1,n))*(Real(0.5)*dzi); } - break; + } else if (bct(Orientation::ylo(),n) == AMREX_LO_NEUMANN) { + ddz = (vel(i,j,k+1,n)-vel(i,j,k-1,n))*(Real(0.5)*dzi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); } - case 7: { - // xhi & zhi - if (vhi.x == dhi.x && vhi.z == dhi.z) { - for (int j = vlo.y; j <= vhi.y; ++j) { - vel (vhi.x+1,j,vhi.z+1,icomp) - = vel(vhi.x ,j,vhi.z+1,icomp) - + vel(vhi.x+1,j,vhi.z ,icomp) - - vel(vhi.x ,j,vhi.z ,icomp); - } - } else if (vhi.x == dhi.x) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,xdir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + } else if (j == dhi.y+1) { + if (bct(Orientation::yhi(),n) == AMREX_LO_DIRICHLET && bvyhi) { + if (k == dlo.z) { + ddz = (bvyhi(i,j,k ,n) * Real(-1.5) + + bvyhi(i,j,k+1,n) * Real(2.) + + bvyhi(i,j,k+2,n) * Real(-0.5)) * dzi; + } else if (k == dhi.z) { + ddz = -(bvyhi(i,j,k ,n) * Real(-1.5) + + bvyhi(i,j,k-1,n) * Real(2.) + + bvyhi(i,j,k-2,n) * Real(-0.5)) * dzi; } else { - for (int j = vlo.y; j <= vhi.y; ++j) { - if (mxhi(vhi.x+1,j,vhi.z+1) != BndryData::covered) { - Box bx(IntVect(vhi.x+1,j,vhi.z+1),IntVect(vhi.x+1,j,vhi.z+1)); - if (mzhi(vhi.x,j,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oxhi; - mllinop_apply_bc_x(Orientation::high, bx, blen.x, - vel, mxhi, bct[offset+icomp], bcl[offset+icomp], - bcvalxhi, maxorder, dxinv[xdir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddz = (bvyhi(i,j,k+1,n)-bvyhi(i,j,k-1,n))*(Real(0.5)*dzi); } - break; + } else if (bct(Orientation::yhi(),n) == AMREX_LO_NEUMANN) { + ddz = (vel(i,j-1,k+1,n)-vel(i,j-1,k-1,n))*(Real(0.5)*dzi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddz = Real(0.); } - case 8: { - // ylo & zlo - if (vlo.y == dlo.y && vlo.z == dlo.z) { - for (int i = vlo.x; i <= vhi.x; ++i) { - vel (i,vlo.y-1,vlo.z-1,icomp) - = vel(i,vlo.y ,vlo.z-1,icomp) - + vel(i,vlo.y-1,vlo.z ,icomp) - - vel(i,vlo.y ,vlo.z ,icomp); - } - } else if (vlo.y == dlo.y) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - Box bx = amrex::adjCellLo(amrex::adjCellLo(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + } else { + ddz = mltensor_dz_on_yface(i,j,k,n,vel,dzi); + } + return ddz; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dx_on_zface (int i, int j, int k, int n, Array4 const& vel, Real dxi, + Array4 const& bvzlo, Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddx; + if (k == dlo.z) { + if (bct(Orientation::zlo(),n) == AMREX_LO_DIRICHLET && bvzlo) { + if (i == dlo.x) { + ddx = (bvzlo(i ,j,k-1,n) * Real(-1.5) + + bvzlo(i+1,j,k-1,n) * Real(2.) + + bvzlo(i+2,j,k-1,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvzlo(i ,j,k-1,n) * Real(-1.5) + + bvzlo(i-1,j,k-1,n) * Real(2.) + + bvzlo(i-2,j,k-1,n) * Real(-0.5)) * dxi; } else { - for (int i = vlo.x; i <= vhi.x; ++i) { - if (mylo(i,vlo.y-1,vlo.z-1) != BndryData::covered) { - Box bx(IntVect(i,vlo.y-1,vlo.z-1),IntVect(i,vlo.y-1,vlo.z-1)); - if (mzlo(i,vlo.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddx = (bvzlo(i+1,j,k-1,n)-bvzlo(i-1,j,k-1,n))*(Real(0.5)*dxi); } - break; + } else if (bct(Orientation::zlo(),n) == AMREX_LO_NEUMANN) { + ddx = (vel(i+1,j,k,n)-vel(i-1,j,k,n))*(Real(0.5)*dxi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); } - case 9: { - // yhi & zlo - if (vhi.y == dhi.y && vlo.z == dlo.z) { - for (int i = vlo.x; i <= vhi.x; ++i) { - vel (i,vhi.y+1,vlo.z-1,icomp) - = vel(i,vhi.y ,vlo.z-1,icomp) - + vel(i,vhi.y+1,vlo.z ,icomp) - - vel(i,vhi.y ,vlo.z ,icomp); - } - } else if (vhi.y == dhi.y) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vlo.z == dlo.z) { - Box bx = amrex::adjCellLo(amrex::adjCellHi(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); + } else if (k == dhi.z+1) { + if (bct(Orientation::zhi(),n) == AMREX_LO_DIRICHLET && bvzhi) { + if (i == dlo.x) { + ddx = (bvzhi(i ,j,k,n) * Real(-1.5) + + bvzhi(i+1,j,k,n) * Real(2.) + + bvzhi(i+2,j,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvzhi(i ,j,k,n) * Real(-1.5) + + bvzhi(i-1,j,k,n) * Real(2.) + + bvzhi(i-2,j,k,n) * Real(-0.5)) * dxi; } else { - for (int i = vlo.x; i <= vhi.x; ++i) { - if (myhi(i,vhi.y+1,vlo.z-1) != BndryData::covered) { - Box bx(IntVect(i,vhi.y+1,vlo.z-1),IntVect(i,vhi.y+1,vlo.z-1)); - if (mzlo(i,vhi.y,vlo.z-1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozlo; - mllinop_apply_bc_z(Orientation::low, bx, blen.z, - vel, mzlo, bct[offset+icomp], bcl[offset+icomp], - bcvalzlo, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddx = (bvzhi(i+1,j,k,n)-bvzhi(i-1,j,k,n))*(Real(0.5)*dxi); } - break; + } else if (bct(Orientation::zhi(),n) == AMREX_LO_NEUMANN) { + ddx = (vel(i+1,j,k-1,n)-vel(i-1,j,k-1,n))*(Real(0.5)*dxi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); } - case 10: { - // ylo & zhi - if (vlo.y == dlo.y && vhi.z == dhi.z) { - for (int i = vlo.x; i <= vhi.x; ++i) { - vel (i,vlo.y-1,vhi.z+1,icomp) - = vel(i,vlo.y ,vhi.z+1,icomp) - + vel(i,vlo.y-1,vhi.z ,icomp) - - vel(i,vlo.y ,vhi.z ,icomp); - } - } else if (vlo.y == dlo.y) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - Box bx = amrex::adjCellHi(amrex::adjCellLo(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + } else { + ddx = mltensor_dx_on_zface(i,j,k,n,vel,dxi); + } + return ddx; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dy_on_zface (int i, int j, int k, int n, Array4 const& vel, Real dyi, + Array4 const& bvzlo, Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddy; + if (k == dlo.z) { + if (bct(Orientation::zlo(),n) == AMREX_LO_DIRICHLET && bvzlo) { + if (j == dlo.y) { + ddy = (bvzlo(i,j ,k-1,n) * Real(-1.5) + + bvzlo(i,j+1,k-1,n) * Real(2.) + + bvzlo(i,j+2,k-1,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvzlo(i,j ,k-1,n) * Real(-1.5) + + bvzlo(i,j-1,k-1,n) * Real(2.) + + bvzlo(i,j-2,k-1,n) * Real(-0.5)) * dyi; } else { - for (int i = vlo.x; i <= vhi.x; ++i) { - if (mylo(i,vlo.y-1,vhi.z+1) != BndryData::covered) { - Box bx(IntVect(i,vlo.y-1,vhi.z+1),IntVect(i,vlo.y-1,vhi.z+1)); - if (mzhi(i,vlo.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oylo; - mllinop_apply_bc_y(Orientation::low, bx, blen.y, - vel, mylo, bct[offset+icomp], bcl[offset+icomp], - bcvalylo, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddy = (bvzlo(i,j+1,k-1,n)-bvzlo(i,j-1,k-1,n))*(Real(0.5)*dyi); } - break; + } else if (bct(Orientation::zlo(),n) == AMREX_LO_NEUMANN) { + ddy = (vel(i,j+1,k,n)-vel(i,j-1,k,n))*(Real(0.5)*dyi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); } - case 11: { - // yhi & zhi - if (vhi.y == dhi.y && vhi.z == dhi.z) { - for (int i = vlo.x; i <= vhi.x; ++i) { - vel (i,vhi.y+1,vhi.z+1,icomp) - = vel(i,vhi.y ,vhi.z+1,icomp) - + vel(i,vhi.y+1,vhi.z ,icomp) - - vel(i,vhi.y ,vhi.z ,icomp); - } - } else if (vhi.y == dhi.y) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else if (vhi.z == dhi.z) { - Box bx = amrex::adjCellHi(amrex::adjCellHi(vbox,ydir,1),zdir,1); - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); + } else if (k == dhi.z+1) { + if (bct(Orientation::zhi(),n) == AMREX_LO_DIRICHLET && bvzhi) { + if (j == dlo.y) { + ddy = (bvzhi(i,j ,k,n) * Real(-1.5) + + bvzhi(i,j+1,k,n) * Real(2.) + + bvzhi(i,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvzhi(i,j ,k,n) * Real(-1.5) + + bvzhi(i,j-1,k,n) * Real(2.) + + bvzhi(i,j-2,k,n) * Real(-0.5)) * dyi; } else { - for (int i = vlo.x; i <= vhi.x; ++i) { - if (myhi(i,vhi.y+1,vhi.z+1) != BndryData::covered) { - Box bx(IntVect(i,vhi.y+1,vhi.z+1),IntVect(i,vhi.y+1,vhi.z+1)); - if (mzhi(i,vhi.y,vhi.z+1) == BndryData::covered) { - int offset = AMREX_SPACEDIM * oyhi; - mllinop_apply_bc_y(Orientation::high, bx, blen.y, - vel, myhi, bct[offset+icomp], bcl[offset+icomp], - bcvalyhi, maxorder, dxinv[ydir], inhomog, icomp); - } else { - int offset = AMREX_SPACEDIM * ozhi; - mllinop_apply_bc_z(Orientation::high, bx, blen.z, - vel, mzhi, bct[offset+icomp], bcl[offset+icomp], - bcvalzhi, maxorder, dxinv[zdir], inhomog, icomp); - } - } - } + ddy = (bvzhi(i,j+1,k,n)-bvzhi(i,j-1,k,n))*(Real(0.5)*dyi); } - break; - } - default: {} + } else if (bct(Orientation::zhi(),n) == AMREX_LO_NEUMANN) { + ddy = (vel(i,j+1,k-1,n)-vel(i,j-1,k-1,n))*(Real(0.5)*dyi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); } + } else { + ddy = mltensor_dy_on_zface(i,j,k,n,vel,dyi); } + return ddy; } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -1057,7 +1844,13 @@ void mltensor_cross_terms_fx (Box const& box, Array4 const& fx, Array4 const& vel, Array4 const& etax, Array4 const& kapx, - GpuArray const& dxinv) noexcept + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept { const Real dyi = dxinv[1]; const Real dzi = dxinv[2]; @@ -1067,12 +1860,11 @@ void mltensor_cross_terms_fx (Box const& box, Array4 const& fx, for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { - AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudy = (vel(i,j+1,k,0)+vel(i-1,j+1,k,0)-vel(i,j-1,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dyi); - Real dvdy = (vel(i,j+1,k,1)+vel(i-1,j+1,k,1)-vel(i,j-1,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dyi); - Real dudz = (vel(i,j,k+1,0)+vel(i-1,j,k+1,0)-vel(i,j,k-1,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dzi); - Real dwdz = (vel(i,j,k+1,2)+vel(i-1,j,k+1,2)-vel(i,j,k-1,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dzi); + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dudz = mltensor_dz_on_xface(i,j,k,0,vel,dzi,bvxlo,bvxhi,bct,dlo,dhi); + Real dwdz = mltensor_dz_on_xface(i,j,k,2,vel,dzi,bvxlo,bvxhi,bct,dlo,dhi); Real divu = dvdy + dwdz; Real xif = kapx(i,j,k); Real mun = Real(0.75)*(etax(i,j,k,0)-xif); // restore the original eta @@ -1090,7 +1882,13 @@ void mltensor_cross_terms_fy (Box const& box, Array4 const& fy, Array4 const& vel, Array4 const& etay, Array4 const& kapy, - GpuArray const& dxinv) noexcept + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept { const Real dxi = dxinv[0]; const Real dzi = dxinv[2]; @@ -1100,12 +1898,11 @@ void mltensor_cross_terms_fy (Box const& box, Array4 const& fy, for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { - AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,k,0)+vel(i+1,j-1,k,0)-vel(i-1,j,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dxi); - Real dvdx = (vel(i+1,j,k,1)+vel(i+1,j-1,k,1)-vel(i-1,j,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dxi); - Real dvdz = (vel(i,j,k+1,1)+vel(i,j-1,k+1,1)-vel(i,j,k-1,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dzi); - Real dwdz = (vel(i,j,k+1,2)+vel(i,j-1,k+1,2)-vel(i,j,k-1,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dzi); + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdz = mltensor_dz_on_yface(i,j,k,1,vel,dzi,bvylo,bvyhi,bct,dlo,dhi); + Real dwdz = mltensor_dz_on_yface(i,j,k,2,vel,dzi,bvylo,bvyhi,bct,dlo,dhi); Real divu = dudx + dwdz; Real xif = kapy(i,j,k); Real mun = Real(0.75)*(etay(i,j,k,1)-xif); // restore the original eta @@ -1123,7 +1920,13 @@ void mltensor_cross_terms_fz (Box const& box, Array4 const& fz, Array4 const& vel, Array4 const& etaz, Array4 const& kapz, - GpuArray const& dxinv) noexcept + GpuArray const& dxinv, + Array4 const& bvzlo, + Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept { const Real dxi = dxinv[0]; const Real dyi = dxinv[1]; @@ -1133,12 +1936,11 @@ void mltensor_cross_terms_fz (Box const& box, Array4 const& fz, for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { - AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,k,0)+vel(i+1,j,k-1,0)-vel(i-1,j,k,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dxi); - Real dwdx = (vel(i+1,j,k,2)+vel(i+1,j,k-1,2)-vel(i-1,j,k,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dxi); - Real dvdy = (vel(i,j+1,k,1)+vel(i,j+1,k-1,1)-vel(i,j-1,k,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dyi); - Real dwdy = (vel(i,j+1,k,2)+vel(i,j+1,k-1,2)-vel(i,j-1,k,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dyi); + Real dudx = mltensor_dx_on_zface(i,j,k,0,vel,dxi,bvzlo,bvzhi,bct,dlo,dhi); + Real dwdx = mltensor_dx_on_zface(i,j,k,2,vel,dxi,bvzlo,bvzhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_zface(i,j,k,1,vel,dyi,bvzlo,bvzhi,bct,dlo,dhi); + Real dwdy = mltensor_dy_on_zface(i,j,k,2,vel,dyi,bvzlo,bvzhi,bct,dlo,dhi); Real divu = dudx + dvdy; Real xif = kapz(i,j,k); Real mun = Real(0.75)*(etaz(i,j,k,2)-xif); // restore the original eta @@ -1242,13 +2044,13 @@ void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, Real dvdx = (vel(i,j,k,1) - vel(i-1,j,k,1))*dxi; Real dwdx = (vel(i,j,k,2) - vel(i-1,j,k,2))*dxi; - Real dudy = (vel(i,j+1,k,0)+vel(i-1,j+1,k,0)-vel(i,j-1,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dyi); - Real dvdy = (vel(i,j+1,k,1)+vel(i-1,j+1,k,1)-vel(i,j-1,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dyi); - Real dwdy = (vel(i,j+1,k,2)+vel(i-1,j+1,k,2)-vel(i,j-1,k,2)-vel(i-1,j-1,k,2))*(Real(0.25)*dyi); + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi); + Real dwdy = mltensor_dy_on_xface(i,j,k,2,vel,dyi); - Real dudz = (vel(i,j,k+1,0)+vel(i-1,j,k+1,0)-vel(i,j,k-1,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dzi); - Real dvdz = (vel(i,j,k+1,1)+vel(i-1,j,k+1,1)-vel(i,j,k-1,1)-vel(i-1,j,k-1,1))*(Real(0.25)*dzi); - Real dwdz = (vel(i,j,k+1,2)+vel(i-1,j,k+1,2)-vel(i,j,k-1,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dzi); + Real dudz = mltensor_dz_on_xface(i,j,k,0,vel,dzi); + Real dvdz = mltensor_dz_on_xface(i,j,k,1,vel,dzi); + Real dwdz = mltensor_dz_on_xface(i,j,k,2,vel,dzi); fx(i,j,k,0) = dudx; fx(i,j,k,1) = dvdx; @@ -1281,17 +2083,17 @@ void mltensor_vel_grads_fy (Box const& box, Array4 const& fy, AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,k,0)+vel(i+1,j-1,k,0)-vel(i-1,j,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dxi); - Real dvdx = (vel(i+1,j,k,1)+vel(i+1,j-1,k,1)-vel(i-1,j,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dxi); - Real dwdx = (vel(i+1,j,k,2)+vel(i+1,j-1,k,2)-vel(i-1,j,k,2)-vel(i-1,j-1,k,2))*(Real(0.25)*dxi); + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi); + Real dwdx = mltensor_dx_on_yface(i,j,k,2,vel,dxi); Real dudy = (vel(i,j,k,0) - vel(i,j-1,k,0))*dyi; Real dvdy = (vel(i,j,k,1) - vel(i,j-1,k,1))*dyi; Real dwdy = (vel(i,j,k,2) - vel(i,j-1,k,2))*dyi; - Real dudz = (vel(i,j,k+1,0)+vel(i,j-1,k+1,0)-vel(i,j,k-1,0)-vel(i,j-1,k-1,0))*(Real(0.25)*dzi); - Real dvdz = (vel(i,j,k+1,1)+vel(i,j-1,k+1,1)-vel(i,j,k-1,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dzi); - Real dwdz = (vel(i,j,k+1,2)+vel(i,j-1,k+1,2)-vel(i,j,k-1,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dzi); + Real dudz = mltensor_dz_on_yface(i,j,k,0,vel,dzi); + Real dvdz = mltensor_dz_on_yface(i,j,k,1,vel,dzi); + Real dwdz = mltensor_dz_on_yface(i,j,k,2,vel,dzi); fy(i,j,k,0) = dudx; fy(i,j,k,1) = dvdx; @@ -1324,13 +2126,13 @@ void mltensor_vel_grads_fz (Box const& box, Array4 const& fz, AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - Real dudx = (vel(i+1,j,k,0)+vel(i+1,j,k-1,0)-vel(i-1,j,k,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dxi); - Real dvdx = (vel(i+1,j,k,1)+vel(i+1,j,k-1,1)-vel(i-1,j,k,1)-vel(i-1,j,k-1,1))*(Real(0.25)*dxi); - Real dwdx = (vel(i+1,j,k,2)+vel(i+1,j,k-1,2)-vel(i-1,j,k,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dxi); + Real dudx = mltensor_dx_on_zface(i,j,k,0,vel,dxi); + Real dvdx = mltensor_dx_on_zface(i,j,k,1,vel,dxi); + Real dwdx = mltensor_dx_on_zface(i,j,k,2,vel,dxi); - Real dudy = (vel(i,j+1,k,0)+vel(i,j+1,k-1,0)-vel(i,j-1,k,0)-vel(i,j-1,k-1,0))*(Real(0.25)*dyi); - Real dvdy = (vel(i,j+1,k,1)+vel(i,j+1,k-1,1)-vel(i,j-1,k,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dyi); - Real dwdy = (vel(i,j+1,k,2)+vel(i,j+1,k-1,2)-vel(i,j-1,k,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dyi); + Real dudy = mltensor_dy_on_zface(i,j,k,0,vel,dyi); + Real dvdy = mltensor_dy_on_zface(i,j,k,1,vel,dyi); + Real dwdy = mltensor_dy_on_zface(i,j,k,2,vel,dyi); Real dudz = (vel(i,j,k,0) - vel(i,j,k-1,0))*dzi; Real dvdz = (vel(i,j,k,1) - vel(i,j,k-1,1))*dzi; @@ -1351,6 +2153,138 @@ void mltensor_vel_grads_fz (Box const& box, Array4 const& fz, } } +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, + Array4 const& vel, + GpuArray const& dxinv, + Array4 const& bvxlo, + Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = (vel(i,j,k,0) - vel(i-1,j,k,0))*dxi; + Real dvdx = (vel(i,j,k,1) - vel(i-1,j,k,1))*dxi; + Real dwdx = (vel(i,j,k,2) - vel(i-1,j,k,2))*dxi; + Real dudy = mltensor_dy_on_xface(i,j,k,0,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_xface(i,j,k,1,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dwdy = mltensor_dy_on_xface(i,j,k,2,vel,dyi,bvxlo,bvxhi,bct,dlo,dhi); + Real dudz = mltensor_dz_on_xface(i,j,k,0,vel,dzi,bvxlo,bvxhi,bct,dlo,dhi); + Real dvdz = mltensor_dz_on_xface(i,j,k,1,vel,dzi,bvxlo,bvxhi,bct,dlo,dhi); + Real dwdz = mltensor_dz_on_xface(i,j,k,2,vel,dzi,bvxlo,bvxhi,bct,dlo,dhi); + fx(i,j,k,0) = dudx; + fx(i,j,k,1) = dvdx; + fx(i,j,k,2) = dwdx; + fx(i,j,k,3) = dudy; + fx(i,j,k,4) = dvdy; + fx(i,j,k,5) = dwdy; + fx(i,j,k,6) = dudz; + fx(i,j,k,7) = dvdz; + fx(i,j,k,8) = dwdz; + + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_vel_grads_fy (Box const& box, Array4 const& fy, + Array4 const& vel, + GpuArray const& dxinv, + Array4 const& bvylo, + Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_yface(i,j,k,0,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdx = mltensor_dx_on_yface(i,j,k,1,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dwdx = mltensor_dx_on_yface(i,j,k,2,vel,dxi,bvylo,bvyhi,bct,dlo,dhi); + Real dudy = (vel(i,j,k,0) - vel(i,j-1,k,0))*dyi; + Real dvdy = (vel(i,j,k,1) - vel(i,j-1,k,1))*dyi; + Real dwdy = (vel(i,j,k,2) - vel(i,j-1,k,2))*dyi; + Real dudz = mltensor_dz_on_yface(i,j,k,0,vel,dzi,bvylo,bvyhi,bct,dlo,dhi); + Real dvdz = mltensor_dz_on_yface(i,j,k,1,vel,dzi,bvylo,bvyhi,bct,dlo,dhi); + Real dwdz = mltensor_dz_on_yface(i,j,k,2,vel,dzi,bvylo,bvyhi,bct,dlo,dhi); + fy(i,j,k,0) = dudx; + fy(i,j,k,1) = dvdx; + fy(i,j,k,2) = dwdx; + fy(i,j,k,3) = dudy; + fy(i,j,k,4) = dvdy; + fy(i,j,k,5) = dwdy; + fy(i,j,k,6) = dudz; + fy(i,j,k,7) = dvdz; + fy(i,j,k,8) = dwdz; + + } + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void mltensor_vel_grads_fz (Box const& box, Array4 const& fz, + Array4 const& vel, + GpuArray const& dxinv, + Array4 const& bvzlo, + Array4 const& bvzhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + const Real dxi = dxinv[0]; + const Real dyi = dxinv[1]; + const Real dzi = dxinv[2]; + const auto lo = amrex::lbound(box); + const auto hi = amrex::ubound(box); + + for (int k = lo.z; k <= hi.z; ++k) { + for (int j = lo.y; j <= hi.y; ++j) { + for (int i = lo.x; i <= hi.x; ++i) { + Real dudx = mltensor_dx_on_zface(i,j,k,0,vel,dxi,bvzlo,bvzhi,bct,dlo,dhi); + Real dvdx = mltensor_dx_on_zface(i,j,k,1,vel,dxi,bvzlo,bvzhi,bct,dlo,dhi); + Real dwdx = mltensor_dx_on_zface(i,j,k,2,vel,dxi,bvzlo,bvzhi,bct,dlo,dhi); + Real dudy = mltensor_dy_on_zface(i,j,k,0,vel,dyi,bvzlo,bvzhi,bct,dlo,dhi); + Real dvdy = mltensor_dy_on_zface(i,j,k,1,vel,dyi,bvzlo,bvzhi,bct,dlo,dhi); + Real dwdy = mltensor_dy_on_zface(i,j,k,2,vel,dyi,bvzlo,bvzhi,bct,dlo,dhi); + Real dudz = (vel(i,j,k,0) - vel(i,j,k-1,0))*dzi; + Real dvdz = (vel(i,j,k,1) - vel(i,j,k-1,1))*dzi; + Real dwdz = (vel(i,j,k,2) - vel(i,j,k-1,2))*dzi; + fz(i,j,k,0) = dudx; + fz(i,j,k,1) = dvdx; + fz(i,j,k,2) = dwdx; + fz(i,j,k,3) = dudy; + fz(i,j,k,4) = dvdy; + fz(i,j,k,5) = dwdy; + fz(i,j,k,6) = dudz; + fz(i,j,k,7) = dvdz; + fz(i,j,k,8) = dwdz; + + } + } + } +} + } #endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensor_K.H b/Src/LinearSolvers/MLMG/AMReX_MLTensor_K.H index 4440f57e7a8..33457ec1ced 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensor_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensor_K.H @@ -5,6 +5,123 @@ #include #include +namespace amrex { + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dy_on_xface (int i, int j, int k, int n, Array4 const& vel, Real dyi) noexcept +{ + return (vel(i,j+1,k,n)+vel(i-1,j+1,k,n)-vel(i,j-1,k,n)-vel(i-1,j-1,k,n))*(Real(0.25)*dyi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dx_on_yface (int i, int j, int k, int n, Array4 const& vel, Real dxi) noexcept +{ + return (vel(i+1,j,k,n)+vel(i+1,j-1,k,n)-vel(i-1,j,k,n)-vel(i-1,j-1,k,n))*(Real(0.25)*dxi); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dy_on_xface (int i, int j, int k, int n, Array4 const& vel, Real dyi, + Array4 const& bvxlo, Array4 const& bvxhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddy; + if (i == dlo.x) { + if (bct(Orientation::xlo(),n) == AMREX_LO_DIRICHLET && bvxlo) { + if (j == dlo.y) { + ddy = (bvxlo(i-1,j ,k,n) * Real(-1.5) + + bvxlo(i-1,j+1,k,n) * Real(2.) + + bvxlo(i-1,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvxlo(i-1,j ,k,n) * Real(-1.5) + + bvxlo(i-1,j-1,k,n) * Real(2.) + + bvxlo(i-1,j-2,k,n) * Real(-0.5)) * dyi; + } else { + ddy = (bvxlo(i-1,j+1,k,n)-bvxlo(i-1,j-1,k,n))*(Real(0.5)*dyi); + } + } else if (bct(Orientation::xlo(),n) == AMREX_LO_NEUMANN) { + ddy = (vel(i,j+1,k,n)-vel(i,j-1,k,n))*(Real(0.5)*dyi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else if (i == dhi.x+1) { + if (bct(Orientation::xhi(),n) == AMREX_LO_DIRICHLET && bvxhi) { + if (j == dlo.y) { + ddy = (bvxhi(i,j ,k,n) * Real(-1.5) + + bvxhi(i,j+1,k,n) * Real(2.) + + bvxhi(i,j+2,k,n) * Real(-0.5)) * dyi; + } else if (j == dhi.y) { + ddy = -(bvxhi(i,j ,k,n) * Real(-1.5) + + bvxhi(i,j-1,k,n) * Real(2.) + + bvxhi(i,j-2,k,n) * Real(-0.5)) * dyi; + } else { + ddy = (bvxhi(i,j+1,k,n)-bvxhi(i,j-1,k,n))*(Real(0.5)*dyi); + } + } else if (bct(Orientation::xhi(),n) == AMREX_LO_NEUMANN) { + ddy = (vel(i-1,j+1,k,n)-vel(i-1,j-1,k,n))*(Real(0.5)*dyi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddy = Real(0.); + } + } else { + ddy = mltensor_dy_on_xface(i,j,k,n,vel,dyi); + } + return ddy; +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Real mltensor_dx_on_yface (int i, int j, int k, int n, Array4 const& vel, Real dxi, + Array4 const& bvylo, Array4 const& bvyhi, + Array2D const& bct, + Dim3 const& dlo, Dim3 const& dhi) noexcept +{ + Real ddx; + if (j == dlo.y) { + if (bct(Orientation::ylo(),n) == AMREX_LO_DIRICHLET && bvylo) { + if (i == dlo.x) { + ddx = (bvylo(i ,j-1,k,n) * Real(-1.5) + + bvylo(i+1,j-1,k,n) * Real(2.) + + bvylo(i+2,j-1,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvylo(i ,j-1,k,n) * Real(-1.5) + + bvylo(i-1,j-1,k,n) * Real(2.) + + bvylo(i-2,j-1,k,n) * Real(-0.5)) * dxi; + } else { + ddx = (bvylo(i+1,j-1,k,n)-bvylo(i-1,j-1,k,n))*(Real(0.5)*dxi); + } + } else if (bct(Orientation::ylo(),n) == AMREX_LO_NEUMANN) { + ddx = (vel(i+1,j,k,n)-vel(i-1,j,k,n))*(Real(0.5)*dxi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else if (j == dhi.y+1) { + if (bct(Orientation::yhi(),n) == AMREX_LO_DIRICHLET && bvyhi) { + if (i == dlo.x) { + ddx = (bvyhi(i ,j,k,n) * Real(-1.5) + + bvyhi(i+1,j,k,n) * Real(2.) + + bvyhi(i+2,j,k,n) * Real(-0.5)) * dxi; + } else if (i == dhi.x) { + ddx = -(bvyhi(i ,j,k,n) * Real(-1.5) + + bvyhi(i-1,j,k,n) * Real(2.) + + bvyhi(i-2,j,k,n) * Real(-0.5)) * dxi; + } else { + ddx = (bvyhi(i+1,j,k,n)-bvyhi(i-1,j,k,n))*(Real(0.5)*dxi); + } + } else if (bct(Orientation::yhi(),n) == AMREX_LO_NEUMANN) { + ddx = (vel(i+1,j-1,k,n)-vel(i-1,j-1,k,n))*(Real(0.5)*dxi); + } else { // AMREX_LO_REFLECT_ODD or homogeneous Dirichlet + ddx = Real(0.); + } + } else { + ddx = mltensor_dx_on_yface(i,j,k,n,vel,dxi); + } + return ddx; +} +} + #if (AMREX_SPACEDIM == 1) #include #elif (AMREX_SPACEDIM == 2)