diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index d352d47a..5e9a70cd 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -8,7 +8,7 @@ on: jobs: CleanUpCcacheCachePostPR: - name: Clean Up Ccahe Cache Post PR + name: Clean Up Ccache Cache Post PR runs-on: ubuntu-latest permissions: actions: write @@ -17,21 +17,27 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 - - name: Clean up ccahe + - name: Clean up ccache run: | gh extension install actions/gh-actions-cache REPO=${{ github.repository }} - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + # For debugging cat ${GITHUB_EVENT_PATH} to see the payload. + + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number') + echo "Post-PR cache cleanup for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge # Setting this to not fail the workflow while deleting cache keys. set +e keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' for k in $keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index 3448d88b..41882334 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -8,7 +8,7 @@ on: jobs: CleanUpCcacheCache: - name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }} + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} runs-on: ubuntu-latest permissions: actions: write @@ -17,7 +17,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 - - name: Clean up ccahe + - name: Clean up ccache run: | gh extension install actions/gh-actions-cache @@ -27,11 +27,14 @@ jobs: EVENT=${{ github.event.workflow_run.event }} # Triggering workflow run name (e.g., LinuxClang) - WORKFLOW_NAME=${{ github.event.workflow_run.name }} + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + + # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload. if [[ $EVENT == "pull_request" ]]; then - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number') + echo "Clean up cache for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge else BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} @@ -45,16 +48,20 @@ jobs: # The goal is to keep the last used key of each job and delete all others. # something like ccache-LinuxClang- - keyprefix=ccache-${WORKFLOW_NAME}- + keyprefix="ccache-${WORKFLOW_NAME}-" - cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq) + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' for j in $cached_jobs do - old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2) + # Delete all entries except the last used one + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done done + unset IFS diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 0a0a7f3e..460cee02 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -143,18 +143,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index cb539a9a..808280bf 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -322,18 +322,3 @@ jobs: export OMP_NUM_THREADS=2 cd ${{ github.workspace }}/incflo build/incflo.ex test_no_eb_2d/benchmark.bouss_bubble_god max_step=10 incflo.verbose=1 mac_proj.verbose=1 nodal_proj.verbose=1 - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 31d1b89f..cce95161 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -75,18 +75,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef37..5f0b1534 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -4,17 +4,13 @@ on: types: - closed +# This workflow does not have the permission to clean up cache for PRs +# originated from a fork. The purpose here is to trigger a workflow_run +# cleanup-cache-postpr.yml that has the right permission. + jobs: - cleanup: + noop: runs-on: ubuntu-latest steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + - name: No OP + run: echo "This workflow is going to trigger CleanUpCachePostPR." diff --git a/.github/workflows/sycl.yml b/.github/workflows/sycl.yml index 678ac755..d5c33980 100644 --- a/.github/workflows/sycl.yml +++ b/.github/workflows/sycl.yml @@ -74,18 +74,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/src/boundary_conditions/README.org b/src/boundary_conditions/README.org index 8adbc3c9..16147d81 100644 --- a/src/boundary_conditions/README.org +++ b/src/boundary_conditions/README.org @@ -1,14 +1,17 @@ * fillpatch -| | pi | po | mi | nsw | sw | dir_dep | -|--------+----------+----------+----------+-------------+---------------------------------------|----------------------- -| v_n | foextrap | foextrap | ext_dir | ext_dir (0) | ext_dir (0) | ext_dir if inflowing | -| v_t | foextrap | foextrap | ext_dir | ext_dir (0) | hoextrap | ext_dir if inflowing | -| rho | foextrap | foextrap | ext_dir | foextrap | if (advection_type == "BDS") foextrap | ext_dir if inflowing | -| | | | | | else hoextrap | ext_dir if inflowing | -| scalar | foextrap | foextrap | ext_dir | foextrap | if (advection_type == "BDS") foextrap | ext_dir if inflowing | -| | | | | | else hoextrap | ext_dir if inflowing | -| force | foextrap | foextrap | foextrap | foextrap | foextrap | foextrap | +| | pi | po | mi | nsw | sw | dir_dep | +|--------+----------+----------+----------+----------------------+---------------------------------------|----------------------- +| v_n | foextrap | foextrap | ext_dir | ext_dir (0) | ext_dir (0) | ext_dir if inflowing | +| | | | | | | foextrap otherwise | +| v_t | foextrap | foextrap | ext_dir | ext_dir (0) | hoextrap | ext_dir if inflowing | +| | | | | | | foextrap otherwise | +| rho | foextrap | foextrap | ext_dir | foextrap | if (advection_type == "BDS") foextrap | ext_dir if inflowing | +| | | | | | else hoextrap | foextrap otherwise | +| tracer | foextrap | foextrap | ext_dir | ext_dir if specified | ext_dir if specified, otherwise | ext_dir if inflowing | +| | | | | foextrap otherwise | if (advection_type == "BDS") foextrap | foextrap otherwise | +| | | | | | else hoextrap | | +| force | foextrap | foextrap | foextrap | foextrap | foextrap | foextrap | * projection @@ -25,8 +28,9 @@ * scalar diffusion -| pi | po | mi | nsw | sw | dir_dep | -|---------+---------+-----------+---------+---------|------------ -| Neumann | Neumann | Dirichlet | Neumann | Neumann | Dirichlet | -| Neumann | Neumann | Dirichlet | Neumann | Neumann | Dirichlet | - +| | pi | po | mi | nsw | sw | dir_dep | +|---------+---------+---------+-----------+------------------------+------------------------|------------ +| v_n | Neumann | Neumann | Dirichlet | Dirichlet | Dirichlet | Dirichlet | +| v_t | Neumann | Neumann | Dirichlet | Dirichlet | Neumann | Dirichlet | +| tracer | Neumann | Neumann | Dirichlet | Dirchelet if specified | Dirichlet if specified | Dirichlet | +| | | | | Neumann otherwise | Neumann otherwise | | diff --git a/src/boundary_conditions/boundary_conditions.cpp b/src/boundary_conditions/boundary_conditions.cpp index a2e3dd72..8cdbb262 100644 --- a/src/boundary_conditions/boundary_conditions.cpp +++ b/src/boundary_conditions/boundary_conditions.cpp @@ -10,6 +10,10 @@ void incflo::init_bcs () { has_inout_bndry = false; + m_bcrec_velocity.resize(AMREX_SPACEDIM); + m_bcrec_density.resize(1); + if (m_ntrac > 0) { m_bcrec_tracer.resize(m_ntrac); } + auto f = [this] (std::string const& bcid, Orientation ori) { m_bc_density[ori] = 1.0; @@ -30,6 +34,13 @@ void incflo::init_bcs () m_bc_type[ori] = BC::pressure_inflow; pp.get("pressure", m_bc_pressure[ori]); + + // Set mathematical BCs here also + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::foextrap);, + m_bcrec_velocity[1].set(ori, BCType::foextrap);, + m_bcrec_velocity[2].set(ori, BCType::foextrap);); + m_bcrec_density[0].set(ori, BCType::foextrap); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::foextrap); } } else if (bc_type == "pressure_outflow" || bc_type == "po") { @@ -38,6 +49,13 @@ void incflo::init_bcs () m_bc_type[ori] = BC::pressure_outflow; pp.get("pressure", m_bc_pressure[ori]); + + // Set mathematical BCs here also + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::foextrap);, + m_bcrec_velocity[1].set(ori, BCType::foextrap);, + m_bcrec_velocity[2].set(ori, BCType::foextrap);); + m_bcrec_density[0].set(ori, BCType::foextrap); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::foextrap); } } else if (bc_type == "mass_inflow" || bc_type == "mi") { @@ -54,6 +72,13 @@ void incflo::init_bcs () pp.query("density", m_bc_density[ori]); pp.queryarr("tracer", m_bc_tracer[ori], 0, m_ntrac); + + // Set mathematical BCs + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::ext_dir);, + m_bcrec_velocity[1].set(ori, BCType::ext_dir);, + m_bcrec_velocity[2].set(ori, BCType::ext_dir);); + m_bcrec_density[0].set(ori, BCType::ext_dir); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::ext_dir); } } else if (bc_type == "direction_dependent" || bc_type == "dd" ) { @@ -72,6 +97,12 @@ void incflo::init_bcs () pp.query("density", m_bc_density[ori]); pp.queryarr("tracer", m_bc_tracer[ori], 0, m_ntrac); + + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::direction_dependent);, + m_bcrec_velocity[1].set(ori, BCType::direction_dependent);, + m_bcrec_velocity[2].set(ori, BCType::direction_dependent);); + m_bcrec_density[0].set(ori, BCType::direction_dependent); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::direction_dependent); } } else if (bc_type == "no_slip_wall" || bc_type == "nsw") { @@ -95,6 +126,17 @@ void incflo::init_bcs () // We potentially read in values at no-slip walls in the event that the // tracer has Dirichlet bcs pp.queryarr("tracer", m_bc_tracer[ori], 0, m_ntrac); + + // Set mathematical BCs + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::ext_dir);, + m_bcrec_velocity[1].set(ori, BCType::ext_dir);, + m_bcrec_velocity[2].set(ori, BCType::ext_dir);); + m_bcrec_density[0].set(ori, BCType::foextrap); + if ( pp.contains("tracer") ) { + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::ext_dir); } + } else { + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::foextrap); } + } } else if (bc_type == "slip_wall" || bc_type == "sw") { @@ -103,13 +145,36 @@ void incflo::init_bcs () m_bc_type[ori] = BC::slip_wall; // These values are set by default above - - // note that we only actually use the zero value for the normal direction; - // the tangential components are set to be first order extrap + // note that we only actually use the zero value for the normal direction. // m_bc_velocity[ori] = {0.0, 0.0, 0.0}; // We potentially read in values at slip walls in the event that the // tracer has Dirichlet bcs pp.queryarr("tracer", m_bc_tracer[ori], 0, m_ntrac); + + // Tangential directions have hoextrap + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::hoextrap);, + m_bcrec_velocity[1].set(ori, BCType::hoextrap);, + m_bcrec_velocity[2].set(ori, BCType::hoextrap);); + // Only normal oriection has ext_dir + m_bcrec_velocity[ori.coordDir()].set(ori, BCType::ext_dir); + if (m_advection_type == "BDS") { + // BDS requires foextrap to avoid introduction of local max/min + m_bcrec_density[0].set(ori, BCType::foextrap); + } else{ + m_bcrec_density[0].set(ori, BCType::hoextrap); + } + if ( pp.contains("tracer") ) { + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::ext_dir); } + } else { + for (auto& b : m_bcrec_tracer) { + if (m_advection_type == "BDS") { + b.set(ori, BCType::foextrap); + } else { + b.set(ori, BCType::hoextrap); + } + } + } } else if (bc_type == "mixed" ) { @@ -143,6 +208,13 @@ void incflo::init_bcs () pp.query("density", m_bc_density[ori]); pp.queryarr("tracer", m_bc_tracer[ori], 0, m_ntrac); + + // Set mathematical BCs. BC_mask will handle Dirichlet part. + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::foextrap);, + m_bcrec_velocity[1].set(ori, BCType::foextrap);, + m_bcrec_velocity[2].set(ori, BCType::foextrap);); + m_bcrec_density[0].set(ori, BCType::foextrap); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::foextrap); } } else { @@ -152,6 +224,13 @@ void incflo::init_bcs () if (geom[0].isPeriodic(ori.coordDir())) { if (m_bc_type[ori] == BC::undefined) { m_bc_type[ori] = BC::periodic; + + // Set mathematical BCs + AMREX_D_TERM(m_bcrec_velocity[0].set(ori, BCType::int_dir);, + m_bcrec_velocity[1].set(ori, BCType::int_dir);, + m_bcrec_velocity[2].set(ori, BCType::int_dir);); + m_bcrec_density[0].set(ori, BCType::int_dir); + for (auto& b : m_bcrec_tracer) { b.set(ori, BCType::int_dir); } } else { amrex::Abort("Wrong BC type for periodic boundary"); } @@ -191,238 +270,25 @@ void incflo::init_bcs () } } - { - m_bcrec_velocity.resize(AMREX_SPACEDIM); - for (OrientationIter oit; oit; ++oit) { - Orientation ori = oit(); - int dir = ori.coordDir(); - Orientation::Side side = ori.faceDir(); - auto const bct = m_bc_type[ori]; - if (bct == BC::pressure_inflow || - bct == BC::pressure_outflow || - bct == BC::mixed ) // BC_mask will handle Dirichlet part. - - { - if (side == Orientation::low) { - AMREX_D_TERM(m_bcrec_velocity[0].setLo(dir, BCType::foextrap);, - m_bcrec_velocity[1].setLo(dir, BCType::foextrap);, - m_bcrec_velocity[2].setLo(dir, BCType::foextrap);); - } else { - AMREX_D_TERM(m_bcrec_velocity[0].setHi(dir, BCType::foextrap);, - m_bcrec_velocity[1].setHi(dir, BCType::foextrap);, - m_bcrec_velocity[2].setHi(dir, BCType::foextrap);); - } - } - else if (bct == BC::mass_inflow || bct == BC::no_slip_wall) - { - if (side == Orientation::low) { - AMREX_D_TERM(m_bcrec_velocity[0].setLo(dir, BCType::ext_dir);, - m_bcrec_velocity[1].setLo(dir, BCType::ext_dir);, - m_bcrec_velocity[2].setLo(dir, BCType::ext_dir);); - } else { - AMREX_D_TERM(m_bcrec_velocity[0].setHi(dir, BCType::ext_dir);, - m_bcrec_velocity[1].setHi(dir, BCType::ext_dir);, - m_bcrec_velocity[2].setHi(dir, BCType::ext_dir);); - } - } - else if (bct == BC::direction_dependent) - { - if (side == Orientation::low) { - AMREX_D_TERM(m_bcrec_velocity[0].setLo(dir, BCType::direction_dependent);, - m_bcrec_velocity[1].setLo(dir, BCType::direction_dependent);, - m_bcrec_velocity[2].setLo(dir, BCType::direction_dependent);); - } else { - AMREX_D_TERM(m_bcrec_velocity[0].setHi(dir, BCType::direction_dependent);, - m_bcrec_velocity[1].setHi(dir, BCType::direction_dependent);, - m_bcrec_velocity[2].setHi(dir, BCType::direction_dependent);); - } - } - else if (bct == BC::slip_wall) - { - if (side == Orientation::low) { - // Tangential directions have hoextrap - AMREX_D_TERM(m_bcrec_velocity[0].setLo(dir, BCType::hoextrap);, - m_bcrec_velocity[1].setLo(dir, BCType::hoextrap);, - m_bcrec_velocity[2].setLo(dir, BCType::hoextrap);); - // Only normal direction has ext_dir - m_bcrec_velocity[dir].setLo(dir, BCType::ext_dir); - - } else { - // Tangential directions have hoextrap - AMREX_D_TERM(m_bcrec_velocity[0].setHi(dir, BCType::hoextrap);, - m_bcrec_velocity[1].setHi(dir, BCType::hoextrap);, - m_bcrec_velocity[2].setHi(dir, BCType::hoextrap);); - // Only normal direction has ext_dir - m_bcrec_velocity[dir].setHi(dir, BCType::ext_dir); - } - } - else if (bct == BC::periodic) - { - if (side == Orientation::low) { - AMREX_D_TERM(m_bcrec_velocity[0].setLo(dir, BCType::int_dir);, - m_bcrec_velocity[1].setLo(dir, BCType::int_dir);, - m_bcrec_velocity[2].setLo(dir, BCType::int_dir);); - } else { - AMREX_D_TERM(m_bcrec_velocity[0].setHi(dir, BCType::int_dir);, - m_bcrec_velocity[1].setHi(dir, BCType::int_dir);, - m_bcrec_velocity[2].setHi(dir, BCType::int_dir);); - } - } - } - m_bcrec_velocity_d.resize(AMREX_SPACEDIM); + // Copy BCRecs to device container + m_bcrec_velocity_d.resize(AMREX_SPACEDIM); #ifdef AMREX_USE_GPU - Gpu::htod_memcpy + Gpu::htod_memcpy #else - std::memcpy + std::memcpy #endif - (m_bcrec_velocity_d.data(), m_bcrec_velocity.data(), sizeof(BCRec)*AMREX_SPACEDIM); - } + (m_bcrec_velocity_d.data(), m_bcrec_velocity.data(), sizeof(BCRec)*AMREX_SPACEDIM); - { - m_bcrec_density.resize(1); - for (OrientationIter oit; oit; ++oit) { - Orientation ori = oit(); - int dir = ori.coordDir(); - Orientation::Side side = ori.faceDir(); - auto const bct = m_bc_type[ori]; - if (bct == BC::pressure_inflow || - bct == BC::pressure_outflow || - bct == BC::no_slip_wall || - bct == BC::mixed) - { - if (side == Orientation::low) { - m_bcrec_density[0].setLo(dir, BCType::foextrap); - } else { - m_bcrec_density[0].setHi(dir, BCType::foextrap); - } - } - else if (bct == BC::slip_wall) - { - if (side == Orientation::low) { - // BDS requires foextrap to avoid introduction of local max/min - if (m_advection_type == "BDS") { - m_bcrec_density[0].setLo(dir, BCType::foextrap); - } else{ - m_bcrec_density[0].setLo(dir, BCType::hoextrap); - } - } else { - // BDS requires foextrap to avoid introduction of local max/min - if (m_advection_type == "BDS") { - m_bcrec_density[0].setHi(dir, BCType::foextrap); - } else { - m_bcrec_density[0].setHi(dir, BCType::hoextrap); - } - } - } - else if (bct == BC::mass_inflow) - { - if (side == Orientation::low) { - m_bcrec_density[0].setLo(dir, BCType::ext_dir); - } else { - m_bcrec_density[0].setHi(dir, BCType::ext_dir); - } - } - else if (bct == BC::direction_dependent) - { - if (side == Orientation::low) { - m_bcrec_density[0].setLo(dir, BCType::direction_dependent); - } else { - m_bcrec_density[0].setHi(dir, BCType::direction_dependent); - } - } - else if (bct == BC::periodic) - { - if (side == Orientation::low) { - m_bcrec_density[0].setLo(dir, BCType::int_dir); - } else { - m_bcrec_density[0].setHi(dir, BCType::int_dir); - } - } - } - m_bcrec_density_d.resize(1); + m_bcrec_density_d.resize(1); #ifdef AMREX_USE_GPU - Gpu::htod_memcpy + Gpu::htod_memcpy #else - std::memcpy + std::memcpy #endif - (m_bcrec_density_d.data(), m_bcrec_density.data(), sizeof(BCRec)); - } + (m_bcrec_density_d.data(), m_bcrec_density.data(), sizeof(BCRec)); if (m_ntrac > 0) { - m_bcrec_tracer.resize(m_ntrac); - for (OrientationIter oit; oit; ++oit) { - Orientation ori = oit(); - int dir = ori.coordDir(); - Orientation::Side side = ori.faceDir(); - auto const bct = m_bc_type[ori]; - if (bct == BC::pressure_inflow || - bct == BC::pressure_outflow || - bct == BC::mixed) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) b.setLo(dir, BCType::foextrap); - } else { - for (auto& b : m_bcrec_tracer) b.setHi(dir, BCType::foextrap); - } - } - else if (bct == BC::no_slip_wall) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) b.setLo(dir, BCType::foextrap); - } else { - for (auto& b : m_bcrec_tracer) b.setHi(dir, BCType::foextrap); - } - } - else if (bct == BC::slip_wall) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) { - // BDS requires foextrap to avoid introduction - // of local max/min - if (m_advection_type == "BDS") { - b.setLo(dir, BCType::foextrap); - } else { - b.setLo(dir, BCType::hoextrap); - } - } - } else { - for (auto& b : m_bcrec_tracer) { - // BDS requires foextrap to avoid introduction - // of local max/min - if (m_advection_type == "BDS") { - b.setHi(dir, BCType::foextrap); - } else { - b.setHi(dir, BCType::hoextrap); - } - } - } - } - else if (bct == BC::mass_inflow) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) b.setLo(dir, BCType::ext_dir); - } else { - for (auto& b : m_bcrec_tracer) b.setHi(dir, BCType::ext_dir); - } - } - else if (bct == BC::direction_dependent) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) b.setLo(dir, BCType::direction_dependent); - } else { - for (auto& b : m_bcrec_tracer) b.setHi(dir, BCType::direction_dependent); - } - } - else if (bct == BC::periodic) - { - if (side == Orientation::low) { - for (auto& b : m_bcrec_tracer) b.setLo(dir, BCType::int_dir); - } else { - for (auto& b : m_bcrec_tracer) b.setHi(dir, BCType::int_dir); - } - } - } m_bcrec_tracer_d.resize(m_ntrac); #ifdef AMREX_USE_GPU Gpu::htod_memcpy diff --git a/src/diffusion/DiffusionScalarOp.cpp b/src/diffusion/DiffusionScalarOp.cpp index 1c5c8bf2..5ccd8af6 100644 --- a/src/diffusion/DiffusionScalarOp.cpp +++ b/src/diffusion/DiffusionScalarOp.cpp @@ -32,8 +32,11 @@ DiffusionScalarOp::DiffusionScalarOp (incflo* a_incflo) m_incflo->DistributionMap(0,finest_level), info_solve, ebfact); m_eb_scal_solve_op->setMaxOrder(m_mg_maxorder); - m_eb_scal_solve_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low ), - m_incflo->get_diffuse_scalar_bc(Orientation::high)); + // For now, code requires (in more than 1 place) that m_ntrac>=1 and all the tracers have the same BCs + m_eb_scal_solve_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low, + m_incflo->m_bcrec_tracer[0].lo()), + m_incflo->get_diffuse_scalar_bc(Orientation::high, + m_incflo->m_bcrec_tracer[0].hi())); if (!m_incflo->useTensorSolve()) { @@ -54,8 +57,10 @@ DiffusionScalarOp::DiffusionScalarOp (incflo* a_incflo) info_apply, ebfact); m_eb_scal_apply_op->setMaxOrder(m_mg_maxorder); - m_eb_scal_apply_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low), - m_incflo->get_diffuse_scalar_bc(Orientation::high)); + m_eb_scal_apply_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low, + m_incflo->m_bcrec_tracer[0].lo()), + m_incflo->get_diffuse_scalar_bc(Orientation::high, + m_incflo->m_bcrec_tracer[0].hi())); } if ( (m_incflo->need_divtau() && !m_incflo->useTensorSolve()) || @@ -78,8 +83,10 @@ DiffusionScalarOp::DiffusionScalarOp (incflo* a_incflo) m_incflo->DistributionMap(0,m_incflo->finestLevel()), info_solve); m_reg_scal_solve_op->setMaxOrder(m_mg_maxorder); - m_reg_scal_solve_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low), - m_incflo->get_diffuse_scalar_bc(Orientation::high)); + m_reg_scal_solve_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low, + m_incflo->m_bcrec_tracer[0].lo()), + m_incflo->get_diffuse_scalar_bc(Orientation::high, + m_incflo->m_bcrec_tracer[0].hi())); if (!m_incflo->useTensorSolve()) { @@ -97,8 +104,10 @@ DiffusionScalarOp::DiffusionScalarOp (incflo* a_incflo) m_incflo->DistributionMap(0,m_incflo->finestLevel()), info_apply); m_reg_scal_apply_op->setMaxOrder(m_mg_maxorder); - m_reg_scal_apply_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low), - m_incflo->get_diffuse_scalar_bc(Orientation::high)); + m_reg_scal_apply_op->setDomainBC(m_incflo->get_diffuse_scalar_bc(Orientation::low, + m_incflo->m_bcrec_tracer[0].lo()), + m_incflo->get_diffuse_scalar_bc(Orientation::high, + m_incflo->m_bcrec_tracer[0].hi())); } if ( (m_incflo->need_divtau() && !m_incflo->useTensorSolve()) || diff --git a/src/diffusion/incflo_diffusion.cpp b/src/diffusion/incflo_diffusion.cpp index 35e81fc3..e869692c 100644 --- a/src/diffusion/incflo_diffusion.cpp +++ b/src/diffusion/incflo_diffusion.cpp @@ -210,7 +210,7 @@ incflo::get_diffuse_velocity_bc (Orientation::Side side, int comp) const noexcep } Array -incflo::get_diffuse_scalar_bc (Orientation::Side side) const noexcept +incflo::get_diffuse_scalar_bc (Orientation::Side side, const int* bcr) const noexcept { Array r; for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { @@ -231,13 +231,12 @@ incflo::get_diffuse_scalar_bc (Orientation::Side side) const noexcept break; } case BC::slip_wall: - { - r[dir] = LinOpBCType::Neumann; - break; - } case BC::no_slip_wall: { r[dir] = LinOpBCType::Neumann; + if ( bcr[dir] == BCType::ext_dir ) { + r[dir] = LinOpBCType::Dirichlet; + } break; } case BC::mass_inflow: diff --git a/src/incflo.H b/src/incflo.H index 78118210..ea630188 100644 --- a/src/incflo.H +++ b/src/incflo.H @@ -345,13 +345,16 @@ private: // Be verbose? int m_verbose = 0; + // number of tracers + int m_ntrac = 1; + // Member variables for initial conditions int m_probtype = 0; amrex::Real m_ic_u = amrex::Real(0.0); amrex::Real m_ic_v = amrex::Real(0.0); amrex::Real m_ic_w = amrex::Real(0.0); amrex::Real m_ic_p = amrex::Real(0.0); - amrex::Real m_ic_t = amrex::Real(0.0); + amrex::Vector m_ic_t = amrex::Vector (m_ntrac, amrex::Real(0.0)); amrex::Vector m_t_old; amrex::Vector m_t_new; @@ -695,8 +698,6 @@ private: amrex::Vector m_iconserv_tracer; amrex::Gpu::DeviceVector m_iconserv_tracer_d; - int m_ntrac = 1; - std::unique_ptr m_diffusion_tensor_op; std::unique_ptr m_diffusion_scalar_op; @@ -858,7 +859,7 @@ private: get_diffuse_tensor_bc (amrex::Orientation::Side side) const noexcept; [[nodiscard]] amrex::Array - get_diffuse_scalar_bc (amrex::Orientation::Side side) const noexcept; + get_diffuse_scalar_bc (amrex::Orientation::Side side, const int* bcr) const noexcept; void fillpatch_velocity (int lev, amrex::Real time, amrex::MultiFab& vel, int ng); void fillpatch_density (int lev, amrex::Real time, amrex::MultiFab& density, int ng); diff --git a/src/prob/prob_init_fluid.cpp b/src/prob/prob_init_fluid.cpp index 06d6fe53..700addf1 100644 --- a/src/prob/prob_init_fluid.cpp +++ b/src/prob/prob_init_fluid.cpp @@ -20,7 +20,9 @@ void incflo::prob_init_fluid (int lev) ld.velocity.setVal(m_ic_v, 1, 1);, ld.velocity.setVal(m_ic_w, 2, 1);); - if (m_ntrac > 0) ld.tracer.setVal(0.0); + for (int comp = 0; comp < m_ntrac; comp++) { + ld.tracer.setVal(m_ic_t[comp], comp, 1); + } for (MFIter mfi(ld.density); mfi.isValid(); ++mfi) { diff --git a/src/setup/init.cpp b/src/setup/init.cpp index c7d610ca..b8747f0b 100644 --- a/src/setup/init.cpp +++ b/src/setup/init.cpp @@ -123,13 +123,23 @@ void incflo::ReadParameters () amrex::Abort("We currently require cfl <= 1.0 when using this advection scheme"); } + pp.query("ntrac", m_ntrac); + + if (m_ntrac <= 0) m_advect_tracer = false; + + if (m_ntrac < 1) { + amrex::Abort("We currently require at least one tracer"); + } + // Initial conditions pp.query("probtype", m_probtype); pp.query("ic_u", m_ic_u); pp.query("ic_v", m_ic_v); pp.query("ic_w", m_ic_w); pp.query("ic_p", m_ic_p); - pp.query("ic_t", m_ic_t); + if ( !pp.queryarr("ic_t", m_ic_t, 0, m_ntrac) ) { + m_ic_t.resize(m_ntrac, 0.); + } // Viscosity (if constant) pp.query("mu", m_mu); @@ -138,14 +148,6 @@ void incflo::ReadParameters () pp.query("ro_0", m_ro_0); AMREX_ALWAYS_ASSERT(m_ro_0 >= 0.0); - pp.query("ntrac", m_ntrac); - - if (m_ntrac <= 0) m_advect_tracer = false; - - if (m_ntrac < 1) { - amrex::Abort("We currently require at least one tracer"); - } - // Scalar diffusion coefficients m_mu_s.resize(m_ntrac, 0.0); pp.queryarr("mu_s", m_mu_s, 0, m_ntrac );