diff --git a/CHANGES.md b/CHANGES.md index 7db160e12f..15ed72d5aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,18 @@ +# 23.08 + + * Time evolution without subcycling on the fine levels, which is enabled via + the runtime parameter amr.subcycling_mode = "None", has been significantly + rewritten to improve computational performance for certain cases. When the + fine levels do not subcycle, the evolution can be done by processing all + of the hydro updates on the fine level together and then immediately doing + the flux correction to sync the coarse and fine level fluxes at the + boundary between levels. This is how many AMR codes that do not subcycle + are written. Castro now does this when the user chooses not to subcycle. + The benefit of this approach is most evidence for problems with Poisson + gravity that use a multigrid solve, as we can significantly reduce the + number of Poisson solves per step, performing only a single composite + (multi-level) solve at the new-time simultaneously for all levels. (#2505) + # 23.07 * The parameter castro.state_nghost, which allowed State_Type to have ghost diff --git a/Source/diffusion/Castro_diffusion.cpp b/Source/diffusion/Castro_diffusion.cpp index 5fffe05542..61b28bfbd5 100644 --- a/Source/diffusion/Castro_diffusion.cpp +++ b/Source/diffusion/Castro_diffusion.cpp @@ -31,7 +31,7 @@ Castro::construct_old_diff_source(MultiFab& source, MultiFab& state_in, Real tim #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -68,7 +68,7 @@ Castro::construct_new_diff_source(MultiFab& source, MultiFab& state_old, MultiFa #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/driver/Castro.H b/Source/driver/Castro.H index fd7e254d50..5781b1e899 100644 --- a/Source/driver/Castro.H +++ b/Source/driver/Castro.H @@ -1036,8 +1036,9 @@ public: /// /// @param crse_level /// @param fine_level +/// @param in_post_timestep /// - void reflux (int crse_level, int fine_level); + void reflux (int crse_level, int fine_level, bool in_post_timestep); /// diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp index f0dab7d807..7ef73d48e4 100644 --- a/Source/driver/Castro.cpp +++ b/Source/driver/Castro.cpp @@ -1951,7 +1951,9 @@ Castro::post_timestep (int iteration_local) // will also do the sync solve associated with the reflux. if (do_reflux && level < parent->finestLevel()) { - reflux(level, level+1); + if (parent->subcyclingMode() != "None") { + reflux(level, level+1, true); + } } // Ensure consistency with finer grids. @@ -2614,9 +2616,31 @@ Castro::FluxRegFineAdd() { } +// reflux() synchronizes fluxes between levels and has two modes of operation. +// +// When in_post_timestep = true, we are performing the reflux in AmrLevel's +// post_timestep() routine. This takes place after this level has completed +// its advance as well as all fine levels above this level have completed +// their advance. The main activities are: +// +// (1) Limit the flux correction so that it won't cause unphysical densities. +// (2) Update any copies of the full flux arrays with the flux correction. +// (3) Perform the flux correction on StateData using FluxRegister's Reflux(). +// (4) Do a sync solve for Poisson gravity now that the density has changed. +// (5) Recompute the new-time sources now that StateData has changed. +// +// When in_post_timestep = false, we are performing the reflux immediately +// after the hydro step has taken place on all levels between crse_level and +// fine_level. This happens when amr.subcycling_mode = None, or in general when +// the fine level does not subcycle relative to the coarse level. In this mode +// we can skip steps (4) and (5), as those steps are only necessary to fix the +// updates on these levels that took place without full information about the +// hydro solve being synchronized. The advance on crse_level and fine_level will +// then be followed by a normal computation of the new-time gravity and new-time +// sources, which will not need to be corrected later. void -Castro::reflux(int crse_level, int fine_level) +Castro::reflux (int crse_level, int fine_level, bool in_post_timestep) { BL_PROFILE("Castro::reflux()"); @@ -2630,7 +2654,7 @@ Castro::reflux(int crse_level, int fine_level) Vector > drho(nlevs); Vector > dphi(nlevs); - if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0) { + if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0 && in_post_timestep) { for (int lev = crse_level; lev <= fine_level; ++lev) { @@ -2783,7 +2807,7 @@ Castro::reflux(int crse_level, int fine_level) // Update the coarse fluxes MultiFabs using the reflux data. This should only make // a difference if we re-evaluate the source terms later. - if (update_sources_after_reflux) { + if (update_sources_after_reflux || !in_post_timestep) { MultiFab::Add(*crse_lev.fluxes[idir], temp_fluxes[idir], 0, 0, crse_lev.fluxes[idir]->nComp(), 0); @@ -2803,7 +2827,7 @@ Castro::reflux(int crse_level, int fine_level) #ifdef GRAVITY int ilev = lev - crse_level - 1; - if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0) { + if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0 && in_post_timestep) { reg->Reflux(*drho[ilev], crse_lev.volume, 1.0, 0, URHO, 1, crse_lev.geom); amrex::average_down(*drho[ilev + 1], *drho[ilev], 0, 1, getLevel(lev).crse_ratio); } @@ -2826,7 +2850,7 @@ Castro::reflux(int crse_level, int fine_level) reg->Reflux(crse_state, dr, 1.0, 0, UMX, 1, crse_lev.geom); - if (update_sources_after_reflux) { + if (update_sources_after_reflux || !in_post_timestep) { MultiFab tmp_fluxes(crse_lev.P_radial.boxArray(), crse_lev.P_radial.DistributionMap(), @@ -2863,7 +2887,7 @@ Castro::reflux(int crse_level, int fine_level) reg->Reflux(crse_lev.get_new_data(Rad_Type), crse_lev.volume, 1.0, 0, 0, Radiation::nGroups, crse_lev.geom); - if (update_sources_after_reflux) { + if (update_sources_after_reflux || !in_post_timestep) { for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { @@ -2893,7 +2917,7 @@ Castro::reflux(int crse_level, int fine_level) #endif #ifdef GRAVITY - if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0) { + if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0 && in_post_timestep) { reg = &getLevel(lev).phi_reg; @@ -2922,7 +2946,7 @@ Castro::reflux(int crse_level, int fine_level) // Do the sync solve across all levels. #ifdef GRAVITY - if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0) { + if (do_grav && gravity->get_gravity_type() == "PoissonGrav" && gravity->NoSync() == 0 && in_post_timestep) { gravity->gravity_sync(crse_level, fine_level, amrex::GetVecOfPtrs(drho), amrex::GetVecOfPtrs(dphi)); } #endif @@ -2938,7 +2962,7 @@ Castro::reflux(int crse_level, int fine_level) // ghost zone fills like diffusion depend on the data in the // coarser levels. - if (update_sources_after_reflux && + if (update_sources_after_reflux && in_post_timestep && (time_integration_method == CornerTransportUpwind || time_integration_method == SimplifiedSpectralDeferredCorrections)) { diff --git a/Source/driver/Castro_advance.cpp b/Source/driver/Castro_advance.cpp index 9868a5054b..00f078226a 100644 --- a/Source/driver/Castro_advance.cpp +++ b/Source/driver/Castro_advance.cpp @@ -38,6 +38,11 @@ Castro::advance (Real time, // amr_ncycle : the number of subcycles at this level { + if (parent->subcyclingMode() == "None" && level > 0) { + amrex::Print() << "\n Advance at this level has already been completed.\n\n"; + return dt; + } + BL_PROFILE("Castro::advance()"); // Save the wall time when we started the step. @@ -48,7 +53,15 @@ Castro::advance (Real time, Real dt_new = dt; - initialize_advance(time, dt, amr_iteration); + int max_level_to_advance = level; + + if (parent->subcyclingMode() == "None" && level == 0) { + max_level_to_advance = parent->finestLevel(); + } + + for (int lev = level; lev <= max_level_to_advance; ++lev) { + getLevel(lev).initialize_advance(time, dt, amr_iteration); + } // Do the advance. @@ -87,27 +100,29 @@ Castro::advance (Real time, post_step_regrid = 1; } + for (int lev = level; lev <= max_level_to_advance; ++lev) { #ifdef AUX_UPDATE - advance_aux(time, dt); + getLevel(lev).advance_aux(time, dt); #endif #ifdef GRAVITY - // Update the point mass. - if (use_point_mass == 1) { - pointmass_update(time, dt); - } + // Update the point mass. + if (use_point_mass == 1) { + getLevel(lev).pointmass_update(time, dt); + } #endif #ifdef RADIATION - MultiFab& S_new = get_new_data(State_Type); - final_radiation_call(S_new, amr_iteration, amr_ncycle); + MultiFab& S_new = getLevel(lev).get_new_data(State_Type); + getLevel(lev).final_radiation_call(S_new, amr_iteration, amr_ncycle); #endif #ifdef AMREX_PARTICLES - advance_particles(amr_iteration, time, dt); + getLevel(lev).advance_particles(amr_iteration, time, dt); #endif - finalize_advance(); + getLevel(lev).finalize_advance(); + } return dt_new; } @@ -574,7 +589,7 @@ Castro::finalize_advance() { BL_PROFILE("Castro::finalize_advance()"); - if (do_reflux == 1) { + if (do_reflux == 1 && parent->subcyclingMode() != "None") { FluxRegCrseInit(); FluxRegFineAdd(); } @@ -584,7 +599,7 @@ Castro::finalize_advance() // the fluxes from the full timestep (this will be used // later during the reflux operation). - if (do_reflux == 1 && update_sources_after_reflux == 1) { + if (do_reflux == 1 && update_sources_after_reflux == 1 && parent->subcyclingMode() != "None") { for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { MultiFab::Copy(*mass_fluxes[idir], *fluxes[idir], URHO, 0, 1, 0); } @@ -628,14 +643,34 @@ Castro::finalize_advance() // Record how many zones we have advanced. - num_zones_advanced += static_cast(grids.numPts()) / static_cast(getLevel(0).grids.numPts()); + int max_level_to_advance = level; + + if (parent->subcyclingMode() == "None") { + max_level_to_advance = parent->finestLevel(); + } + + long num_pts_advanced = 0; + + for (int lev = level; lev <= max_level_to_advance; ++lev) { + num_pts_advanced += getLevel(lev).grids.numPts(); + } + + num_zones_advanced += static_cast(num_pts_advanced) / static_cast(getLevel(0).grids.numPts()); Real wall_time = ParallelDescriptor::second() - wall_time_start; - Real fom_advance = static_cast(grids.numPts()) / wall_time / 1.e6; + + Real fom_advance = static_cast(num_pts_advanced) / wall_time / 1.e6; if (verbose >= 1) { - amrex::Print() << " Zones advanced per microsecond at this level: " - << fom_advance << std::endl << std::endl; + if (max_level_to_advance > 0) { + if (level == 0) { + amrex::Print() << " Zones advanced per microsecond from level " << level << " to level " + << max_level_to_advance << ": " << fom_advance << std::endl << std::endl; + } + } + else { + amrex::Print() << " Zones advanced per microsecond at this level: " + << fom_advance << std::endl << std::endl; + } } - } diff --git a/Source/driver/Castro_advance_ctu.cpp b/Source/driver/Castro_advance_ctu.cpp index e6eeb2fb34..77e9358b81 100644 --- a/Source/driver/Castro_advance_ctu.cpp +++ b/Source/driver/Castro_advance_ctu.cpp @@ -26,90 +26,110 @@ Castro::do_advance_ctu (Real time, Real dt) #ifndef TRUE_SDC + // Advance simultaneously on all levels that are not subcycling + // relative to this level. + + int max_level_to_advance = level; + + if (parent->subcyclingMode() == "None" && level == 0) { + max_level_to_advance = parent->finestLevel(); + } + const Real prev_time = state[State_Type].prevTime(); const Real cur_time = state[State_Type].curTime(); - // Perform initialization steps. + for (int lev = level; lev <= max_level_to_advance; ++lev) { + // Perform initialization steps. - status = initialize_do_advance(time, dt); + status = getLevel(lev).initialize_do_advance(time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } - // Perform all pre-advance operations and then initialize - // the new-time state with the output of those operators. + // Perform all pre-advance operations and then initialize + // the new-time state with the output of those operators. - status = pre_advance_operators(prev_time, dt); + status = getLevel(lev).pre_advance_operators(prev_time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } - // Construct the old-time sources from Sborder. This will already - // be applied to S_new (with full dt weighting), to be corrected - // later. Note that this does not affect the prediction of the - // interface state; an explicit source will be traced there as - // needed. + // Construct the old-time sources from Sborder. This will already + // be applied to S_new (with full dt weighting), to be corrected + // later. Note that this does not affect the prediction of the + // interface state; an explicit source will be traced there as + // needed. - status = do_old_sources(prev_time, dt); + status = getLevel(lev).do_old_sources(prev_time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } - // Perform any operations that occur after the sources but before the hydro. + // Perform any operations that occur after the sources but before the hydro. - status = pre_hydro_operators(prev_time, dt); + status = getLevel(lev).pre_hydro_operators(prev_time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } - // Do the hydro update. We build directly off of Sborder, which - // is the state that has already seen the burn. + // Do the hydro update. We build directly off of Sborder, which + // is the state that has already seen the burn. #ifndef MHD - status = construct_ctu_hydro_source(prev_time, dt); + status = getLevel(lev).construct_ctu_hydro_source(prev_time, dt); #else - status = construct_ctu_mhd_source(prev_time, dt); + status = getLevel(lev).construct_ctu_mhd_source(prev_time, dt); #endif - if (status.success == false) { - return status; + if (status.success == false) { + return status; + } } - // Perform any operations that occur after the hydro but before - // the corrector sources. - - status = post_hydro_operators(cur_time, dt); + // We can perform the reflux immediately if there's no subcycling + // above this level. - if (status.success == false) { - return status; + if (do_reflux && level < max_level_to_advance) { + reflux(level, max_level_to_advance, false); } - // Construct and apply new-time source terms. + for (int lev = level; lev <= max_level_to_advance; ++lev) { + // Perform any operations that occur after the hydro but before + // the corrector sources. - status = do_new_sources(cur_time, dt); + status = getLevel(lev).post_hydro_operators(cur_time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } - // Do the second half of the reactions for Strang, or the full burn for simplified SDC. + // Construct and apply new-time source terms. - status = post_advance_operators(cur_time, dt); + status = getLevel(lev).do_new_sources(cur_time, dt); - if (status.success == false) { - return status; - } + if (status.success == false) { + return status; + } + + // Do the second half of the reactions for Strang, or the full burn for simplified SDC. + + status = getLevel(lev).post_advance_operators(cur_time, dt); + + if (status.success == false) { + return status; + } - // Perform finalization steps. + // Perform finalization steps. - status = finalize_do_advance(cur_time, dt); + status = getLevel(lev).finalize_do_advance(cur_time, dt); - if (status.success == false) { - return status; + if (status.success == false) { + return status; + } } #endif @@ -230,6 +250,12 @@ Castro::subcycle_advance_ctu(const Real time, const Real dt, int amr_iteration, sub_iteration = 0; + int max_level_to_advance = level; + + if (parent->subcyclingMode() == "None" && level == 0) { + max_level_to_advance = parent->finestLevel(); + } + // Subcycle until we've reached the target time. // Compare against a slightly smaller number to // avoid roundoff concerns. @@ -351,7 +377,10 @@ Castro::subcycle_advance_ctu(const Real time, const Real dt, int amr_iteration, for (int n = 0; n < num_sub_iters; ++n) { if (time_integration_method == SimplifiedSpectralDeferredCorrections) { - sdc_iteration = n; + for (int lev = level; lev <= max_level_to_advance; ++lev) { + getLevel(lev).sdc_iteration = n; + } + amrex::Print() << "Beginning SDC iteration " << n + 1 << " of " << num_sub_iters << "." << std::endl << std::endl; } diff --git a/Source/gravity/Castro_gravity.cpp b/Source/gravity/Castro_gravity.cpp index f0cecf9f44..094727e6a8 100644 --- a/Source/gravity/Castro_gravity.cpp +++ b/Source/gravity/Castro_gravity.cpp @@ -39,8 +39,17 @@ Castro::construct_old_gravity (Real time) // difference between the multilevel and the single level solutions. // Note that we don't need to do this solve for single-level runs, // since the solution at the end of the last timestep won't have changed. + // Similarly, we can skip this if we aren't subcycling on this level + // or all levels above this level, since the new-time composite solve + // at the new time in the last step will still be valid. - if (gravity->get_gravity_type() == "PoissonGrav" && parent->finestLevel() > 0) + bool do_old_solve = true; + + if (parent->subcyclingMode() == "None" || parent->finestLevel() == 0) { + do_old_solve = false; + } + + if (gravity->get_gravity_type() == "PoissonGrav" && do_old_solve) { // Create a copy of the current (composite) data on this level. @@ -122,7 +131,7 @@ Castro::construct_old_gravity (Real time) #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -154,78 +163,94 @@ Castro::construct_new_gravity (Real time) } - // If we're doing Poisson gravity, do the new-time level solve here. + // If we're doing Poisson gravity, do the new-time level or composite solve here. if (gravity->get_gravity_type() == "PoissonGrav") { + if (level == 0 && parent->subcyclingMode() == "None") { + if (castro::verbose > 0) { + amrex::Print() << "\n... new-time composite Poisson gravity solve from level " << level << " to level " << parent->finestLevel() << std::endl << std::endl; + } - // Use the "old" phi from the current time step as a guess for this solve. - - MultiFab& phi_old = get_old_data(PhiGrav_Type); + // Use the "old" phi from the current time step as a guess for this solve. - MultiFab::Copy(phi_new, phi_old, 0, 0, 1, phi_new.nGrow()); + for (int lev = level; lev <= parent->finestLevel(); ++lev) { + MultiFab& lev_phi_old = getLevel(lev).get_old_data(PhiGrav_Type); + MultiFab& lev_phi_new = getLevel(lev).get_new_data(PhiGrav_Type); - // Subtract off the (composite - level) contribution for the purposes - // of the level solve. We'll add it back later. + MultiFab::Copy(lev_phi_new, lev_phi_old, 0, 0, 1, lev_phi_new.nGrow()); + } - if (gravity->DoCompositeCorrection() && level < parent->finestLevel() && level <= gravity->get_max_solve_level()) { - phi_new.minus(comp_minus_level_phi, 0, 1, 0); + gravity->multilevel_solve_for_new_phi(level, parent->finestLevel()); } + else if (parent->subcyclingMode() != "None") { + // Use the "old" phi from the current time step as a guess for this solve. - if (castro::verbose && ParallelDescriptor::IOProcessor()) { - std::cout << "... new-time level Poisson gravity solve at level " << level << std::endl << std::endl; - } + MultiFab& phi_old = get_old_data(PhiGrav_Type); - int is_new = 1; + MultiFab::Copy(phi_new, phi_old, 0, 0, 1, phi_new.nGrow()); - gravity->solve_for_phi(level, - phi_new, - amrex::GetVecOfPtrs(gravity->get_grad_phi_curr(level)), - is_new); + // Subtract off the (composite - level) contribution for the purposes + // of the level solve. We'll add it back later. - if (gravity->DoCompositeCorrection() == 1 && level < parent->finestLevel() && level <= gravity->get_max_solve_level()) { + if (gravity->DoCompositeCorrection() && level < parent->finestLevel() && level <= gravity->get_max_solve_level()) { + phi_new.minus(comp_minus_level_phi, 0, 1, 0); + } - if (gravity->test_results_of_solves() == 1) { + if (castro::verbose && ParallelDescriptor::IOProcessor()) { + std::cout << "... new-time level Poisson gravity solve at level " << level << std::endl << std::endl; + } - if (castro::verbose && ParallelDescriptor::IOProcessor()) { - std::cout << " " << '\n'; - std::cout << "... testing grad_phi_curr before adding comp_minus_level_grad_phi " << '\n'; - } + int is_new = 1; - gravity->test_level_grad_phi_curr(level); + gravity->solve_for_phi(level, + phi_new, + amrex::GetVecOfPtrs(gravity->get_grad_phi_curr(level)), + is_new); - } + if (gravity->DoCompositeCorrection() == 1 && level < parent->finestLevel() && level <= gravity->get_max_solve_level()) { - // Add back the (composite - level) contribution. This ensures that - // if we are not doing a sync solve, then we still get the difference - // between the composite and level solves added to the force we - // calculate, so it is slightly more accurate than it would have been. + if (gravity->test_results_of_solves() == 1) { - phi_new.plus(comp_minus_level_phi, 0, 1, 0); - for (int n = 0; n < AMREX_SPACEDIM; ++n) { - gravity->get_grad_phi_curr(level)[n]->plus(*comp_minus_level_grad_phi[n], 0, 1, 0); - } + if (castro::verbose && ParallelDescriptor::IOProcessor()) { + std::cout << " " << '\n'; + std::cout << "... testing grad_phi_curr before adding comp_minus_level_grad_phi " << '\n'; + } - if (gravity->test_results_of_solves() == 1) { + gravity->test_level_grad_phi_curr(level); - if (castro::verbose && ParallelDescriptor::IOProcessor()) { - std::cout << " " << '\n'; - std::cout << "... testing grad_phi_curr after adding comp_minus_level_grad_phi " << '\n'; } - gravity->test_level_grad_phi_curr(level); + // Add back the (composite - level) contribution. This ensures that + // if we are not doing a sync solve, then we still get the difference + // between the composite and level solves added to the force we + // calculate, so it is slightly more accurate than it would have been. - } + phi_new.plus(comp_minus_level_phi, 0, 1, 0); + for (int n = 0; n < AMREX_SPACEDIM; ++n) { + gravity->get_grad_phi_curr(level)[n]->plus(*comp_minus_level_grad_phi[n], 0, 1, 0); + } - } + if (gravity->test_results_of_solves() == 1) { + if (castro::verbose && ParallelDescriptor::IOProcessor()) { + std::cout << " " << '\n'; + std::cout << "... testing grad_phi_curr after adding comp_minus_level_grad_phi " << '\n'; + } + + gravity->test_level_grad_phi_curr(level); + + } + + } + } } // Define new gravity vector. gravity->get_new_grav_vector(level, grav_new, time); - if (gravity->get_gravity_type() == "PoissonGrav" && level <= gravity->get_max_solve_level()) { + if (gravity->get_gravity_type() == "PoissonGrav" && level <= gravity->get_max_solve_level() && parent->subcyclingMode() != "None") { if (gravity->DoCompositeCorrection() == 1 && level < parent->finestLevel()) { @@ -262,7 +287,7 @@ Castro::construct_new_gravity (Real time) #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_gravity() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_gravity() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -414,7 +439,7 @@ void Castro::construct_old_gravity_source(MultiFab& source, MultiFab& state_in, #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -651,7 +676,7 @@ void Castro::construct_new_gravity_source(MultiFab& source, MultiFab& state_old, #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/gravity/Gravity.cpp b/Source/gravity/Gravity.cpp index cb7d1c1c95..b5e3876e71 100644 --- a/Source/gravity/Gravity.cpp +++ b/Source/gravity/Gravity.cpp @@ -470,7 +470,7 @@ Gravity::solve_for_phi (int level, Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(end,IOProc); - amrex::Print() << "Gravity::solve_for_phi() time = " << end << std::endl << std::endl; + amrex::Print() << "Gravity::solve_for_phi() time = " << end << " on level " << level << std::endl << std::endl; #ifdef BL_LAZY }); #endif diff --git a/Source/hydro/Castro_ctu_hydro.cpp b/Source/hydro/Castro_ctu_hydro.cpp index 2973c97bce..d3215ee6b7 100644 --- a/Source/hydro/Castro_ctu_hydro.cpp +++ b/Source/hydro/Castro_ctu_hydro.cpp @@ -33,7 +33,7 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // divergence) using the CTU framework for unsplit hydrodynamics if (verbose) { - amrex::Print() << "... Entering construct_ctu_hydro_source()" << std::endl << std::endl; + amrex::Print() << "... Entering construct_ctu_hydro_source() on level " << level << std::endl << std::endl; } #ifdef HYBRID_MOMENTUM @@ -1514,8 +1514,17 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) } #endif + // Perform reflux (for non-subcycling advances). + + if (parent->subcyclingMode() == "None") { + if (do_reflux == 1) { + FluxRegCrseInit(); + FluxRegFineAdd(); + } + } + if (verbose) { - amrex::Print() << "... Leaving construct_ctu_hydro_source()" << std::endl << std::endl; + amrex::Print() << "... Leaving construct_ctu_hydro_source() on level " << level << std::endl << std::endl; } if (verbose > 0) @@ -1528,7 +1537,7 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/mhd/Castro_mhd.cpp b/Source/mhd/Castro_mhd.cpp index bc9c939ad4..a77a1b054d 100644 --- a/Source/mhd/Castro_mhd.cpp +++ b/Source/mhd/Castro_mhd.cpp @@ -746,5 +746,14 @@ Castro::construct_ctu_mhd_source(Real time, Real dt) check_for_nan(S_new); + // Perform reflux (for non-subcycling advances). + + if (parent->subcyclingMode() == "None") { + if (do_reflux == 1) { + FluxRegCrseInit(); + FluxRegFineAdd(); + } + } + return status; } diff --git a/Source/reactions/Castro_react.cpp b/Source/reactions/Castro_react.cpp index 17003f0c11..8d77908a7f 100644 --- a/Source/reactions/Castro_react.cpp +++ b/Source/reactions/Castro_react.cpp @@ -172,7 +172,7 @@ Castro::react_state(MultiFab& s, MultiFab& r, Real time, Real dt, const int stra const int ng = s.nGrow(); if (verbose) { - amrex::Print() << "... Entering burner and doing half-timestep of burning." << std::endl << std::endl; + amrex::Print() << "... Entering burner on level " << level << " and doing half-timestep of burning." << std::endl << std::endl; } ReduceOps reduce_op; @@ -407,7 +407,7 @@ Castro::react_state(MultiFab& s, MultiFab& r, Real time, Real dt, const int stra } if (verbose) { - amrex::Print() << "... Leaving burner after completing half-timestep of burning." << std::endl << std::endl; + amrex::Print() << "... Leaving burner on level " << level << " after completing half-timestep of burning." << std::endl << std::endl; } if (verbose > 0) @@ -420,7 +420,7 @@ Castro::react_state(MultiFab& s, MultiFab& r, Real time, Real dt, const int stra #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::react_state() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -463,7 +463,7 @@ Castro::react_state(Real time, Real dt) const Real strt_time = ParallelDescriptor::second(); if (verbose) { - amrex::Print() << "... Entering burner and doing full timestep of burning." << std::endl << std::endl; + amrex::Print() << "... Entering burner on level " << level << " and doing full timestep of burning." << std::endl << std::endl; } MultiFab& S_old = get_old_data(State_Type); @@ -790,7 +790,7 @@ Castro::react_state(Real time, Real dt) if (verbose) { - amrex::Print() << "... Leaving burner after completing full timestep of burning." << std::endl << std::endl; + amrex::Print() << "... Leaving burner on level " << level << " after completing full timestep of burning." << std::endl << std::endl; const int IOProc = ParallelDescriptor::IOProcessorNumber(); Real run_time = ParallelDescriptor::second() - strt_time; @@ -800,7 +800,7 @@ Castro::react_state(Real time, Real dt) #endif ParallelDescriptor::ReduceRealMax(run_time, IOProc); - amrex::Print() << "Castro::react_state() time = " << run_time << std::endl << std::endl; + amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << std::endl << std::endl; #ifdef BL_LAZY }); #endif diff --git a/Source/rotation/Castro_rotation.cpp b/Source/rotation/Castro_rotation.cpp index e2e37f5b84..ce8b0ccb0b 100644 --- a/Source/rotation/Castro_rotation.cpp +++ b/Source/rotation/Castro_rotation.cpp @@ -42,7 +42,7 @@ Castro::construct_old_rotation_source(MultiFab& source, MultiFab& state_in, Real #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -96,7 +96,7 @@ Castro::construct_new_rotation_source(MultiFab& source, MultiFab& state_old, Mul #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_rotation_source() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_rotation_source() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/sources/Castro_sources.cpp b/Source/sources/Castro_sources.cpp index ab31fb5513..507a782929 100644 --- a/Source/sources/Castro_sources.cpp +++ b/Source/sources/Castro_sources.cpp @@ -167,7 +167,7 @@ Castro::do_old_sources( #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::do_old_sources() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::do_old_sources() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -250,7 +250,7 @@ Castro::do_new_sources( #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::do_new_sources() time = " << run_time << "\n" << "\n"; + amrex::Print() << "Castro::do_new_sources() time = " << run_time << " on level " << level << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -542,7 +542,7 @@ Castro::pre_advance_operators (Real time, Real dt) #endif #endif - // If we are using gravity, solve for the potential and gravatational field. + // If we are using gravity, solve for the potential and gravitational field. #ifdef GRAVITY construct_old_gravity(time);