Merge branch 'development' into initialize_keep_prev_state

AMReX-Astro · Jul 28, 2024 · 6b4e71d · 6b4e71d
2 parents dee7d5f + bb30ff4
commit 6b4e71d
Show file tree

Hide file tree

Showing 98 changed files with 3,748 additions and 687 deletions.
diff --git a/.github/workflows/good_defines.txt b/.github/workflows/good_defines.txt
@@ -1,3 +1,4 @@
+ALLOW_GPU_PRINTF
 AMREX_DEBUG
 AMREX_PARTICLES
 AMREX_SPACEDIM

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,27 @@
+# 24.07
+
+  * Reorganizing of the existing 2-shock and HLL Riemann solvers
+    (#2887, #2888, #2889, #2890)
+
+  * Some clean-up, accuracy improvements, and optimization of the
+    exact Riemann solver in preparation for coupling it to simulations
+    (#2868, #2869, #2875)
+
+  * a new problem setup, `subch_planar`, for exploring He detonations
+    (#2870, #2873, #2881, #2883, #2886, #2877, #2878)
+
+  * clean-up the computation of the shock variable for hybrid Riemann
+    solves (#2865)
+
+  * code clean-ups (#2852, #2871, #2891)
+
+  * documentation improvements (#2843, #2879)
+
+  * add an EditorConfig (#2872)
+
+  * add the circular detonation problem to test multidimensional shock
+    algorithms (#2858)
+
 # 24.06
 
   * Doc updates (#2839, #2842, #2846, #2851, #2854, #2860)

diff --git a/Diagnostics/DustCollapse/GNUmakefile b/Diagnostics/DustCollapse/GNUmakefile
@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 # programs to be compiled
 ALL: dustcollapse_$(DIM)d.ex
 

diff --git a/Diagnostics/Radiation/GNUmakefile b/Diagnostics/Radiation/GNUmakefile
@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 ALL: radhelp
 
 radhelp:

diff --git a/Diagnostics/Sedov/GNUmakefile b/Diagnostics/Sedov/GNUmakefile
@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 # programs to be compiled
 ALL: sedov_$(DIM)d.ex
 

diff --git a/Docs/source/mpi_plus_x.rst b/Docs/source/mpi_plus_x.rst
@@ -130,6 +130,37 @@ To enable this, compile with::
   USE_HIP = TRUE
 
 
+Printing Warnings from GPU Kernels
+==================================
+
+.. index:: USE_GPU_PRINTF
+
+Castro will output warnings if several assumptions are violated (often
+triggering a retry in the process).  On GPUs, printing from a kernel
+(using ``printf()``) can increase the number of registers a kernel needs,
+causing performance problems.  As a result, warnings are disabled by
+wrapping them in ``#ifndef AMREX_USE_GPU``.
+
+However, for debugging GPU runs, sometimes we want to see these
+warnings.  The build option ``USE_GPU_PRINTF=TRUE`` will enable these
+(by setting the preprocessor flag ``ALLOW_GPU_PRINTF``).
+
+.. note::
+
+   Not every warning has been enabled for GPUs.
+
+.. tip::
+
+   On AMD architectures, it seems necessary to use unbuffered I/O.  This
+   can be accomplished in the job submission script (for SLURM) by doing
+
+   ::
+
+      srun -u ./Castro...
+
+
+
+
 Working at Supercomputing Centers
 =================================
 

diff --git a/Exec/Make.Castro b/Exec/Make.Castro
@@ -55,6 +55,9 @@ endif
 # Require C++17
 CXXSTD := c++17
 
+# Use Lazy QueueReduction for the timing outputs
+LAZY := TRUE
+
 # default integrator
 INTEGRATOR_DIR ?= VODE
 
@@ -136,6 +139,10 @@ ifeq ($(USE_GPU),TRUE)
   endif
 endif
 
+ifeq ($(USE_GPU_PRINTF),TRUE)
+  DEFINES += -DALLOW_GPU_PRINTF
+endif
+
 CASTRO_AUTO_SOURCE_DIR := $(TmpBuildDir)/castro_sources/$(optionsSuffix).EXE
 
 
@@ -150,12 +157,6 @@ build_status:
 # The default is to include the sponge functionality
 DEFINES += -DSPONGE
 
-# OpenACC support
-ifeq ($(USE_ACC), TRUE)
-  DEFINES += -DACC
-endif
-
-
 
 #------------------------------------------------------------------------------
 # Castro directories

diff --git a/Exec/mhd_tests/LoopAdvection/problem_initialize.H b/Exec/mhd_tests/LoopAdvection/problem_initialize.H
@@ -16,8 +16,8 @@ void problem_initialize ()
       eos_state.rho = problem::rho_0;
       eos_state.p = problem::p_0;
       eos_state.T = 100000.0_rt;  // initial guess
-      for (int n = 0; n < NumSpec; n++) {
-          eos_state.xn[n] = 0.0_rt;
+      for (auto & X : eos_state.xn) {
+          X = 0.0_rt;
       }
       eos_state.xn[0] = 1.0_rt;
 

diff --git a/Exec/mhd_tests/LoopAdvection/problem_initialize_mhd_data.H b/Exec/mhd_tests/LoopAdvection/problem_initialize_mhd_data.H
@@ -7,6 +7,8 @@ Real A_z (int i, int j, int k,
           const GeometryData& geomdata)
 {
 
+    amrex::ignore_unused(k);
+
     // Compute A_z.  This lives on edges, e.g., {A_z}_{i-1/2,j-1/2,k}
     // so it is centered only in the z-direction.
 

diff --git a/Exec/mhd_tests/LoopAdvection/problem_initialize_state_data.H b/Exec/mhd_tests/LoopAdvection/problem_initialize_state_data.H
@@ -10,6 +10,8 @@ void problem_initialize_state_data (int i, int j, int k,
                                     const GeometryData& geomdata)
 {
 
+    amrex::ignore_unused(geomdata);
+
     state(i,j,k,URHO) = problem::rho_0;
     state(i,j,k,UMX) = problem::rho_0 * problem::u_x;
     state(i,j,k,UMY) = problem::rho_0 * problem::u_y;

diff --git a/Exec/science/Detonation/ci-benchmarks/sdc_det_plt00040_extrema.out b/Exec/science/Detonation/ci-benchmarks/sdc_det_plt00040_extrema.out
@@ -1,79 +1,79 @@
  plotfile = det_x_plt00040
  time = 5.1558159140336702e-06
               variables          minimum value          maximum value
- density                          185260165.55           216617920.63
- xmom                             -63255523503       2.9519346608e+16
+ density                          185257544.55           216468146.43
+ xmom                             -37913405153       2.9568274388e+16
  ymom                                        0                      0
  zmom                                        0                      0
- rho_E                        1.3062473821e+26       2.7891343298e+26
- rho_e                        1.3062473821e+26       2.7748842906e+26
- Temp                             50000000.026           7845854931.5
- rho_H1                       2.1204369735e-22         0.020000115169
- rho_He3                       0.0017224919811          0.02102205401
- rho_He4                          94366339.651           200001593.93
- rho_C12                        0.020000000216           21203829.102
- rho_N14                       1.999999523e-22           9.5660401467
- rho_O16                                  0.02           19209.488217
- rho_Ne20                                 0.02           3464.1596069
- rho_Mg24                                 0.02           23322.458406
- rho_Si28                                 0.02           2015700.9685
- rho_S32                                  0.02           1656602.4644
- rho_Ar36                                 0.02           822007.67426
- rho_Ca40                                 0.02           724910.24417
- rho_Ti44                                 0.02           34179.295535
- rho_Cr48                                 0.02           78164.189237
- rho_Fe52                                 0.02           278034.23746
- rho_Fe54                                 0.02           95026946.079
- rho_Ni56                                 0.02           2239485.5719
- rho_n                        2.1204369735e-22            234580.3659
- rho_p                          0.019999995438           3610755.3081
- rho_enuc                    -4.6808330292e+29        3.576498348e+32
- pressure                     5.5236728651e+25       1.1610548795e+26
- kineng                                      0       2.0467474279e+24
- soundspeed                       612864631.21            895226097.3
- Gamma_1                          1.3599756287           1.3819899024
- MachNumber                                  0          0.16110770916
- uplusc                           612864631.21           999815567.76
- uminusc                         -895226406.71          -612860598.79
- entropy                          98214767.758           336273469.46
+ rho_E                        1.3062473821e+26       2.7893330499e+26
+ rho_e                        1.3062473821e+26         2.77508064e+26
+ Temp                             50000000.026           7845891849.2
+ rho_H1                       2.1211722106e-22         0.020000096415
+ rho_He3                       0.0017224293869         0.021023770747
+ rho_He4                          94366787.107           200001406.23
+ rho_C12                        0.020000000216           21736627.589
+ rho_N14                      1.9999999481e-22         0.020000141427
+ rho_O16                                  0.02            21564.50879
+ rho_Ne20                                 0.02           4189.5544151
+ rho_Mg24                                 0.02           23323.799421
+ rho_Si28                                 0.02           2015729.7513
+ rho_S32                                  0.02           1656696.6915
+ rho_Ar36                                 0.02           822096.59228
+ rho_Ca40                                 0.02           724138.29448
+ rho_Ti44                                 0.02           34187.547129
+ rho_Cr48                                 0.02           78085.100557
+ rho_Fe52                                 0.02           278150.27827
+ rho_Fe54                                 0.02           94990080.243
+ rho_Ni56                                 0.02           2242328.7066
+ rho_n                        2.1211722106e-22           234640.58163
+ rho_p                          0.019999995438           3609403.7361
+ rho_enuc                    -4.7086157677e+29       3.5777644631e+32
+ pressure                     5.5236728651e+25       1.1610599873e+26
+ kineng                                      0       2.0519264828e+24
+ soundspeed                       612864631.21           895228345.47
+ Gamma_1                          1.3599756425           1.3822442674
+ MachNumber                                  0          0.16119509216
+ uplusc                           612864631.21           999919125.29
+ uminusc                         -895228523.09          -612861073.53
+ entropy                          98214767.758            336273484.2
  magvort                                     0                      0
- divu                            -97879.470548           34017.033805
- eint_E                       6.5312369103e+17        1.380434464e+18
- eint_e                       6.5312369103e+17        1.380434464e+18
- logden                           8.2677820478           8.3356943826
- StateErr_0                       185260165.55           216617920.63
- StateErr_1                       50000000.026           7845854931.5
+ divu                            -97996.218171           33763.099005
+ eint_E                       6.5312369103e+17       1.3804410664e+18
+ eint_e                       6.5312369103e+17       1.3804410664e+18
+ logden                           8.2677759035           8.3353939984
+ StateErr_0                       185257544.55           216468146.43
+ StateErr_1                       50000000.026           7845891849.2
  StateErr_2                              1e-30       9.9999779324e-11
  X(H1)                                   1e-30       9.9999779324e-11
- X(He3)                       8.9861142144e-12       9.9999601254e-11
- X(He4)                          0.48402941802           0.9999999982
- X(C12)                       1.0000000108e-10          0.09999745037
- X(N14)                                  1e-30       4.5113532098e-08
- X(O16)                                  1e-10       9.6047388579e-05
- X(Ne20)                                 1e-10       1.6337008127e-05
- X(Mg24)                                 1e-10       0.00011119757992
- X(Si28)                                 1e-10         0.010055548658
- X(S32)                                  1e-10        0.0083462743796
- X(Ar36)                                 1e-10        0.0041650262219
- X(Ca40)                                 1e-10        0.0037107481152
- X(Ti44)                                 1e-10       0.00017303069502
- X(Cr48)                                 1e-10       0.00040092438857
- X(Fe52)                                 1e-10        0.0013387480381
- X(Fe54)                                 1e-10          0.46489869655
- X(Ni56)                                 1e-10         0.010338413209
- X(n)                                    1e-30        0.0011729011118
- X(p)                          9.999997703e-11         0.017599035758
- abar                              4.000000001           6.7313315116
- Ye                              0.49998670822          0.50001557428
- x_velocity                      -316.27744461           138671594.27
+ X(He3)                        8.985892423e-12       9.9999601254e-11
+ X(He4)                          0.48406961191           0.9999999982
+ X(C12)                       1.0000000108e-10          0.10247460098
+ X(N14)                                  1e-30       1.0000000384e-10
+ X(O16)                                  1e-10       0.00010166316852
+ X(Ne20)                                 1e-10       1.9751128146e-05
+ X(Mg24)                                 1e-10       0.00011117683475
+ X(Si28)                                 1e-10         0.010049207028
+ X(S32)                                  1e-10        0.0083399755634
+ X(Ar36)                                 1e-10        0.0041613939849
+ X(Ca40)                                 1e-10         0.003708082998
+ X(Ti44)                                 1e-10       0.00017289116777
+ X(Cr48)                                 1e-10       0.00040055008209
+ X(Fe52)                                 1e-10        0.0013373223366
+ X(Fe54)                                 1e-10          0.46488586209
+ X(Ni56)                                 1e-10         0.010358700546
+ X(n)                                    1e-30        0.0011732028047
+ X(p)                         9.9999977002e-11         0.017592025438
+ abar                              4.000000001           6.7312151705
+ Ye                              0.49998627651          0.50001570244
+ x_velocity                      -189.56702545           138792440.56
  y_velocity                                  0                      0
  z_velocity                                  0                      0
- t_sound_t_enuc               3.4411968038e-13          0.97442435993
- enuc                        -2.4887686639e+21       1.6510630042e+24
- magvel                                      0           138671594.27
- radvel                          -316.27744461           138671594.27
- circvel                                     0           2.4494897428
- magmom                                      0       2.9519346608e+16
+ t_sound_t_enuc               3.4412406685e-13          0.97569821534
+ enuc                        -2.5034688832e+21       1.6527902706e+24
+ magvel                                      0           138792440.56
+ radvel                          -189.56702545           138792440.56
+ circvel                                     0                      2
+ magmom                                      0       2.9568274388e+16
  angular_momentum_x                          0                      0
  angular_momentum_y                          0                      0
  angular_momentum_z                          0                      0

diff --git a/Exec/science/Detonation/problem_initialize_state_data.H b/Exec/science/Detonation/problem_initialize_state_data.H
@@ -22,7 +22,15 @@ void problem_initialize_state_data (int i, int j, int k,
 
     state(i,j,k,URHO) = problem::dens;
 
-    Real sigma = 1.0_rt / (1.0_rt + std::exp(-(c_T - xcen)/ width));
+    Real sigma_arg = -(c_T - xcen) / width;
+    Real sigma;
+    // need to avoid FP overflow for sigma_arg >= 709
+    // 1/(1 + exp(100)) ~= 3e-44, which is much smaller than machine epsilon
+    if (sigma_arg < 100) {
+        sigma = 1.0_rt / (1.0_rt + std::exp(sigma_arg));
+    } else {
+        sigma = 0.0_rt;
+    }
 
     state(i,j,k,UTEMP) = problem::T_l + (problem::T_r - problem::T_l) * (1.0_rt - sigma);
 

diff --git a/Exec/science/Detonation/shock_paper/README.md b/Exec/science/Detonation/shock_paper/README.md
@@ -0,0 +1,47 @@
+# Shock Burning Experiments
+
+This directory is meant to explore shock burning with detonation.  Compile as:
+
+```
+make USE_SIMPLIFIED_SDC=TRUE USE_SHOCK_VAR=TRUE NETWORK_DIR=aprox13 -j 4
+```
+
+Then the script `setup_runs.py` will setup a suite of simulations with
+the following resolutions into separate directories (using the
+`inputs-shock-burn.template`):
+
+
+| resolution   |  base grid  |  levels (4x jumps)  |
+| ------------ | ----------- | ------------------- |
+|       24 km  |       48    |         1           |
+|       12 km  |       96    |         1           |
+|        6 km  |      192    |         1           |
+|        3 km  |      384    |         1           |
+|      1.5 km  |      768    |         1           |
+|   0.1875 km  |     6144    |         1           |
+|  2343.74 cm  |    12288    |         2           |
+
+you can set the value of the shock detection threshold there
+and the directory names will reflect that setting.
+
+## plotting
+
+The following scripts can make useful plots (some use the
+`detonation.py` module as support):
+
+* `det_speed_comp.py` : plot detonation speed vs. resolution, using
+  simple differencing to estimate the detonation speed from the last 3
+  plotfiles.
+
+* `profile_compare.py` : given a list of pruns (from different
+  resolutions), make a plot showing all of their profiles together.
+
+* `profiles.py` : for a single run, plot profiles of T and enuc for
+  several times.
+
+* `show_shock_flag.py` : simply plot the shock variable on top of T
+  and enuc profiles.
+
+* `zoom_summary.py` : given a list of runs (from different
+  resolutions), plot the last plotfile from each run, zoomed in on
+  where the peak energy generation is.