Merge branch 'main' into GUI

lanl · Apr 8, 2024 · 14f8a9c · 14f8a9c
2 parents dc0e440 + 5748d03
commit 14f8a9c
Show file tree

Hide file tree

Showing 31 changed files with 21,943 additions and 21,531 deletions.
diff --git a/.conda/evpfft/cpu/build.sh b/.conda/evpfft/cpu/build.sh
@@ -13,4 +13,4 @@ cmake ../src/ \
     -D CMAKE_CXX_COMPILER=$BUILD_PREFIX/bin/mpicxx \
     -D CMAKE_C_COMPILER=$BUILD_PREFIX/bin/mpicc \
 
-make install
+make install
diff --git a/.conda/evpfft/cpu/meta.yaml b/.conda/evpfft/cpu/meta.yaml
@@ -18,6 +18,7 @@ build:
 requirements:
   build:
     - cmake >=3.17.0
+    - make
     - {{ compiler('c') }}={{ linux_compiler_version }} # [linux]
     - {{ compiler('c') }}={{ macos_compiler_version }} # [osx]
     - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux]

diff --git a/.conda/fierro-dev/cpu/meta.yaml b/.conda/fierro-dev/cpu/meta.yaml
@@ -1,5 +1,6 @@
 {% set version = "1.0.0" %}
-{% set compiler_version = "10.4.0" %}
+{% set linux_compiler_version = "10.4.0" %}
+{% set macos_compiler_version = "12" %}
 
 package:
   name: fierro-dev
@@ -13,9 +14,23 @@ build:
     - PLATFORM={{ target_platform }}
 
 requirements:
-  run:
+  host:
     - cmake >=3.17
-    - {{ compiler('cxx') }}={{ compiler_version }}
+    - {{ compiler('c') }}={{ linux_compiler_version }} # [linux]
+    - {{ compiler('c') }}={{ macos_compiler_version }} # [osx]
+    - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux]
+    - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx]
+    - {{ compiler('fortran') }}
+    - fierro-trilinos-cpu
+    - mpi
+    - elements
+  run:
+    #- cmake >=3.17
+    - {{ compiler('c') }}={{ linux_compiler_version }} # [linux]
+    - {{ compiler('c') }}={{ macos_compiler_version }} # [osx]
+    - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux]
+    - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx]
+    - {{ compiler('fortran') }}
     - fierro-trilinos-cpu
     - mpi
     - elements

diff --git a/.conda/fierro/cpu/meta.yaml b/.conda/fierro/cpu/meta.yaml
@@ -25,6 +25,7 @@ requirements:
     - {{ compiler('c') }}={{ macos_compiler_version }} # [osx]
     - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux]
     - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx]
+    - {{ compiler('fortran') }}
     - openmpi={{ mpi_version }}
   host:
     - _openmp_mutex # [linux]

diff --git a/.conda/heffte/cpu/meta.yaml b/.conda/heffte/cpu/meta.yaml
@@ -23,10 +23,10 @@ requirements:
     - {{ compiler('cxx') }} # [osx]
     - openmpi
   host:
-    - openmpi
+    - openmpi >=4.1.6,<5.0a0
     - fftw
   run:
-    - openmpi
+    - openmpi >=4.1.6,<5.0a0
     - fftw
 
 about:

diff --git a/.conda/trilinos/cpu/meta.yaml b/.conda/trilinos/cpu/meta.yaml
@@ -27,7 +27,7 @@ requirements:
   host:
     - libblas
     - liblapack
-    - openmpi
+    - openmpi >=4.1.6,<5.0a0
     - boost-cpp
     - libnetcdf
     - libmatio

diff --git a/.etc/conda_build_environment.yaml b/.etc/conda_build_environment.yaml
@@ -5,4 +5,5 @@ channels:
   - defaults
 dependencies:
   - boa # For building
+  - conda-build
   - anaconda-client # For uploading
diff --git a/.github/workflows/build-conda-package.yaml b/.github/workflows/build-conda-package.yaml
@@ -57,11 +57,11 @@ jobs:
 
       - name: Update environment
         run:
-          mamba env update -n build-env -f .etc/conda_build_environment.yaml
+          conda env update -n build-env -f .etc/conda_build_environment.yaml
         #if: steps.cache.outputs.cache-hit != 'true'
       - run: |
           if [ ${{ inputs.debug }} = true ] ; then export DEBUG="--debug"; fi;
-          mamba mambabuild $DEBUG ${{ inputs.recipe_dir }} -e .conda/build_variants.yaml -m ${{ inputs.additional_variant }} --token ${{ secrets.ANACONDA_TOKEN }} --no-test -c fierromechanics -c conda-forge -c nvidia
+          conda build $DEBUG ${{ inputs.recipe_dir }} -e .conda/build_variants.yaml -m ${{ inputs.additional_variant }} --token ${{ secrets.ANACONDA_TOKEN }} --no-test -c fierromechanics -c conda-forge -c nvidia
         continue-on-error: ${{ inputs.debug }}
       
       - name: Setup tmate session

diff --git a/.github/workflows/build-evpfft.yaml b/.github/workflows/build-evpfft.yaml
@@ -0,0 +1,37 @@
+name: Ubuntu Build EVPFFT
+
+# Controls when the workflow will run. Currently manually ran
+on: 
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - name: update the package list
+      run:
+        sudo apt-get update
+      shell: bash
+    - name: Install build tools
+      run:
+        sudo apt-get install build-essential
+      shell: bash
+    - name: Install MPI
+      run: |
+        sudo apt-get install libopenmpi-dev openmpi-bin
+        echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV
+    - name: Install cmake
+      run:
+        sudo apt-get -y install cmake
+      shell: bash
+    - name: Run build script
+      run: |
+         chmod +x ./scripts/build-scripts/build_evpfft.sh # Make sure permission is correct
+         ./scripts/build-scripts/build_evpfft.sh --heffte_build_type=fftw --build_fftw --build_hdf5 --kokkos_build_type=openmp  --machine=linux --num_jobs=4
+      working-directory: ./src/EVPFFT
+      shell: bash
diff --git a/BUILD.md b/BUILD.md
@@ -23,27 +23,8 @@ openmpi
 ```
 
 ### Environment Variables
-We need to have several environment variables point to these homebrew installations so that our build system knows where to look. These variables can either be set individually, or put into your ***~/.bashrc*** file and sourced. 
-The current setup for the build scripts is to add the below lines to your environment automatically. For user that are using a different software stack such as GCC, these path's would need to point to those paths in the script file they are pulled from. This file is
-```
-scripts/<solver>/machines/mac-env.sh
-```
-If you are changing the environment paths in your ***~/.bashrc*** then the file could look like this
-```
-export CC=/opt/homebrew/opt/llvm/bin/clang
-export CXX=/opt/homebrew/opt/llvm/bin/clang++
-export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
-export LDFLAGS="-L/opt/homebrew/opt/llvm/lib"
-export CPPFLAGS="-I/opt/homebrew/opt/llvm/include"
-export OMPI_CC=${CC}
-export OMPI_CXX=${CXX}
-```
-and then you would source those variables with 
-```
-source ~/.bashrc
-```
-For those that noticed, this looks exactly like the ***mac-env.sh*** file mentioned above.
-There are two things of note with this example. First, it is using the llvm/clang compiler. This is one of the possible options. If you prefer to use GCC as your base compiler, all of the variables (CC,CXX,PATH,LDFLAGS,CPPFLAGS) would need to be modified to point to the corresonding spot in the gcc directory. The second thing to note with this example is the path that homebrew has installed your software. By default on Apple silicon hardware (M1/M2), it should be the one listed as in the above example
+The paths are set in the **scripts/machine/mac.sh** script.
+There are two things of note. First, it is using the llvm/clang compiler. This is one of the possible options. If you prefer to use GCC as your base compiler, all of the variables (CC,CXX,PATH,LDFLAGS,CPPFLAGS) would need to be modified to point to the corresonding spot in the gcc directory. The second thing to note with this example is the path that homebrew has installed your software. By default on Apple silicon hardware (M1/M2), it should be the one listed as in the above example
 ```
 /opt/homebrew/opt/<software>
 ```
@@ -55,10 +36,11 @@ If you explicitally tell homebrew to install them elsewhere, or it does it in a
 
 ### Running the build scripts
 Now the assumption is that you have the software installed and the system knows where to look for that software. The code can be built using the scripts in the ***scripts*** directory. There are several options here depending on the problems you'd like to run, but the build process is identical, so we will use the *Explicit* folder for our example. The script can be run from anywhere in the Fierro repository. If we were at the top of the directory would run
+Now the assumption is that you have the software installed and the system knows where to look for that software. The code can be built as normal now, with the exception that we always need to pass in the machine argument and set it to mac (also of note, unlike linux systems, we do not ***source*** script, we instead have to just run the script with ./)
 ```
-./scripts/Explicit/build-it.sh --machine=mac --kokkos_build_type=openmp
+./{path-to-repo}/scripts/build-fierro.sh --machine=mac [any other parameters we need to pass]
 ```
-***Note*** you can see more information on the script arguments in the *README.md* or running the script with the argument ***--help***. However this is the standard script you would run on a Mac. You could subistitue 'serial' in for 'openmp', but since the build supports OpenMP that is the suggestion option for better runtime performance.
+***Note*** you can still see more information on the script arguments by running the script with the argument ***--help***. However this is the standard script you would run on a Mac.
 
 ## Running the example
 Assuming a smooth build, you will be placed at the top of the repository again. You can move yourself into the build directory
@@ -69,20 +51,4 @@ From there you can run this example with
 ```
 ./bin/fierro-parallel-explicit ../src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml
 ```
-***Note*** For this example the mesh file that is passed into *example_simple.yaml* needs to be present in your build directory. 
-
-## Important Notes
-Hopefully the above steps result in a smooth build process. When making changes to code that require another cmake configuration, we suggest running the full ***build-it.sh <args>*** script again. This is because it sources the necessaryfiles to setup environment paths. Trilinos will *not* get re-built as a result of this.
-The reason for this suggestion is ease of environment. It is possible to simply source the cmake script
-```
-source cmake_build.sh
-```
-However you may end up in an unexpected directory since all internal paths were not set. It is possible to
-```
-source setup-env.sh --machine=mac --kokkos_build_type=openmp
-```
-before the cmake step. This would set all path's globally, instead of self-contained inside the script run. However any software changes made after this needs to have environment-correcting commands.
-Also, it is not advised to source the ***build-it.sh*** script from anywhere as the initial step, because of how Macs handle that source. In bash shells the behavior would be more or less expected. However on others (for instance Mac's default zsh shell) the paths would not be expected.
-Therefore, we suggest one of the two options
-- run the script with ***./<path to script>/build-it.sh*** from anywhere or
-- ***source build-it.sh*** from *inside* the scripts/[solver] directory
+***Note*** For this example the mesh file that is passed into *example_simple.yaml* needs to be present in your build directory. For the examples, you can copy the one in {path-to-repo}/single-node/src/Explicit-Lagrange/meshes/mesh_Sedov_32.geo 
diff --git a/README.md b/README.md
@@ -59,46 +59,17 @@ You are welcome to only compile one solver or the other, and the one(s) that you
 As for Trilinos, we recommend installing the Anaconda package for the desired build into a new Anaconda environment to satisfy **Fierro**'s dependency rather than building it from source. If you do wish to build it from source, however, sample build scripts for Trilinos can be found in `Fierro/Trilinos-Build-Scripts`. Build scripts for all Anaconda packages can be found in `Fierro/Anaconda-Packages/`.
 
 ## Alternative Build Workflows
-In addition to the primary build workflow described above, there are build scripts for a variety of alternative workflows. These scripts can be found under `Fierro/scripts`.
+In addition to the primary build workflow described above, there are build scripts for a variety of alternative workflows. These scripts can be found under `Fierro/scripts`. Although there are multiple scripts shown in this directory, the only one that will be run directly by the user is ***build-fierro.sh***
 ### Building the explicit and implicit Lagrangian methods with Trilinos+Kokkos
 Explicit Lagrangian codes are being added to the repository that are written using MATAR+Kokkos and run with fine-grained parallellism on multi-core CPUs and GPUs.  Build scripts are provided for each Lagrangian code, and those scripts follow those used in the [MATAR](https://github.com/lanl/MATAR/) GitHub repository. The scripts to build the Lagrangian codes (that use MATAR+Kokkos) are in the scripts folder.  The user must update the modules loaded by the build scripts (for the compiler etc.), and then type
 Immediate help with all scripts can be had running
 ```
-<script> --help
-```
-The build-it script can take up to 3 arguments (with a minimum of 2)
-```
-source build-it.sh --machine=<arg> --kokkos_build_type=<arg> (optional)--build_cores=<arg>
-```
-machine currently has three options: darwin,linux,mc
-```
-    darwin: builds by loading modules on the darwin cluster. and can perform parallel builds (make -j). Changes can be made to scripts/[solver]/machines/darwin-setup.sh to reflect your machine.
-    linux: does not load any modules and instead sources some paths related to your loaded software. Additionally, the builds can be parallel (make -j <build_cores>)
-    macos: does not load any modules and instead sources some paths related to your loaded software. Additionally, the builds will be forced to beserial (make)
-For Mac builds, please see the ***BUILD.md*** file for more info
-```
-kokkos_build_type  has four options: 'cuda', 'hip', 'openmp', 'serial'
-***Note*** - all builds use Trilinos with Kokkos. The 'serial' option will utilize the Kokkos serial build
-```
-    cuda: loads cuda module and a working gcc module pairing
-    hip: loads hip module and a working clang module pairing
-    openmp: loads gcc module and sets openmp environment variables
-    serial: loads gcc module
-```
-***Note*** - compiler can be changed with the appropriate variables in *setup-env.sh*, the ones provided are simply known to work together
-If running on Mac, only the openmp and serial options are enabled
-
-All other scripts will be called with the appropriate arguments as a result of running build-it.
-
-If you need to simply rebuild Fierro without making a new Trilinos installation, simply
-```
-source cmake_build.sh
-```
-If you are getting back on to a machine or allocation to continue development, you will need to run
-```
-source setup-env.sh <same args you passed to build-it>
+source {path-to-repo}/build-fierro.sh --help
 ```
+The default run will build the **full-app** settting up the **Explicit*** solver with **serial** Kokkos for a **linux** computer (non-HPC machine)
+Running with ```--help``` option will list all parameters and their accepted arguments
 If the scripts fail to build a Lagrangian code, then carefully review the modules used and the computer architecture settings.  
+For building on a Mac, please see the extra info in the BUILD.md
 #A more lenghtly discussion of the build scripts is provided in the MATAR GitHub repository. 
 
 For help with [Trilinos](https://github.com/trilinos/Trilinos/wiki/New-Trilinos-Developers)

diff --git a/integrated-tests/Test-scripts/MueLu_Thermal_3D_Params.xml b/integrated-tests/Test-scripts/MueLu_Thermal_3D_Params.xml
@@ -0,0 +1,72 @@
+<ParameterList name="MueLu">
+
+  <!--
+    For 3D elasticity, these are the recommended settings for MueLu.
+  -->
+
+  <!-- ===========  GENERAL ================ -->
+    <Parameter        name="problem: type"                        type="string"   value="Poisson-3D"/>
+
+    <Parameter        name="verbosity"                            type="string"   value="none"/>
+
+    <Parameter        name="coarse: max size"                     type="int"      value="2000"/>
+
+    <Parameter        name="multigrid algorithm"                  type="string"   value="sa"/>
+
+    <Parameter        name="coarse: type"                         type="string"   value="Klu2"/>
+
+    <!-- reduces setup cost for symmetric problems -->
+    <Parameter        name="transpose: use implicit"              type="bool"     value="true"/>
+
+    <!-- start of default values for general options (can be omitted) -->
+    <Parameter        name="max levels"                	        type="int"      value="10"/>
+    <Parameter        name="number of equations"                  type="int"      value="1"/>
+    <Parameter        name="sa: use filtered matrix"              type="bool"     value="true"/>
+    <!-- end of default values -->
+
+  <!-- ===========  AGGREGATION  =========== -->
+    <Parameter        name="aggregation: type"                    type="string"   value="uncoupled"/>
+    <Parameter        name="aggregation: drop scheme"             type="string"   value="classical"/>
+
+   <!-- ===========  REUSE  =========== -->
+    <Parameter        name="reuse: type"                          type="string"   value="S"/>
+    <!-- Uncomment the next line to enable dropping of weak connections, which can help AMG convergence
+         for anisotropic problems.  The exact value is problem dependent. -->
+    <!-- <Parameter        name="aggregation: drop tol"                type="double"   value="0.02"/> -->
+
+  <!-- ===========  SMOOTHING  ===========-->
+    <Parameter        name="smoother: type"                       type="string"   value="CHEBYSHEV"/>
+    <ParameterList    name="smoother: params">
+      <Parameter      name="chebyshev: degree"                    type="int"      value="2"/>>
+      <Parameter      name="chebyshev: ratio eigenvalue"          type="double"   value="7"/>
+      <Parameter      name="chebyshev: min eigenvalue"            type="double"   value="1"/>
+      <Parameter      name="chebyshev: zero starting solution"    type="bool"     value="true"/>
+      <Parameter      name="chebyshev: eigenvalue max iterations" type="int"      value="100"/>
+    </ParameterList>
+<!-- ===========  SMOOTHING  ===========
+    <Parameter        name="smoother: type"                       type="string"   value="RELAXATION"/>
+  <ParameterList    name="smoother: params">
+    <Parameter      name="relaxation: type"                     type="string"   value="Jacobi"/>
+    <Parameter      name="relaxation: sweeps"                   type="int"      value="1"/>
+    <Parameter      name="relaxation: damping factor"           type="double"   value="0.3"/>
+  </ParameterList>
+    -->
+
+  <!-- ===========  REPARTITIONING  =========== -->
+    <Parameter        name="repartition: enable"                  type="bool"     value="true"/>
+    <Parameter        name="repartition: partitioner"             type="string"   value="zoltan2"/>
+    <Parameter        name="repartition: start level"             type="int"      value="2"/>
+    <Parameter        name="repartition: min rows per proc"       type="int"      value="1000"/>
+    <Parameter        name="repartition: max imbalance"           type="double"   value="1.10"/>
+    <!-- start of default values for repartitioning (can be omitted) -->
+    <Parameter name="repartition: remap parts"                type="bool"     value="true"/>
+    <Parameter name="repartition: rebalance P and R"          type="bool"     value="true"/>
+    <ParameterList name="repartition: params">
+       <Parameter name="algorithm" type="string" value="multijagged"/>
+       <Parameter name="mj_premigration_option" type="int" value="1"/>
+    </ParameterList> 
+    <!-- end of default values -->
+
+      <!-- for runs on with OpenMP or Cuda backend, enable -->
+  <Parameter        name="use kokkos refactor"                  type="bool"     value="false"/>
+</ParameterList>
diff --git a/...scripts/Solver-Inputs/SGH_Noh_simple.yaml → ...-Inputs/fierro-parallel-explicit/Noh.yaml b/...scripts/Solver-Inputs/SGH_Noh_simple.yaml → ...-Inputs/fierro-parallel-explicit/Noh.yaml
@@ -49,10 +49,10 @@ materials:
     strength_model: none
     elastic_modulus: 10
     poisson_ratio: 0.3
-    q1: 0.06
-    q2: 1.2
-    q1ex: 0.06
-    q2ex: 1.2
+    q1: 1.0
+    q2: 1.333
+    q1ex: 1.0
+    q2ex: 0.0
     eos_global_vars:
       - 1.666666666666667
       - 1.0E-14

diff --git a/...ripts/Solver-Inputs/SGH_Sedov_simple.yaml → ...nputs/fierro-parallel-explicit/Sedov.yaml b/...ripts/Solver-Inputs/SGH_Sedov_simple.yaml → ...nputs/fierro-parallel-explicit/Sedov.yaml
@@ -45,13 +45,10 @@ fea_module_parameters:
 materials:
   - id: 0
     eos_model: ideal_gas
-    strength_model: none
-    elastic_modulus: 10
-    poisson_ratio: 0.3
-    q1: 0.06
-    q2: 1.0
-    q1ex: 0.06
-    q2ex: 1.0
+    q1: 1.0
+    q2: 1.33333
+    q1ex: 1.0
+    q2ex: 0.0
     eos_global_vars:
       - 1.666666666666667
       - 1.0E-14
@@ -75,7 +72,7 @@ regions:
       radius2: 0.1
     material_id: 0
     den: 1.0
-    ie:  0.25833839995946534
+    ie:  0.493390
     velocity: cartesian
     u: 0.0
     v: 0.0

diff --git a/...scripts/Solver-Inputs/SGH_Sod_simple.yaml → ...-Inputs/fierro-parallel-explicit/Sod.yaml b/...scripts/Solver-Inputs/SGH_Sod_simple.yaml → ...-Inputs/fierro-parallel-explicit/Sod.yaml
@@ -64,8 +64,8 @@ materials:
   - id: 0
     eos_model: ideal_gas
     # Artificial viscosity linear and quadratic terms
-    q1: 0.06
-    q2: 1.2
+    q1: 1.0
+    q2: 1.3333
     q1ex: 0.06
     q2ex: 1.2
     eos_global_vars: