diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 73d6eaf0903..978e9c28f04 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -31,7 +31,10 @@ jobs: set +e keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' for k in $keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index f224ace0ef3..cafae38bf1c 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -27,7 +27,7 @@ jobs: EVENT=${{ github.event.workflow_run.event }} # Triggering workflow run name (e.g., LinuxClang) - WORKFLOW_NAME=${{ github.event.workflow_run.name }} + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" if [[ $EVENT == "pull_request" ]]; then gh run download ${{ github.event.workflow_run.id }} -n pr_number @@ -45,16 +45,19 @@ jobs: # The goal is to keep the last used key of each job and delete all others. # something like ccache-LinuxClang- - keyprefix=ccache-${WORKFLOW_NAME}- + keyprefix="ccache-${WORKFLOW_NAME}-" - cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq) + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' for j in $cached_jobs do - old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2) + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do - gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm done done + unset IFS diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0e03e384633..f240930bd8b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,6 +14,7 @@ concurrency: jobs: analyze: + if: ${{ github.repository == 'AMReX-Codes/amrex' || github.event_name != 'schedule' }} name: Analyze runs-on: ubuntu-latest permissions: diff --git a/CHANGES b/CHANGES index f9f2d4febe8..82b28a03e87 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,44 @@ +# 23.12 + + -- solve_cg: avoid use of MF `z` (#3637) + + -- Fix: nosmt OMP Threads Default (#3647) + `amrex.omp_threads`: Can Avoid SMT (#3607) + + -- When checking for periodic outs on GPU, copy full particle data (#3646) + + -- MLEBABecLap: Support Robin BC at Domain Boundaries (#3617) + + -- Ascent: SoA Particle Support (#3350) + + -- solve_bicgstab: use fewer MFs (#3635) + + -- solve_bicgstab: cut use of `s` (#3629) + + -- Bug fix for amrex::Subtract when called with interger nghost (#3634) + + -- Fix typo in `MLMGT::getGradSolution` when `MF` is different from `AMF` (#3631) + + -- SUNDIALS: Use sunrealtype instead of realtype (#3632) + + -- SYCL: Use get_multi_ptr instead of get_pointer (#3630) + + -- Plotfile Tools: GPU support (#3626) + + -- solve_cg: use linop.make instead of MF constructor (#3627) + + -- CArena: shrink_in_place and operator<< (#3621) + + -- solve_bicgstab: use linop.make instead of MF constructor (#3619) + + -- replace AMREX_DEVICE_COMPILE with AMREX_IF_ON_DEVICE and AMREX_IF_ON_HOST (#3591) + + -- [Breaking] Prefix `amrex_` to each plotfile Tool (#3600) + + -- FillRandom: Use MKL host API (#3536) + + -- use hipPointerAttribute_t.type as HIP is removing hipPointerAttribute_t.memoryType (#3610) + # 23.11 -- Give FlashFluxRegisters ways to accumulate data in registers (#3597) diff --git a/Docs/sphinx_documentation/source/LinearSolvers.rst b/Docs/sphinx_documentation/source/LinearSolvers.rst index 153d259f04e..b8c38fe6c1c 100644 --- a/Docs/sphinx_documentation/source/LinearSolvers.rst +++ b/Docs/sphinx_documentation/source/LinearSolvers.rst @@ -483,7 +483,9 @@ To set homogeneous Dirichlet boundary conditions, call ml_ebabeclap->setEBHomogDirichlet(lev, coeff); where coeff can be a real number (i.e. the value is the same at every cell) -or is the MultiFab holding the coefficient of the gradient at each cell with an EB face. +or a MultiFab holding the coefficient of the gradient at each cell with an EB face. +In other words, coeff is :math:`\beta` in the canonical form given in equation :eq:`eqn::abeclap` +located at the EB surface centroid. To set inhomogeneous Dirichlet boundary conditions, call @@ -494,8 +496,9 @@ To set inhomogeneous Dirichlet boundary conditions, call ml_ebabeclap->setEBDirichlet(lev, phi_on_eb, coeff); where phi_on_eb is the MultiFab holding the Dirichlet values in every cut cell, -and coeff again is a real number (i.e. the value is the same at every cell) -or a MultiFab holding the coefficient of the gradient at each cell with an EB face. +and coeff again is a real number +or a MultiFab holding the coefficient of the gradient at each cell with an EB face, +i.e. :math:`\beta` in equation :eq:`eqn::abeclap` located at the EB surface centroid. Currently there are options to define the face-based coefficients on face centers vs face centroids, and to interpret the solution variable diff --git a/Docs/sphinx_documentation/source/conf.py b/Docs/sphinx_documentation/source/conf.py index dc29ab6e041..8cb17c78e1f 100644 --- a/Docs/sphinx_documentation/source/conf.py +++ b/Docs/sphinx_documentation/source/conf.py @@ -42,7 +42,7 @@ def get_amrex_version(): intersphinx_mapping = { 'amrex_tutorials': ('https://amrex-codes.github.io/amrex/tutorials_html/', None), - 'amrex_hydro':('https://amrex-codes.github.io/amrex/hydro_html/', None) + 'amrex_hydro':('https://amrex-fluids.github.io/amrex-hydro/docs_html/', None) } # Add any paths that contain templates here, relative to this directory. diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index efccf318315..c0e191d79e4 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -1082,7 +1082,7 @@ AmrMesh::checkInput () for (int i = 0; i < max_level; i++) { if (MaxRefRatio(i) < 2) { - amrex::Error("Amr::checkInput: bad ref_ratios"); + amrex::Warning("Amr::checkInput: ref_ratios all equal to one!"); } } diff --git a/Src/Base/AMReX_BaseFab.H b/Src/Base/AMReX_BaseFab.H index 99132038397..006d7639ad6 100644 --- a/Src/Base/AMReX_BaseFab.H +++ b/Src/Base/AMReX_BaseFab.H @@ -90,54 +90,6 @@ makeArray4 (T* p, Box const& bx, int ncomp) noexcept return Array4{p, amrex::begin(bx), amrex::end(bx), ncomp}; } -/** -* \brief A Fortran Array-like Object -* BaseFab emulates the Fortran array concept. -* Useful operations can be performed upon -* BaseFabs in C++, and they provide a convenient interface to -* Fortran when it is necessary to retreat into that language. - -* BaseFab is a template class. Through use of the -* template, a BaseFab may be based upon any class. So far at least, -* most applications have been based upon simple types like integers, -* real*4s, or real*8s. Most applications do not use BaseFabs -* directly, but utilize specialized classes derived from BaseFab. - -* Classes derived from BaseFab include FArrayBox, IArrayBox, TagBox, -* Mask, EBFArrayBox, EBCellFlag and CutFab. - -* BaseFab objects depend on the dimensionality of space -* (indirectly through the DOMAIN Box member). It is -* typical to define the macro SPACEDIM to be 1, 2, or 3 to indicate -* the dimension of space. See the discussion of class Box for more -* information. A BaseFab contains a Box DOMAIN, which indicates the -* integer indexing space over which the array is defined. A BaseFab -* also has NVAR components. By components, we mean that for each -* point in the rectangular indexing space, there are NVAR values -* associated with that point. A Fortran array corresponding to a -* BaseFab would have (SPACEDIM+1) dimensions. - -* By design, the array layout in a BaseFab mirrors that of a -* Fortran array. The first index (x direction for example) varies -* most rapidly, the next index (y direction), if any, varies next -* fastest. The component index varies last, after all the spatial -* indices. - -* It is sometimes convenient to be able to treat a sub-array within an -* existing BaseFab as a BaseFab in its own right. This is often -* referred to as aliasing the BaseFab. Note that when aliasing is -* used, the BaseFabs domain will not, in general, be the same as the -* parent BaseFabs domain, nor will the number of components. -* BaseFab is a dimension dependent class, so SPACEDIM must be -* defined as either 1, 2, or 3 when compiling. - -* This is NOT a polymorphic class. - -* It does NOT provide a copy constructor or assignment operator. - -* T MUST have a default constructor and an assignment operator. -*/ - template typename std::enable_if::value>::type placementNew (T* const /*ptr*/, Long /*n*/) @@ -178,6 +130,54 @@ placementDelete (T* const ptr, Long n) }); } +/** + * \brief A FortranArrayBox(FAB)-like object + * + * BaseFab emulates the Fortran array concept. + * Useful operations can be performed upon + * BaseFabs in C++, and they provide a convenient interface to + * Fortran when it is necessary to retreat into that language. + * + * BaseFab is a template class. Through use of the + * template, a BaseFab may be based upon any class. So far at least, + * most applications have been based upon simple types like integers, + * real*4s, or real*8s. Most applications do not use BaseFabs + * directly, but utilize specialized classes derived from BaseFab. + * + * Classes derived from BaseFab include FArrayBox, IArrayBox, TagBox, + * Mask, EBFArrayBox, EBCellFlag and CutFab. + * + * BaseFab objects depend on the dimensionality of space + * (indirectly through the DOMAIN Box member). It is + * typical to define the macro SPACEDIM to be 1, 2, or 3 to indicate + * the dimension of space. See the discussion of class Box for more + * information. A BaseFab contains a Box DOMAIN, which indicates the + * integer indexing space over which the array is defined. A BaseFab + * also has NVAR components. By components, we mean that for each + * point in the rectangular indexing space, there are NVAR values + * associated with that point. A Fortran array corresponding to a + * BaseFab would have (SPACEDIM+1) dimensions. + * + * By design, the array layout in a BaseFab mirrors that of a + * Fortran array. The first index (x direction for example) varies + * most rapidly, the next index (y direction), if any, varies next + * fastest. The component index varies last, after all the spatial + * indices. + * + * It is sometimes convenient to be able to treat a sub-array within an + * existing BaseFab as a BaseFab in its own right. This is often + * referred to as aliasing the BaseFab. Note that when aliasing is + * used, the BaseFabs domain will not, in general, be the same as the + * parent BaseFabs domain, nor will the number of components. + * BaseFab is a dimension dependent class, so SPACEDIM must be + * defined as either 1, 2, or 3 when compiling. + * + * This is NOT a polymorphic class. + * + * It does NOT provide a copy constructor or assignment operator. + * + * \tparam T MUST have a default constructor and an assignment operator. + */ template class BaseFab : public DataAllocator diff --git a/Src/Base/AMReX_BoxArray.H b/Src/Base/AMReX_BoxArray.H index 807cd9d8516..94358f580a4 100644 --- a/Src/Base/AMReX_BoxArray.H +++ b/Src/Base/AMReX_BoxArray.H @@ -515,16 +515,16 @@ struct BATransformer // for backward compatibility using BndryBATransformer = BATransformer; -/** -* \brief A collection of Boxes stored in an Array. It is a -* reference-counted concrete class, not a polymorphic one; i.e. you -* cannot use any of the List member functions with a BoxList. -*/ - class MFIter; class AmrMesh; class FabArrayBase; +/** + * \brief A collection of Boxes stored in an Array. + * + * It is a reference-counted concrete class, not a polymorphic one; i.e. you + * cannot use any of the List member functions with a BoxList. + */ class BoxArray { public: diff --git a/Src/Base/AMReX_FArrayBox.H b/Src/Base/AMReX_FArrayBox.H index b6dc4e887c8..2dda4e6b0a0 100644 --- a/Src/Base/AMReX_FArrayBox.H +++ b/Src/Base/AMReX_FArrayBox.H @@ -26,7 +26,6 @@ class FArrayBox; * primarily for FArrayBox implementers; i.e. user's shouldn't * call any of the member functions in this class directly. */ - class FABio // NOLINT(cppcoreguidelines-special-member-functions) { public: @@ -224,7 +223,6 @@ private: * This class does NOT provide a copy constructor or assignment operator, * but it has a move constructor. */ - class FArrayBox : public BaseFab diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H index 96efc1f18fa..56300d4cfef 100644 --- a/Src/Base/AMReX_FabArray.H +++ b/Src/Base/AMReX_FabArray.H @@ -56,64 +56,11 @@ Long nBytesOwned (T const&) noexcept { return 0; } template Long nBytesOwned (BaseFab const& fab) noexcept { return fab.nBytesOwned(); } -/* - A Collection of Fortran Array-like Objects - - - The FabArray class implements a collection (stored as an array) of - Fortran array-like objects. The parameterized type FAB is intended to be - any class derived from BaseFab. For example, FAB may be a BaseFab of - integers, so we could write: - - FabArray > int_fabs; - - Then int_fabs is a FabArray that can hold a collection of BaseFab - objects. - - FabArray is not just a general container class for Fortran arrays. It is - intended to hold "grid" data for use in finite difference calculations in - which the data is defined on a union of (usually disjoint) rectangular - regions embedded in a uniform index space. This region, called the valid - region, is represented by a BoxArray. For the purposes of this discussion, - the Kth Box in the BoxArray represents the interior region of the Kth grid. - - Since the intent is to be used with finite difference calculations a - FabArray also includes the notion of a boundary region for each grid. The - boundary region is specified by the ngrow parameter which tells the FabArray - to allocate each FAB to be ngrow cells larger in all directions than the - underlying Box. The larger region covered by the union of all the FABs is - called the region of definition. The underlying notion is that the valid - region contains the grid interior data and the region of definition includes - the interior region plus the boundary areas. - - Operations are available to copy data from the valid regions into these - boundary areas where the two overlap. The number of components, that is, - the number of values that can be stored in each cell of a FAB, is either - given as an argument to the constructor or is inherent in the definition of - the underlying FAB. Each FAB in the FabArray will have the same number of - components. - - In summary, a FabArray is an array of FABs. The Kth element contains a FAB - that holds the data for the Kth grid, a Box that defines the valid region - of the Kth grid. - - A typical use for a FabArray would be to hold the solution vector or - right-hand-side when solving a linear system of equations on a union of - rectangular grids. The copy operations would be used to copy data from the - valid regions of neighboring grids into the boundary regions after each - relaxation step of the iterative method. If a multigrid method is used, a - FabArray could be used to hold the data at each level in the multigrid - hierarchy. - - This class is a concrete class not a polymorphic one. - - This class does NOT provide a copy constructor or assignment operator. -*/ - -// -// alloc: allocate memory or not -// +/** + * \brief FabArray memory allocation information + */ struct MFInfo { + // alloc: allocate memory or not bool alloc = true; Arena* arena = nullptr; Vector tags; @@ -314,6 +261,60 @@ Add (FabArray& dst, FabArray const& src, int srccomp, int dstcomp, int } } +/** + * \brief An Array of FortranArrayBox(FAB)-like Objects + * + * The FabArray class implements a collection (stored as an array) of + * Fortran array box-like ( \p FAB ) objects. The parameterized type \p FAB is intended to be + * any class derived from BaseFab. For example, \p FAB may be a BaseFab of + * integers, so we could write: + * + * FabArray > int_fabs; + * + * Then int_fabs is a FabArray that can hold a collection of BaseFab + * objects. + * + * FabArray is not just a general container class for Fortran arrays. It is + * intended to hold "grid" data for use in finite difference calculations in + * which the data is defined on a union of (usually disjoint) rectangular + * regions embedded in a uniform index space. This region, called the valid + * region, is represented by a BoxArray. For the purposes of this discussion, + * the Kth Box in the BoxArray represents the interior region of the Kth grid. + * + * Since the intent is to be used with finite difference calculations a + * FabArray also includes the notion of a boundary region for each grid. The + * boundary region is specified by the ngrow parameter which tells the FabArray + * to allocate each \p FAB to be ngrow cells larger in all directions than the + * underlying Box. The larger region covered by the union of all the \p FABs is + * called the region of definition. The underlying notion is that the valid + * region contains the grid interior data and the region of definition includes + * the interior region plus the boundary areas. + * + * Operations are available to copy data from the valid regions into these + * boundary areas where the two overlap. The number of components, that is, + * the number of values that can be stored in each cell of a \p FAB, is either + * given as an argument to the constructor or is inherent in the definition of + * the underlying \p FAB. Each \p FAB in the FabArray will have the same number of + * components. + * + * In summary, a FabArray is an array of \p FABs. The Kth element contains a \p FAB + * that holds the data for the Kth grid, a Box that defines the valid region + * of the Kth grid. + * + * A typical use for a FabArray would be to hold the solution vector or + * right-hand-side when solving a linear system of equations on a union of + * rectangular grids. The copy operations would be used to copy data from the + * valid regions of neighboring grids into the boundary regions after each + * relaxation step of the iterative method. If a multigrid method is used, a + * FabArray could be used to hold the data at each level in the multigrid + * hierarchy. + * + * This class is a concrete class not a polymorphic one. + * + * This class does NOT provide a copy constructor or assignment operator. + * + * \tparam FAB FortranArrayBox-like object. Typically a derived class of BaseFab. Not to be confused with FabArrayBase. + */ template class FabArray : @@ -338,8 +339,9 @@ public: FabArray () noexcept; /** - * \brief Construct an empty FabArray that has a default Arena. If - * `define` is called later with a nullptr as MFInfo's arena, the + * \brief Construct an empty FabArray that has a default Arena. + * + * If `define` is called later with a nullptr as MFInfo's arena, the * default Arena `a` will be used. If the arena in MFInfo is not a * nullptr, the MFInfo's arena will be used. */ diff --git a/Src/Base/AMReX_FabArrayBase.H b/Src/Base/AMReX_FabArrayBase.H index 29d3d63b29e..d8bc4411874 100644 --- a/Src/Base/AMReX_FabArrayBase.H +++ b/Src/Base/AMReX_FabArrayBase.H @@ -28,6 +28,12 @@ template class FabArray; namespace EB2 { class IndexSpace; } +/** + * \brief Base class for FabArray. + * + * Not to be confused with FArrayBox or `FAB` shorthands. + * Can be read as FArrayBox-like Array Base. + */ class FabArrayBase { friend class MFIter; diff --git a/Src/Base/AMReX_FabConv.H b/Src/Base/AMReX_FabConv.H index 78554000f25..25dae063de7 100644 --- a/Src/Base/AMReX_FabConv.H +++ b/Src/Base/AMReX_FabConv.H @@ -13,20 +13,18 @@ namespace amrex { -// -// A Descriptor of the Long Integer type - /** -* This class is meant to hold all information needed to completely -* describe the "int" or "Long" type on a machine. To describe an integer both -* the number of bytes and their ordering, relative to canonical -* ordering 1 .. sizeof(Long), needs to be specified. -* This allows us to write out integers in the native format on a machine, -* and then by also saving the IntDescriptor, we can read them back in on -* another machine and have enough information to construct the exact same -* values. -*/ - + * \brief A Descriptor of the Long Integer type + * + * This class is meant to hold all information needed to completely + * describe the "int" or "Long" type on a machine. To describe an integer both + * the number of bytes and their ordering, relative to canonical + * ordering 1 .. sizeof(Long), needs to be specified. + * This allows us to write out integers in the native format on a machine, + * and then by also saving the IntDescriptor, we can read them back in on + * another machine and have enough information to construct the exact same + * values. + */ class IntDescriptor { @@ -72,39 +70,37 @@ std::ostream& operator<< (std::ostream& os, const IntDescriptor& id); //! std::istream& operator>> (std::istream& is, IntDescriptor& id); - - //A Descriptor of the Real Type - /** -* \brief This class is meant to hold all information needed to completely -* describe the "Real" floating-point type on a machine. By "Real" here we -* mean either the "float" or "double" type that this version of AMReX -* was built with, which corresponds to whether BL_USE_FLOAT or -* BL_USE_DOUBLE was used to build the version of the library. -* -* To describe a "Real" type two arrays are needed: one detailing the ordering -* of the bytes in the Real, relative to the canonical ordering -* 1 .. sizeof(Real) and the other detailing the format of the floating-point -* number. -* -* The array detailing the format of a floating-point number is an eight-element -* array of longs containing the following information: -* -* format[0] = number of bits per number -* format[1] = number of bits in exponent -* format[2] = number of bits in mantissa -* format[3] = start bit of sign -* format[4] = start bit of exponent -* format[5] = start bit of mantissa -* format[6] = high order mantissa bit (CRAY needs this) -* format[7] = bias of exponent -* -* This allows us to write out "Real"s in the native format on a machine, -* and then by also saving the IntDescriptor, we can read them back in on -* another machine and have enough information to construct the exact same -* "Real" values, provided the Reals have the same size on the two machines. -*/ - + * \brief A Descriptor of the Real Type + * + * This class is meant to hold all information needed to completely + * describe the "Real" floating-point type on a machine. By "Real" here we + * mean either the "float" or "double" type that this version of AMReX + * was built with, which corresponds to whether BL_USE_FLOAT or + * BL_USE_DOUBLE was used to build the version of the library. + * + * To describe a "Real" type two arrays are needed: one detailing the ordering + * of the bytes in the Real, relative to the canonical ordering + * 1 .. sizeof(Real) and the other detailing the format of the floating-point + * number. + * + * The array detailing the format of a floating-point number is an eight-element + * array of longs containing the following information: + * + * format[0] = number of bits per number + * format[1] = number of bits in exponent + * format[2] = number of bits in mantissa + * format[3] = start bit of sign + * format[4] = start bit of exponent + * format[5] = start bit of mantissa + * format[6] = high order mantissa bit (CRAY needs this) + * format[7] = bias of exponent + * + * This allows us to write out "Real"s in the native format on a machine, + * and then by also saving the IntDescriptor, we can read them back in on + * another machine and have enough information to construct the exact same + * "Real" values, provided the Reals have the same size on the two machines. + */ class RealDescriptor { public: diff --git a/Src/Base/AMReX_MultiFab.H b/Src/Base/AMReX_MultiFab.H index 1a6c1d7f15f..fdf2e67cbd8 100644 --- a/Src/Base/AMReX_MultiFab.H +++ b/Src/Base/AMReX_MultiFab.H @@ -24,16 +24,17 @@ using fMultiFab = FabArray >; class iMultiFab; /** - * \brief - * A collection (stored as an array) of FArrayBox objects. + * \brief A collection (stored as an array) of FArrayBox objects. + * * This class is useful for storing floating point data on a domain defined by * a union of rectangular regions embedded in a uniform index space. * MultiFab class extends the function of the underlying FabArray class just * as the FArrayBox class extends the function of BaseFab. - * Additional member functions are defined for I/O and simple arithmetic operations on these aggregate objects. + * Additional member functions are defined for I/O and simple arithmetic + * operations on these aggregate objects. + * * This class does NOT provide a copy constructor or assignment operator. */ - class MultiFab : public FabArray @@ -41,34 +42,36 @@ class MultiFab public: /** - * \brief Constructs an empty MultiFab. Data can be defined at a later - * time using the define member functions inherited - * from FabArray. + * \brief Constructs an empty MultiFab. + * + * Data can be defined at a later time using the define member functions + * inherited from FabArray. */ MultiFab () noexcept; /** - * \brief Constructs an empty MultiFab. Data can be defined at a later - * time using the define member functions inherited from FabArray. If - * `define` is called later with a nullptr as MFInfo's arena, the default - * Arena `a` will be used. If the arena in MFInfo is not a nullptr, the - * MFInfo's arena will be used. + * \brief Constructs an empty MultiFab. + * + * Data can be defined at a later time using the define member functions. + * If `define` is called later with a nullptr as MFInfo's arena, the + * default Arena `a` will be used. If the arena in MFInfo is not a + * nullptr, the MFInfo's arena will be used. */ explicit MultiFab (Arena* a) noexcept; /** - * \brief - * Constructs a MultiFab - * \param bs a valid region - * \param dm a DistributionMapping + * \brief Constructs a MultiFab + * + * The size of the FArrayBox is given by the Box grown by \p ngrow, and + * the number of components is given by \p ncomp. If \p info is set to + * not allocating memory, then no FArrayBoxes are allocated at + * this time but can be defined later. + * + * \param bxs a valid region + * \param dm a DistribuionMapping * \param ncomp number of components * \param ngrow number of cells the region grows * \param info MFInfo - - * The size of the FArrayBox is given by the Box grown by ngrow, and - * the number of components is given by ncomp. If info is set to - * not allocating memory, then no FArrayBoxes are allocated at - * this time but can be defined later. */ MultiFab (const BoxArray& bxs, const DistributionMapping& dm, @@ -95,10 +98,11 @@ public: #endif /** - * \brief Make an alias MultiFab. maketype must be - * amrex::make_alias. scomp is the starting component of the - * alias and ncomp is the number of components in the new aliasing - * MultiFab. + * \brief Make an alias MultiFab. + * + * Note that \p maketype must be `amrex::make_alias`, + * \p scomp is the starting component of the alias, and + * \p ncomp is the number of components in the new aliasing MultiFab. */ MultiFab (const MultiFab& rhs, MakeType maketype, int scomp, int ncomp); @@ -135,11 +139,13 @@ public: #endif MultiFab& operator= (Real r); - // + /** - * \brief Returns the minimum value contained in component comp of the - * MultiFab. The parameter nghost determines the number of - * boundary cells to search for the minimum. The default is to + * \brief Returns the minimum value contained in component \p comp of the + * MultiFab. + * + * The parameter \p nghost determines the number of + * boundary cells to search for the minimum. The default is to * search only the valid regions of the FArrayBoxes. */ [[nodiscard]] Real min (int comp, @@ -154,16 +160,18 @@ public: int nghost = 0, bool local = false) const; /** - * \brief Returns the maximum value contained in component comp of the - * MultiFab. The parameter nghost determines the number of - * boundary cells to search for the maximum. The default is to + * \brief Returns the maximum value contained in component \p comp of the + * MultiFab. + * + * The parameter \p nghost determines the number of + * boundary cells to search for the maximum. The default is to * search only the valid regions of the FArrayBoxes. */ [[nodiscard]] Real max (int comp, int nghost = 0, bool local = false) const; /** - * \brief Identical to the previous max() function, but confines its + * \brief Identical to the previous `max()` function, but confines its * search to intersection of Box b and the MultiFab. */ [[nodiscard]] Real max (const Box& region, @@ -191,7 +199,7 @@ public: /** * \brief Returns the maximum *absolute* values contained in - * each component of "comps" of the MultiFab. "nghost" ghost cells are used. + * each component of \p comps of the MultiFab. \p nghost ghost cells are used. */ [[nodiscard]] Vector norm0 (const Vector& comps, int nghost = 0, bool local = false, bool ignore_covered = false ) const; [[nodiscard]] Vector norminf (const Vector& comps, int nghost = 0, bool local = false, bool ignore_covered = false) const { @@ -199,13 +207,14 @@ public: } /** - * \brief Returns the L1 norm of component "comp" over the MultiFab. + * \brief Returns the L1 norm of component \p comp over the MultiFab. + * * No ghost cells are used. This version has no double counting for nodal data. */ [[nodiscard]] Real norm1 (int comp, const Periodicity& period, bool ignore_covered = false) const; /** - * \brief Returns the L1 norm of component "comp" over the MultiFab. - * ngrow ghost cells are used. + * \brief Returns the L1 norm of component \p comp over the MultiFab. + * \p ngrow ghost cells are used. */ [[nodiscard]] Real norm1 (int comp = 0, int ngrow = 0, bool local = false) const; /** @@ -214,12 +223,12 @@ public: */ [[nodiscard]] Vector norm1 (const Vector& comps, int ngrow = 0, bool local = false) const; /** - * \brief Returns the L2 norm of component "comp" over the MultiFab. + * \brief Returns the L2 norm of component \p comp over the MultiFab. * No ghost cells are used. */ [[nodiscard]] Real norm2 (int comp = 0) const; /** - * \brief Returns the L2 norm of component "comp" over the MultiFab. + * \brief Returns the L2 norm of component \p comp over the MultiFab. * No ghost cells are used. This version has no double counting for nodal data. */ [[nodiscard]] Real norm2 (int comp, const Periodicity& period) const; @@ -236,16 +245,17 @@ public: using FabArray::sum; /** - * \brief Same as sum with local=false, but for non-cell-centered data, this - * skips non-unique points that are owned by multiple boxes. + * \brief Same as sum with \p local =false, but for non-cell-centered data, this + * skips non-unique points that are owned by multiple boxes. */ [[nodiscard]] Real sum_unique (int comp = 0, bool local = false, const Periodicity& period = Periodicity::NonPeriodic()) const; /** - * \brief Adds the scalar value val to the value of each cell in the - * specified subregion of the MultiFab. The subregion consists - * of the num_comp components starting at component comp. + * \brief Adds the scalar value \p val to the value of each cell in the + * specified subregion of the MultiFab. + * + * The subregion consists of the \p num_comp components starting at component \p comp. * The value of nghost specifies the number of cells in the * boundary region of each FArrayBox in the subregion that should * be modified. diff --git a/Src/Base/AMReX_MultiFabUtil.H b/Src/Base/AMReX_MultiFabUtil.H index 064b8a1b19f..29af89ba88e 100644 --- a/Src/Base/AMReX_MultiFabUtil.H +++ b/Src/Base/AMReX_MultiFabUtil.H @@ -19,67 +19,76 @@ namespace amrex const MultiFab& nd, int scomp, int ncomp, int ngrow = 0); - //! Average edge-based MultiFab onto cell-centered MultiFab. This fills in - //! ngrow ghost cells in the cell-centered MultiFab. Both cell centered and - //! edge centered MultiFabs need to have ngrow ghost values + /** + * \brief Average edge-based MultiFab onto cell-centered MultiFab. + * + * This fills in \p ngrow ghost cells in the cell-centered MultiFab. Both cell centered and + * edge centered MultiFabs need to have \p ngrow ghost values. + */ void average_edge_to_cellcenter (MultiFab& cc, int dcomp, const Vector& edge, int ngrow = 0); - //! Average face-based MultiFab onto cell-centered MultiFab. void average_face_to_cellcenter (MultiFab& cc, int dcomp, const Vector& fc, int ngrow = 0); - + //! Average face-based FabArray onto cell-centered FabArray. template && IsFabArray_v, int> = 0> void average_face_to_cellcenter (CMF& cc, int dcomp, const Array& fc, int ngrow = 0); - + //! Average face-based MultiFab onto cell-centered MultiFab with geometric weighting. void average_face_to_cellcenter (MultiFab& cc, const Vector& fc, const Geometry& geom); + //! Average face-based MultiFab onto cell-centered MultiFab with geometric weighting. void average_face_to_cellcenter (MultiFab& cc, const Array& fc, const Geometry& geom); - - //! Average cell-centered MultiFab onto face-based MultiFab. + //! Average cell-centered MultiFab onto face-based MultiFab with geometric weighting. void average_cellcenter_to_face (const Vector& fc, const MultiFab& cc, const Geometry& geom, int ncomp = 1, bool use_harmonic_averaging = false); + //! Average cell-centered MultiFab onto face-based MultiFab with geometric weighting. void average_cellcenter_to_face (const Array& fc, const MultiFab& cc, const Geometry& geom, int ncomp = 1, bool use_harmonic_averaging = false); - //! Average fine face-based MultiFab onto crse face-based MultiFab. + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Vector& fine, const Vector& crse, const IntVect& ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Vector& fine, const Vector& crse, int ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Array& fine, const Array& crse, const IntVect& ratio, int ngcrse = 0); + //! Average fine face-based FabArray onto crse face-based FabArray. template ::value,int> = 0> void average_down_faces (const Array& fine, const Array& crse, int ratio, int ngcrse = 0); - //! This version does average down for one direction. - //! It uses the IndexType of MultiFabs to determine the direction. - //! It is expected that one direction is nodal and the rest are cell-centered. + /** + * \brief This version does average down for one face direction. + * + * It uses the IndexType of MultiFabs to determine the direction. + * It is expected that one direction is nodal and the rest are cell-centered. + */ template void average_down_faces (const FabArray& fine, FabArray& crse, const IntVect& ratio, int ngcrse=0); @@ -117,9 +126,12 @@ namespace amrex int ngcrse = 0, bool mfiter_is_definitely_safe=false); - //! Average fine cell-based MultiFab onto crse cell-centered MultiFab using - //! volume-weighting. This routine DOES NOT assume that the crse BoxArray is - //! a coarsened version of the fine BoxArray. + /** + * \brief Volume weighed average of fine MultiFab onto coarse MultiFab. + * + * Both MultiFabs are assumed to be cell-centered. This routine DOES NOT assume that + * the crse BoxArray is a coarsened version of the fine BoxArray. + */ void average_down (const MultiFab& S_fine, MultiFab& S_crse, const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, const IntVect& ratio); diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp index 5520c6164d7..27efbffca0f 100644 --- a/Src/Base/AMReX_MultiFabUtil.cpp +++ b/Src/Base/AMReX_MultiFabUtil.cpp @@ -308,9 +308,6 @@ namespace amrex // ************************************************************************************************************* - // Average fine cell-based MultiFab onto crse cell-centered MultiFab. - // We do NOT assume that the coarse layout is a coarsened version of the fine layout. - // This version DOES use volume-weighting. void average_down (const MultiFab& S_fine, MultiFab& S_crse, const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, int rr) diff --git a/Src/Base/AMReX_OpenMP.cpp b/Src/Base/AMReX_OpenMP.cpp index 5ddd9944411..c0c33ce962f 100644 --- a/Src/Base/AMReX_OpenMP.cpp +++ b/Src/Base/AMReX_OpenMP.cpp @@ -152,12 +152,13 @@ namespace amrex::OpenMP // default or OMP_NUM_THREADS environment variable } else if (omp_threads == "nosmt") { char const *env_omp_num_threads = std::getenv("OMP_NUM_THREADS"); - if (env_omp_num_threads != nullptr && amrex::system::verbose > 1) { + if (env_omp_num_threads == nullptr) { + omp_set_num_threads(numUniquePhysicalCores()); + } + else if (amrex::system::verbose > 1) { amrex::Print() << "amrex.omp_threads was set to nosmt," << "but OMP_NUM_THREADS was set. Will keep " << "OMP_NUM_THREADS=" << env_omp_num_threads << ".\n"; - } else { - omp_set_num_threads(numUniquePhysicalCores()); } } else { std::optional num_omp_threads = to_int(omp_threads); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H index 4768f06fc41..3afa56ee245 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H @@ -262,7 +262,6 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) MF sorig = Lp.make(amrlev, mglev, nghost); MF r = Lp.make(amrlev, mglev, nghost); - MF z = Lp.make(amrlev, mglev, nghost); MF q = Lp.make(amrlev, mglev, nghost); sorig.LocalCopy(sol,0,0,ncomp,nghost); @@ -295,9 +294,7 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) for (; iter <= maxiter; ++iter) { - z.LocalCopy(r,0,0,ncomp,nghost); - - RT rho = dotxy(z,r); + RT rho = dotxy(r,r); if ( rho == 0 ) { @@ -305,12 +302,12 @@ MLCGSolverT::solve_cg (MF& sol, const MF& rhs, RT eps_rel, RT eps_abs) } if (iter == 1) { - p.LocalCopy(z,0,0,ncomp,nghost); + p.LocalCopy(r,0,0,ncomp,nghost); } else { RT beta = rho/rho_1; - MF::Xpay(p, beta, z, 0, 0, ncomp, nghost); // p = z + beta * p + MF::Xpay(p, beta, r, 0, 0, ncomp, nghost); // p = r + beta * p } Lp.apply(amrlev, mglev, q, p, MLLinOpT::BCMode::Homogeneous, MLLinOpT::StateMode::Correction); diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index ea24b1e5f81..e6969c9b1bd 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -32,6 +32,14 @@ ParticleContainer_impl + AMREX_GPU_HOST_DEVICE + int operator() (const P& p) const { + return p.id() > 0; + } +}; + template class Allocator, class CellAssignor> void @@ -77,10 +85,7 @@ ParticleContainer_impl int - { - return p.id() > 0; - }, true); + FilterPositiveID{}, true); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template 0; - }); + FilterPositiveID{}); } template p_prime; - AMREX_D_TERM(p_prime.pos(0) = src_data.pos(0, i+this_offset);, - p_prime.pos(1) = src_data.pos(1, i+this_offset);, - p_prime.pos(2) = src_data.pos(2, i+this_offset);); - + auto p_prime = src_data.getSuperParticle(i+this_offset); enforcePeriodic(p_prime, plo, phi, rlo, rhi, is_per); auto tup_prime = ploc(p_prime, lev_min, lev_max, nGrow, assignor); assigned_grid = amrex::get<0>(tup_prime); diff --git a/Tools/GNUMake/Make.machines b/Tools/GNUMake/Make.machines index 1e6293dd001..db91bb99947 100644 --- a/Tools/GNUMake/Make.machines +++ b/Tools/GNUMake/Make.machines @@ -123,9 +123,11 @@ ifeq ($(findstring asterix, $(host_name)), asterix) endif ifeq ($(findstring aurora, $(host_name)), aurora) +ifneq ($(findstring alcf.anl.gov, $(host_name)),alcf.anl.gov) which_site := hs which_computer := aurora endif +endif ifeq ($(findstring kestrel, $(NREL_CLUSTER)), kestrel) which_site := nrel diff --git a/Tools/Postprocessing/C_Src/particle_compare.cpp b/Tools/Postprocessing/C_Src/particle_compare.cpp index 9967625cca6..e9efc69cf62 100644 --- a/Tools/Postprocessing/C_Src/particle_compare.cpp +++ b/Tools/Postprocessing/C_Src/particle_compare.cpp @@ -438,12 +438,15 @@ int main_main() std::string fn2; std::string pt; Real rtol = 0.0; + Real atol = 0.0; int farg=1; while (farg <= narg) { const std::string fname = amrex::get_command_argument(farg); if (fname == "-r" || fname == "--rel_tol") { rtol = std::stod(amrex::get_command_argument(++farg)); + } else if (fname == "--abs_tol") { + atol = std::stod(amrex::get_command_argument(++farg)); } else { break; } @@ -472,6 +475,7 @@ int main_main() << "\n" << " optional arguments:\n" << " -r|--rel_tol rtol : relative tolerance (default is 0)\n" + << " --abs_tol atol : absolute tolerance (default is 0)\n" << std::endl; return EXIT_SUCCESS; } @@ -555,16 +559,25 @@ int main_main() int exit_code = 0; for (int i = 0; i < header1.num_comp; ++i) { - if (global_norms[i+header1.num_comp] > rtol) exit_code = 1; + if (global_norms[i ] > atol && + global_norms[i+header1.num_comp] > rtol) exit_code = 1; } if (exit_code == 0) { - amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << "\n"; + if (atol > 0.) { + amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << " and/or absolute tolerance " << atol << "\n"; + } else { + amrex::Print() << " PARTICLES AGREE to relative tolerance " << rtol << "\n"; + } } else { - amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << "\n"; + if (atol > 0.) { + amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << " and/or absolute tolerance " << atol << "\n"; + } else { + amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << "\n"; + } } return exit_code;