Remove reliance on managed memory in HDF5 mesh IO (#4047)

The proposed changes:
- [x] fix a bug or incorrect behavior in AMReX
- [ ] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

Src/Extern/HDF5/AMReX_PlotFileUtilHDF5.cpp

@@ -696,12 +696,24 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
         for(MFIter mfi(*data); mfi.isValid(); ++mfi) {
             const FArrayBox &fab = (*data)[mfi];
             writeDataItems = fab.box().numPts() * (*data).nComp();
+            Real const* fabdata = fab.dataPtr();
+#ifdef AMREX_USE_GPU
+            std::unique_ptr<FArrayBox> hostfab;
+            if (fab.arena()->isManaged() || fab.arena()->isDevice()) {
+                hostfab = std::make_unique<FArrayBox>(fab.box(), fab.nComp(),
+                                                      The_Pinned_Arena());
+                Gpu::dtoh_memcpy_async(hostfab->dataPtr(), fab.dataPtr(),
+                                       fab.size()*sizeof(Real));
+                Gpu::streamSynchronize();
+                fabdata = hostfab->dataPtr();
+            }
+#endif
             if(doConvert) {
                 RealDescriptor::convertFromNativeFormat(static_cast<void *> (a_buffer.dataPtr()+writeDataSize),
-                                                        writeDataItems, fab.dataPtr(), *whichRD);
+                                                        writeDataItems, fabdata, *whichRD);
             } else {    // ---- copy from the fab
                 memcpy(static_cast<void *> (a_buffer.dataPtr()+writeDataSize),
-                       fab.dataPtr(), writeDataItems * whichRDBytes);
+                       fabdata, writeDataItems * whichRDBytes);
             }
             writeDataSize += writeDataItems;
         }