Fix mechanical simulation with MPI

application/adamantine.hh

@@ -304,18 +304,18 @@ template <int dim, int p_order, typename MaterialStates,
 void refine_and_transfer(
     std::unique_ptr<adamantine::ThermalPhysicsInterface<dim, MemorySpaceType>>
         &thermal_physics,
+    std::unique_ptr<adamantine::MechanicalPhysics<
+        dim, p_order, MaterialStates, MemorySpaceType>> &mechanical_physics,
     adamantine::MaterialProperty<dim, p_order, MaterialStates, MemorySpaceType>
         &material_properties,
     dealii::DoFHandler<dim> &dof_handler,
     dealii::LA::distributed::Vector<double, MemorySpaceType> &solution)
 {
+	std::cout << "refine_and_transfer" << std::endl;
 #ifdef ADAMANTINE_WITH_CALIPER
   CALI_CXX_MARK_FUNCTION;
 #endif
 
-  // TODO transfer mechanical data is present, need to be aware that the data
-  // does not exist on some cells because it's liquid
-
   dealii::parallel::distributed::Triangulation<dim> &triangulation =
       dynamic_cast<dealii::parallel::distributed::Triangulation<dim> &>(
           const_cast<dealii::Triangulation<dim> &>(
@@ -414,6 +414,13 @@ void refine_and_transfer(
       cell_data_trans(triangulation);
   cell_data_trans.prepare_for_coarsening_and_refinement(data_to_transfer);
 
+  if (mechanical_physics)
+  {
+    // Thermo-mechanical simulation
+    mechanical_physics->prepare_transfer(
+        thermal_physics->get_dof_handler());
+  }
+
 #ifdef ADAMANTINE_WITH_CALIPER
   CALI_MARK_BEGIN("refine triangulation");
 #endif
@@ -512,6 +519,20 @@ void refine_and_transfer(
     }
   }
 #endif
+
+  if (mechanical_physics)
+  {
+	        thermal_physics->set_state_to_material_properties();
+          dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host>
+              temperature_host(solution.get_partitioner());
+          temperature_host.import(solution, dealii::VectorOperation::insert);
+          mechanical_physics->setup_dofs(
+              thermal_physics->get_dof_handler(), temperature_host,
+              thermal_physics->get_has_melted_vector());
+    // Thermo-mechanical simulation
+    mechanical_physics->complete_transfer();
+  }
+
 }
 
 template <int dim>
@@ -570,6 +591,8 @@ template <int dim, int p_order, int fe_degree, typename MaterialStates,
 void refine_mesh(
     std::unique_ptr<adamantine::ThermalPhysicsInterface<dim, MemorySpaceType>>
         &thermal_physics,
+    std::unique_ptr<adamantine::MechanicalPhysics<
+        dim, p_order, MaterialStates, MemorySpaceType>> &mechanical_physics,
     adamantine::MaterialProperty<dim, p_order, MaterialStates, MemorySpaceType>
         &material_properties,
     dealii::LA::distributed::Vector<double, MemorySpaceType> &solution,
@@ -578,6 +601,7 @@ void refine_mesh(
     unsigned int const time_steps_refinement,
     boost::property_tree::ptree const &refinement_database)
 {
+	std::cout << "refine_mesh" << std::endl;
 #ifdef ADAMANTINE_WITH_CALIPER
   CALI_CXX_MARK_FUNCTION;
 #endif
@@ -610,6 +634,8 @@ void refine_mesh(
   const double refinement_beam_cutoff =
       refinement_database.get<double>("beam_cutoff", 1.0e-15);
 
+  std::cout << "n_refinements: " << n_refinements << std::endl;
+
   for (unsigned int i = 0; i < n_refinements; ++i)
   {
     // Compute the cells to be refined.
@@ -645,9 +671,11 @@ void refine_mesh(
         cell->set_refine_flag();
     }
 
+    std::cout << "call refine_and_transfer" << std::endl;
+
     // Execute the refinement and transfer the solution onto the new mesh.
-    refine_and_transfer(thermal_physics, material_properties, dof_handler,
-                        solution);
+    refine_and_transfer(thermal_physics, mechanical_physics,
+                        material_properties, dof_handler, solution);
   }
 
   // Recompute the inverse of the mass matrix
@@ -659,6 +687,8 @@ template <int dim, int p_order, typename MaterialStates,
 void refine_mesh(
     std::unique_ptr<adamantine::ThermalPhysicsInterface<dim, MemorySpaceType>>
         &thermal_physics,
+    std::unique_ptr<adamantine::MechanicalPhysics<
+        dim, p_order, MaterialStates, MemorySpaceType>> &mechanical_physics,
     adamantine::MaterialProperty<dim, p_order, MaterialStates, MemorySpaceType>
         &material_properties,
     dealii::LA::distributed::Vector<double, MemorySpaceType> &solution,
@@ -675,36 +705,41 @@ void refine_mesh(
   case 1:
   {
     refine_mesh<dim, p_order, 1, MaterialStates>(
-        thermal_physics, material_properties, solution, heat_sources, time,
-        next_refinement_time, time_steps_refinement, refinement_database);
+        thermal_physics, mechanical_physics, material_properties, solution,
+        heat_sources, time, next_refinement_time, time_steps_refinement,
+        refinement_database);
     break;
   }
   case 2:
   {
     refine_mesh<dim, p_order, 2, MaterialStates>(
-        thermal_physics, material_properties, solution, heat_sources, time,
-        next_refinement_time, time_steps_refinement, refinement_database);
+        thermal_physics, mechanical_physics, material_properties, solution,
+        heat_sources, time, next_refinement_time, time_steps_refinement,
+        refinement_database);
     break;
   }
   case 3:
   {
     refine_mesh<dim, p_order, 3, MaterialStates>(
-        thermal_physics, material_properties, solution, heat_sources, time,
-        next_refinement_time, time_steps_refinement, refinement_database);
+        thermal_physics, mechanical_physics, material_properties, solution,
+        heat_sources, time, next_refinement_time, time_steps_refinement,
+        refinement_database);
     break;
   }
   case 4:
   {
     refine_mesh<dim, p_order, 4, MaterialStates>(
-        thermal_physics, material_properties, solution, heat_sources, time,
-        next_refinement_time, time_steps_refinement, refinement_database);
+        thermal_physics, mechanical_physics, material_properties, solution,
+        heat_sources, time, next_refinement_time, time_steps_refinement,
+        refinement_database);
     break;
   }
   case 5:
   {
     refine_mesh<dim, p_order, 5, MaterialStates>(
-        thermal_physics, material_properties, solution, heat_sources, time,
-        next_refinement_time, time_steps_refinement, refinement_database);
+        thermal_physics, mechanical_physics, material_properties, solution,
+        heat_sources, time, next_refinement_time, time_steps_refinement,
+        refinement_database);
     break;
   }
   default:
@@ -797,6 +832,7 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
   }
   material_reference_temps.push_back(initial_temperature);
 
+  std::cout << "create mechanicalphysics" << std::endl;
   // Create MechanicalPhysics
   std::unique_ptr<adamantine::MechanicalPhysics<dim, p_order, MaterialStates,
                                                 MemorySpaceType>>
@@ -872,6 +908,7 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
   {
     if (restart == false)
     {
+	       std::cout << "setup_dofs thermal" << std::endl;
       thermal_physics->setup();
       thermal_physics->initialize_dof_vector(initial_temperature, temperature);
     }
@@ -893,12 +930,14 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
     adamantine::ASSERT_THROW(
         restart == false,
         "Mechanical simulation cannot be restarted from a file");
+   std::cout << "setup_dofs mechanical begin" << std::endl;   
     if (use_thermal_physics)
     {
       // Thermo-mechanical simulation
       dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host>
           temperature_host(temperature.get_partitioner());
       temperature_host.import(temperature, dealii::VectorOperation::insert);
+      mechanical_physics->set_active_fe_indices(thermal_physics->get_dof_handler());
       mechanical_physics->setup_dofs(thermal_physics->get_dof_handler(),
                                      temperature_host,
                                      thermal_physics->get_has_melted_vector());
@@ -986,8 +1025,8 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
     {
       timers[adamantine::refine].start();
       double next_refinement_time = time + time_steps_refinement * time_step;
-      refine_mesh(thermal_physics, material_properties, temperature,
-                  heat_sources, time, next_refinement_time,
+      refine_mesh(thermal_physics, mechanical_physics, material_properties,
+                  temperature, heat_sources, time, next_refinement_time,
                   time_steps_refinement, refinement_database);
       timers[adamantine::refine].stop();
       if ((rank == 0) && (verbose_output == true))
@@ -1120,25 +1159,7 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
     {
       // Since there is no history dependence in the model, only calculate
       // mechanics when outputting
-      if (n_time_step % time_steps_output == 0)
-      {
-        if (use_thermal_physics)
-        {
-          // Update the material state
-          thermal_physics->set_state_to_material_properties();
-          dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host>
-              temperature_host(temperature.get_partitioner());
-          temperature_host.import(temperature, dealii::VectorOperation::insert);
-          mechanical_physics->setup_dofs(
-              thermal_physics->get_dof_handler(), temperature_host,
-              thermal_physics->get_has_melted_vector());
-        }
-        else
-        {
-          mechanical_physics->setup_dofs();
-        }
-        displacement = mechanical_physics->solve();
-      }
+      displacement = mechanical_physics->solve();
     }
 
     timers[adamantine::evol_time].stop();
@@ -1788,7 +1809,11 @@ run_ensemble(MPI_Comm const &global_communicator,
 
       for (unsigned int member = 0; member < local_ensemble_size; ++member)
       {
-        refine_mesh(thermal_physics_ensemble[member],
+        // FIXME
+        std::unique_ptr<adamantine::MechanicalPhysics<
+            dim, p_order, MaterialStates, MemorySpaceType>>
+            dummy;
+        refine_mesh(thermal_physics_ensemble[member], dummy,
                     *material_properties_ensemble[member],
                     solution_augmented_ensemble[member].block(base_state),
                     heat_sources_ensemble[member], time, next_refinement_time,

source/MaterialProperty.templates.hh

@@ -210,6 +210,7 @@ template <int dim, int p_order, typename MaterialStates,
 void MaterialProperty<dim, p_order, MaterialStates,
                       MemorySpaceType>::reinit_dofs()
 {
+  std::cout << "Distribute dofs Material" << std::endl;
   _mp_dof_handler.distribute_dofs(_fe);
 
   // Initialize _dofs_map