9 #include <use_cases/UseCase_Rebal_4.hpp> 11 #include <stk_util/parallel/Parallel.hpp> 12 #include <stk_util/parallel/ParallelReduce.hpp> 14 #include <stk_mesh/base/FieldData.hpp> 15 #include <stk_mesh/base/GetEntities.hpp> 16 #include <stk_mesh/base/MetaData.hpp> 17 #include <stk_mesh/base/BulkData.hpp> 19 #include <stk_mesh/fem/CoordinateSystems.hpp> 20 #include <stk_mesh/fem/FEMMetaData.hpp> 21 #include <stk_mesh/fem/FEMHelpers.hpp> 22 #include <stk_mesh/fem/CreateAdjacentEntities.hpp> 24 #include <stk_mesh/fixtures/HexFixture.hpp> 30 #include <stk_rebalance_utils/RebalanceUtils.hpp> 42 class GreedySideset :
public Partition {
45 std::vector<mesh::Entity *> mesh_entities;
48 std::vector<unsigned> dest_proc_ids ;
52 nodal_coord_ref(NULL),
53 elem_weight_ref(NULL) {}
60 mesh::BulkData & bulk_data);
61 virtual ~GreedySideset();
62 virtual void reset_dest_proc_data();
63 virtual void set_mesh_info (
const std::vector<mesh::Entity *> &mesh_entities,
66 virtual void determine_new_partition(
bool &RebalancingNeeded);
67 virtual unsigned num_elems()
const;
68 virtual int get_new_partition(stk_classic::mesh::EntityProcVec &new_partition);
69 virtual bool partition_dependents_needed()
const;
70 bool find_mesh_entity(
const mesh::Entity * entity,
unsigned & moid)
const;
71 unsigned destination_proc(
const unsigned moid)
const;
72 void set_destination_proc(
const unsigned moid,
const unsigned proc );
73 MeshInfo mesh_information_;
74 unsigned total_number_entities_;
76 mesh::BulkData & bulk_data_;
81 mesh::BulkData & bulk_data) :
85 bulk_data_(bulk_data) {}
86 GreedySideset::~GreedySideset() {}
87 void GreedySideset::reset_dest_proc_data() {
89 const unsigned size = mesh_information_.mesh_entities.size();
90 mesh_information_.dest_proc_ids.assign(size, proc);
92 void GreedySideset::set_mesh_info (
const std::vector<mesh::Entity *> &mesh_entities,
98 total_number_entities_ = mesh_entities.size();
100 mesh_info.mesh_entities = mesh_entities;
101 mesh_info.nodal_coord_ref = nodal_coord_ref;
102 mesh_info.elem_weight_ref = elem_weight_ref;
111 mesh_information_ = mesh_info;
114 unsigned GreedySideset::num_elems()
const {
return total_number_entities_ ;}
115 int GreedySideset::get_new_partition(stk_classic::mesh::EntityProcVec &new_partition){
116 std::vector<mesh::Entity*>::iterator i=mesh_information_.mesh_entities.begin();
117 std::vector<unsigned> ::iterator j=mesh_information_.dest_proc_ids.begin();
118 for (;i != mesh_information_.mesh_entities.end(),
119 j != mesh_information_.dest_proc_ids.end();
121 mesh::Entity * mesh_entity = *i;
124 new_partition.push_back(et);
128 bool GreedySideset::partition_dependents_needed()
const{
return true;}
130 bool GreedySideset::find_mesh_entity(
const mesh::Entity * entity,
unsigned & moid)
const 132 unsigned len = mesh_information_.mesh_entities.size();
133 for(moid = 0; moid < len; ++moid)
135 if(mesh_information_.mesh_entities[moid] == entity)
return true;
139 unsigned GreedySideset::destination_proc(
const unsigned moid)
const 141 return mesh_information_.dest_proc_ids[ moid ];
143 void GreedySideset::set_destination_proc(
const unsigned moid,
144 const unsigned proc )
146 mesh_information_.dest_proc_ids[ moid ] = proc;
150 void GreedySideset::determine_new_partition(
bool &RebalancingNeeded) {
152 reset_dest_proc_data();
155 const stk_classic::mesh::EntityRank side_rank = fem_meta.
side_rank();
156 const stk_classic::mesh::EntityRank elem_rank = fem_meta.
element_rank();
160 stk_classic::mesh::selectIntersection(surfaces_));
162 mesh::EntityVector sides;
165 const unsigned p_rank = bulk_data_.parallel_rank();
166 size_t local_changes = 0;
167 const unsigned nSide = sides.size();
168 for(
unsigned iSide = 0; iSide < nSide; ++iSide)
170 const mesh::Entity & side = *sides[iSide];
171 const unsigned sideProc = side.owner_rank();
172 ThrowRequireMsg(sideProc!=p_rank,
173 "When iterating Non-locally owned sides, found a locally owned side.");
176 for ( ; iElem.first != iElem.second; ++iElem.first ) {
177 const mesh::Entity & elem = *iElem.first->entity();
179 const bool mesh_entity_found = find_mesh_entity(&elem, moid);
180 if (mesh_entity_found) {
181 const unsigned elemProc = elem.owner_rank();
182 ThrowRequireMsg(elemProc==p_rank,
183 "When iterating locally owned elements, found a non-locally owned element.");
184 const unsigned destProc = destination_proc(moid);
185 ThrowRequireMsg(destProc==p_rank || destProc==sideProc,
186 " Sanity check failed: " 187 "It's possible that an element is connected to " 188 "two sides that are owned by different procs. We don't " 189 "yet handle that situation here but we can, at least, " 191 if(elemProc != sideProc)
194 set_destination_proc(moid, sideProc);
199 size_t global_changes = 0;
201 RebalancingNeeded = global_changes > 0;
204 enum { nx = 3, ny = 3 };
208 unsigned spatial_dimension = 2;
209 std::vector<std::string> rank_names = stk_classic::mesh::fem::entity_rank_names(spatial_dimension);
214 const stk_classic::mesh::EntityRank element_rank = fem_meta.
element_rank();
215 const stk_classic::mesh::EntityRank node_rank = fem_meta.
node_rank();
217 stk_classic::mesh::fem::CellTopology quad_top(shards::getCellTopologyData<shards::Quadrilateral<4> >());
218 stk_classic::mesh::fem::CellTopology line_top(shards::getCellTopologyData<shards::Line<2> >());
228 const unsigned p_rank = bulk_data.parallel_rank();
229 bulk_data.modification_begin();
233 std::vector<std::vector<stk_classic::mesh::Entity*> > quads(nx);
234 for (
unsigned ix = 0 ; ix < nx ; ++ix ) quads[ix].resize(ny);
236 const unsigned nnx = nx + 1 ;
237 for (
unsigned iy = 0 ; iy < ny ; ++iy ) {
238 for (
unsigned ix = 0 ; ix < nx ; ++ix ) {
239 stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
240 stk_classic::mesh::EntityId nodes[4] ;
241 nodes[0] = 1 + ix + iy * nnx ;
242 nodes[1] = 2 + ix + iy * nnx ;
243 nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
244 nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;
251 for (
unsigned iy = 0 ; iy < ny ; ++iy ) {
252 for (
unsigned ix = 0 ; ix < nx ; ++ix ) {
253 stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
259 for (
unsigned iy = 0 ; iy <= ny ; ++iy ) {
260 for (
unsigned ix = 0 ; ix <= nx ; ++ix ) {
261 stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
271 bulk_data.modification_end();
275 stk_classic::mesh::create_adjacent_entities(bulk_data, add_parts);
277 bulk_data.modification_begin();
283 const stk_classic::mesh::EntityRank side_rank = fmeta.
side_rank();
285 mesh::EntityVector sides;
288 const unsigned nSide = sides.size();
289 for(
unsigned iSide = 0; iSide < nSide; ++iSide)
291 mesh::Entity & side = *sides[iSide];
292 if (side.identifier()==7) {
293 bulk_data.change_entity_parts(side, surfaces, empty_remove_parts);
297 bulk_data.modification_end();
301 Teuchos::ParameterList emptyList;
302 stk_classic::rebalance::Zoltan zoltan_partition(comm, spatial_dimension, emptyList);
306 const int print_stats = 1;
308 double entity_wgt = 0;
313 const int ierr = zoltan_partition.evaluate (print_stats, &nentity, &entity_wgt, &ncuts, &cut_wgt, &nboundary, &nadj);
314 std::cout <<
" Information returned from the Zoltan evaluate function:"<<std::endl;
315 std::cout <<
" Error Code: :"<<ierr <<std::endl;
316 std::cout <<
" Number of entities: :"<<nentity <<std::endl;
317 std::cout <<
" Number of cuts: :"<<ncuts <<std::endl;
318 std::cout <<
" Cut Weight: :"<<cut_wgt <<std::endl;
319 std::cout <<
" Number on Boundary: :"<<nboundary <<std::endl;
320 std::cout <<
" Number Adjancent: :"<<nadj <<std::endl;
323 const stk_classic::mesh::EntityRank side_rank = fmeta.
side_rank();
324 const stk_classic::mesh::EntityRank elem_rank = fmeta.
element_rank();
325 const mesh::Entity *s = bulk_data.get_entity(side_rank,7);
327 const mesh::Entity & side = *s;
328 if (p_rank == side.owner_rank()) {
330 for ( ; iElem.first != iElem.second; ++iElem.first ) {
331 const mesh::Entity & elem = *iElem.first->entity();
332 const unsigned elemProc = elem.owner_rank();
333 if (elemProc!=p_rank) {
334 std::cout <<p_rank<<
" Good: Found element of of side 7 not owned." 335 <<
" Element "<<elemProc
336 <<
" on processor "<<p_rank
337 <<
" owned by "<<elemProc<<std::endl;
345 const double imbalance_threshold = 1.5;
346 bool do_rebal = imbalance_threshold < stk_classic::rebalance::check_balance(bulk_data, NULL, element_rank);
349 std::cout << std::endl
350 <<
"Use Case 4: imbalance_threshold after rebalance 1 = " << imbalance_threshold <<
", "<<do_rebal << std::endl;
353 stk_classic::rebalance::use_cases::GreedySideset greedy_sideset(comm, surfaces, bulk_data);
358 do_rebal = imbalance_threshold < stk_classic::rebalance::check_balance(bulk_data, NULL, element_rank);
361 std::cout << std::endl
362 <<
"Use Case 4: imbalance_threshold after rebalance 2 = " << imbalance_threshold <<
", "<<do_rebal << std::endl;
365 const stk_classic::mesh::EntityRank side_rank = fmeta.
side_rank();
366 const stk_classic::mesh::EntityRank elem_rank = fmeta.
element_rank();
367 mesh::Entity *s = bulk_data.get_entity(side_rank,7);
369 mesh::Entity & side = *s;
370 if (p_rank == side.owner_rank()) {
372 for ( ; iElem.first != iElem.second; ++iElem.first ) {
373 const mesh::Entity & elem = *iElem.first->entity();
374 const unsigned elemProc = elem.owner_rank();
375 if (elemProc!=p_rank) {
376 std::cerr <<p_rank<<
" Error: Found element of of side 7 not owned:"<<elemProc<<std::endl;
378 ThrowRequireMsg(elemProc==p_rank,
379 "Use case 4 error check failed. Found element of of side 7 not owned.");
385 bool result = !do_rebal ;
389 stk_classic::mesh::EntityVector entities;
393 result &= verify_dependent_ownership(element_rank, entities);
395 result &= verify_dependent_ownership(element_rank, entities);
bool rebalance(mesh::BulkData &bulk_data, const mesh::Selector &selector, const VectorField *coord_ref, const ScalarField *elem_weight_ref, Partition &partition, const stk_classic::mesh::EntityRank rank=stk_classic::mesh::InvalidEntityRank)
Rebalance with a Partition object.
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Entity & declare_element(BulkData &mesh, Part &part, const EntityId elem_id, const EntityId node_id[])
Declare an element member of a Part with a CellTopology and nodes conformal to that topology...
void all_reduce_sum(ParallelMachine comm, const double *local, double *global, unsigned count)
Parallel summation to all processors.
This is a class for selecting buckets based on a set of meshparts and set logic.
field_type & put_field(field_type &field, EntityRank entity_rank, const Part &part, const void *init_value=NULL)
Declare a field to exist for a given entity type and Part.
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
An application-defined subset of a problem domain.
void get_selected_entities(const Selector &selector, const std::vector< Bucket * > &input_buckets, std::vector< Entity * > &entities)
Get entities in selected buckets (selected by the given selector instance), and sorted by ID...
unsigned parallel_machine_rank(ParallelMachine parallel_machine)
Member function parallel_machine_rank ...
Manager for an integrated collection of entities, entity relations, and buckets of field data...
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
For partitioning of mesh entities over a processing grid.
Static functions for dynamic load balancing.
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.