20 #include <stk_util/parallel/ParallelComm.hpp> 21 #include <stk_util/parallel/ParallelReduce.hpp> 23 #include <stk_mesh/base/Ghosting.hpp> 24 #include <stk_mesh/base/BulkData.hpp> 25 #include <stk_mesh/base/MetaData.hpp> 26 #include <stk_mesh/base/FieldData.hpp> 27 #include <stk_mesh/base/EntityComm.hpp> 28 #include <stk_mesh/base/Comm.hpp> 35 bool verify_parallel_attributes( BulkData & M , std::ostream & error_log );
37 void pack_owned_verify( CommAll & all ,
const BulkData & mesh );
39 bool unpack_not_owned_verify( CommAll & comm_all ,
40 const BulkData & mesh ,
41 std::ostream & error_log );
45 bool comm_mesh_verify_parallel_consistency(
46 BulkData & M , std::ostream & error_log )
52 result = verify_parallel_attributes( M , error_log );
54 if (M.parallel_size() > 1) {
55 all_reduce( M.parallel() , ReduceMin<1>( & result ) );
61 CommAll all( M.parallel() );
63 pack_owned_verify( all , M );
65 all.allocate_buffers( all.parallel_size() / 4 );
67 pack_owned_verify( all , M );
71 result = unpack_not_owned_verify( all , M , error_log );
73 if (M.parallel_size() > 1) {
74 all_reduce( M.parallel() , ReduceMin<1>( & result ) );
83 bool ordered_comm(
const Entity & entity )
86 const size_t n = ec.size();
87 for (
size_t i = 1 ; i < n ; ++i ) {
88 if ( ! ( ec[i-1] < ec[i] ) ) {
95 bool verify_parallel_attributes( BulkData & M , std::ostream & error_log )
99 const MetaData & S = MetaData::get(M);
100 Part & owns_part = S.locally_owned_part();
101 Part & shares_part = S.globally_shared_part();
103 const unsigned p_rank = M.parallel_rank();
105 const size_t EntityRankEnd = MetaData::get(M).entity_rank_count();
107 size_t comm_count = 0 ;
109 for (
size_t itype = 0 ; itype < EntityRankEnd ; ++itype ) {
110 const std::vector< Bucket * > & all_buckets = M.buckets( itype );
112 const std::vector<Bucket*>::const_iterator i_end = all_buckets.end();
113 std::vector<Bucket*>::const_iterator i = all_buckets.begin();
115 while ( i != i_end ) {
116 Bucket & bucket = **i ; ++i ;
118 const bool has_owns_part =
has_superset( bucket , owns_part );
119 const bool has_shares_part =
has_superset( bucket , shares_part );
121 const Bucket::iterator j_end = bucket.end();
122 Bucket::iterator j = bucket.begin();
124 while ( j != j_end ) {
125 Entity & entity = *j ; ++j ;
127 bool this_result = true ;
130 const bool ordered = ordered_comm( entity );
131 const bool shares = in_shared( entity );
132 const bool recv_ghost = in_receive_ghost( entity );
133 const bool send_ghost = in_send_ghost( entity );
134 const bool owned_closure = in_owned_closure( entity , p_rank );
137 error_log <<
"Problem is unordered" << std::endl;
138 this_result = false ;
143 if ( has_owns_part && p_owner != p_rank ) {
144 error_log <<
"problem is owner-consistency check 1: " 145 <<
"has_owns_part: " << has_owns_part <<
", " 146 <<
"p_owner: " << p_owner <<
", " 147 <<
"p_rank: " << p_rank << std::endl;
148 this_result = false ;
151 if ( ! has_owns_part && p_owner == p_rank ) {
152 error_log <<
"problem is owner-consistency check 2: " 153 <<
"has_owns_part: " << has_owns_part <<
", " 154 <<
"p_owner: " << p_owner <<
", " 155 <<
"p_rank: " << p_rank << std::endl;
156 this_result = false ;
159 if ( has_shares_part != shares ) {
160 error_log <<
"problem is owner-consistency check 3: " 161 <<
"has_shares_part: " << has_shares_part <<
", " 162 <<
"shares: " << shares << std::endl;
163 this_result = false ;
168 if ( ( has_owns_part || has_shares_part ) != owned_closure ) {
169 error_log <<
"problem is closure check 1: " 170 <<
"has_owns_part: " << has_owns_part <<
", " 171 <<
"has_shares_part: " << has_shares_part <<
", " 172 <<
"owned_closure: " << owned_closure << std::endl;
173 this_result = false ;
178 if ( owned_closure && recv_ghost ) {
179 error_log <<
"problem is recv ghost check 1: " 180 <<
"owned_closure: " << owned_closure <<
", " 181 <<
"recv_ghost: " << recv_ghost << std::endl;
182 this_result = false ;
184 if ( ! owned_closure && ! recv_ghost ) {
185 error_log <<
"problem is recv ghost check 2: " 186 <<
"owned_closure: " << owned_closure <<
", " 187 <<
"recv_ghost: " << recv_ghost << std::endl;
188 this_result = false ;
193 if ( ! has_owns_part && send_ghost ) {
194 error_log <<
"problem is send ghost check 1: " 195 <<
"has_owns_part: " << has_owns_part <<
", " 196 <<
"send_ghost: " << send_ghost << std::endl;
197 this_result = false ;
202 if ( shares && p_owner != p_rank ) {
204 for ( ; ! ip.empty() && p_owner != ip->proc ; ++ip );
206 error_log <<
"problem is shared check 1" << std::endl;
207 this_result = false ;
211 if ( shares || recv_ghost || send_ghost ) { ++comm_count ; }
213 if ( ! this_result ) {
215 error_log <<
"P" << M.parallel_rank() <<
": " ;
216 print_entity_key( error_log , MetaData::get(M), entity.
key() );
217 error_log <<
" ERROR: owner(" << p_owner
218 <<
") owns(" << has_owns_part
219 <<
") shares(" << has_shares_part
220 <<
") owned_closure(" << owned_closure
221 <<
") recv_ghost(" << recv_ghost
222 <<
") send_ghost(" << send_ghost
225 for ( ; ! ip.empty() ; ++ip ) {
226 error_log <<
" " << ip->ghost_id <<
":" << ip->proc ;
228 error_log <<
" )" << std::endl ;
234 for ( std::vector<Entity*>::const_iterator
235 i = M.entity_comm().begin() ;
236 i != M.entity_comm().end() ; ++i ) {
241 print_entity_key( error_log , MetaData::get(M), (*i)->key() );
242 error_log <<
" ERROR: in entity_comm but has no comm info" << std::endl ;
247 if ( M.entity_comm().size() != comm_count ) {
248 error_log <<
" ERROR: entity_comm.size() = " << M.entity_comm().size();
249 error_log <<
" != " << comm_count <<
" = entities with comm info" ;
250 error_log << std::endl ;
260 void insert( std::vector<unsigned> & vec ,
unsigned val )
262 std::vector<unsigned>::iterator j =
263 std::lower_bound( vec.begin() , vec.end() , val );
264 if ( j == vec.end() || *j != val ) {
265 vec.insert( j , val );
269 void pack_owned_verify( CommAll & all ,
const BulkData & mesh )
271 const std::vector<Entity*> & entity_comm = mesh.entity_comm();
272 const unsigned p_rank = all.parallel_rank();
274 for ( std::vector<Entity*>::const_iterator
275 i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
277 Entity & entity = **i ;
281 std::vector<unsigned> share_proc ;
282 std::vector<unsigned> ghost_proc ;
286 for (
size_t j = 0 ; j < comm.size() ; ++j ) {
287 if ( comm[j].ghost_id == 0 ) {
289 share_proc.push_back( comm[j].proc );
293 insert( ghost_proc , comm[j].proc );
297 const unsigned share_count = share_proc.size();
299 for (
size_t j = 0 ; j < share_proc.size() ; ++j ) {
303 const unsigned p = share_proc[j] ;
305 CommBuffer & buf = all.send_buffer( p );
307 pack_entity_info( buf , entity );
309 buf.pack<
unsigned>( share_count );
314 for ( ; k < share_count && share_proc[k] < p_rank ; ++k ) {
315 if ( k != j ) { buf.pack<
unsigned>( share_proc[k] ); }
317 buf.pack<
unsigned>( p_rank );
318 for ( ; k < share_count ; ++k ) {
319 if ( k != j ) { buf.pack<
unsigned>( share_proc[k] ); }
323 for (
size_t j = 0 ; j < ghost_proc.size() ; ++j ) {
324 const unsigned p = ghost_proc[j] ;
326 CommBuffer & buf = all.send_buffer( p );
328 pack_entity_info( buf , entity );
332 for (
size_t k = 0 ; k < comm.size() ; ++k ) {
333 if ( comm[k].ghost_id != 0 && comm[k].proc == p ) {
337 buf.pack<
unsigned>(
count );
338 for (
size_t k = 0 ; k < comm.size() ; ++k ) {
339 if ( comm[k].ghost_id != 0 && comm[k].proc == p ) {
340 buf.pack<
unsigned>( comm[k].ghost_id );
351 bool unpack_not_owned_verify( CommAll & comm_all ,
352 const BulkData & mesh ,
353 std::ostream & error_log )
355 const MetaData & meta = MetaData::get(mesh);
356 Part *
const owns_part = & meta.locally_owned_part();
357 Part *
const shares_part = & meta.globally_shared_part();
358 const PartVector & mesh_parts = meta.get_parts();
359 const unsigned p_rank = mesh.parallel_rank();
360 const std::vector<Entity*> & entity_comm = mesh.entity_comm();
364 EntityKey recv_entity_key ;
365 unsigned recv_owner_rank = 0 ;
366 unsigned recv_comm_count = 0 ;
367 std::vector<Part*> recv_parts ;
368 std::vector<Relation> recv_relations ;
369 std::vector<unsigned> recv_comm ;
371 for ( std::vector<Entity*>::const_iterator
372 i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
374 Entity & entity = **i ;
378 const Bucket & bucket = entity.
bucket();
380 std::pair<const unsigned *,const unsigned *>
383 CommBuffer & buf = comm_all.recv_buffer( entity.
owner_rank() );
385 unpack_entity_info( buf , mesh ,
386 recv_entity_key , recv_owner_rank ,
387 recv_parts , recv_relations );
389 recv_comm_count = 0 ;
390 buf.unpack<
unsigned>( recv_comm_count );
391 recv_comm.resize( recv_comm_count );
392 buf.unpack<
unsigned>( & recv_comm[0] , recv_comm_count );
396 const bool bad_key = entity.
key() != recv_entity_key ;
397 const bool bad_own = entity.
owner_rank() != recv_owner_rank ;
398 bool bad_part = false ;
399 bool bad_rel = false ;
400 bool bad_comm = false ;
404 if ( ! bad_key && ! bad_own ) {
406 const unsigned ec_size = ec.size();
407 bad_comm = ec_size != recv_comm.size();
410 if ( in_shared( entity ) ) {
411 for ( ; j < ec_size &&
412 ec[j].ghost_id == 0 &&
413 ec[j].proc == recv_comm[j] ; ++j );
414 bad_comm = j != ec_size ;
417 for ( ; j < ec_size &&
418 ec[j].ghost_id == recv_comm[j] &&
420 bad_comm = j != ec_size ;
427 if ( ! bad_key && ! bad_own && ! bad_comm ) {
429 const unsigned * k = part_ordinals.first ;
431 std::vector<Part*>::iterator ip = recv_parts.begin();
433 for ( ; ! bad_part && ip != recv_parts.end() ; ++ip ) {
434 if ( owns_part != *ip ) {
435 if ( shares_part != *ip ) {
437 bad_part = k == part_ordinals.second ||
438 (*ip)->mesh_meta_data_ordinal() != *k ;
441 else if ( k != part_ordinals.second &&
442 *k == shares_part->mesh_meta_data_ordinal() ) {
451 if ( ! bad_key && ! bad_own && ! bad_comm && ! bad_part ) {
453 PairIterRelation ir = entity.
relations();
455 std::vector<Relation>::iterator jr = recv_relations.begin() ;
457 for ( ; ! bad_rel && jr != recv_relations.end() &&
458 jr->entity_rank() < entity.
entity_rank() ; ++jr , ++ir ) {
459 bad_rel = ir.empty() || *jr != *ir ;
464 if ( bad_key || bad_own || bad_comm || bad_part || bad_rel ) {
465 error_log <<
"P" << p_rank <<
": " ;
466 print_entity_key( error_log , meta, entity.
key() );
467 error_log <<
" owner(" << entity.
owner_rank() <<
")" ;
469 if ( bad_key || bad_own ) {
470 error_log <<
" != received " ;
471 print_entity_key( error_log , meta, recv_entity_key );
472 error_log <<
" owner(" << recv_owner_rank
473 <<
")" << std::endl ;
475 else if ( bad_comm ) {
477 if ( in_shared( entity ) ) {
478 error_log <<
" sharing(" ;
479 for (
size_t j = 0 ; j < ec.size() &&
480 ec[j].ghost_id == 0 ; ++j ) {
481 error_log <<
" " << ec[j].proc ;
483 error_log <<
" ) != received sharing(" ;
484 for (
size_t j = 0 ; j < recv_comm.size() ; ++j ) {
485 error_log <<
" " << recv_comm[j] ;
487 error_log <<
" )" << std::endl ;
490 error_log <<
" ghosting(" ;
491 for (
size_t j = 0 ; j < ec.size() ; ++j ) {
492 error_log <<
" (g" << ec[j].ghost_id ;
493 error_log <<
",p" << ec[j].proc ;
496 error_log <<
" ) != received ghosting(" ;
497 for (
size_t j = 0 ; j < recv_comm.size() ; ++j ) {
498 error_log <<
" (g" << recv_comm[j] ;
502 error_log <<
" )" << std::endl ;
505 else if ( bad_part ) {
506 error_log <<
" Parts( " ;
508 for (
const unsigned * k = part_ordinals.first ;
509 k < part_ordinals.second ; ++k ) {
510 error_log <<
" \"" << mesh_parts[ *k ]->name() <<
"\"" ;
512 error_log <<
" ) != received Parts( " ;
514 for ( std::vector<Part*>::iterator
515 ip = recv_parts.begin();
516 ip != recv_parts.end() ; ++ip ) {
517 error_log <<
" \"" << (*ip)->name() <<
"\"" ;
519 error_log <<
" )" << std::endl ;
521 else if ( bad_rel ) {
522 error_log <<
" Relations(" ;
523 PairIterRelation ir = entity.
relations();
524 for ( ; ! ir.empty() &&
525 ir->entity_rank() < entity.
entity_rank() ; ++ir ) {
526 error_log <<
" " << *ir ;
528 error_log <<
" ) != received Relations(" ;
529 std::vector<Relation>::iterator jr = recv_relations.begin() ;
530 for ( ; jr != recv_relations.end() &&
531 jr->entity_rank() < entity.
entity_rank() ; ++jr ) {
532 error_log <<
" " << *jr ;
534 error_log <<
" )" << std::endl ;
PairIterEntityComm comm() const
Complete communicaiton list for this entity.
bool has_superset(const Bucket &bucket, const unsigned &ordinal)
Is this bucket a subset of the given part by partID.
PairIterEntityComm sharing() const
Parallel processes which share this entity.
Bucket & bucket() const
The bucket which holds this mesh entity's field data.
const EntityKey & key() const
The globally unique key ( entity type + identifier ) of this entity.
std::pair< const unsigned *, const unsigned * > superset_part_ordinals() const
PairIterRelation relations() const
All Entity relations for which this entity is a member. The relations are ordered from lowest entity-...
EntityRank entity_rank() const
The rank of this entity.
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
PairIter< std::vector< EntityCommInfo >::const_iterator > PairIterEntityComm
Span of ( communication-subset-ordinal , process-rank ) pairs for the communication of an entity...
unsigned owner_rank() const
Parallel processor rank of the processor which owns this entity.
bool insert(PartVector &v, Part &part)
Insert a part into a properly ordered collection of parts. Returns true if this is a new insertion...
eastl::iterator_traits< InputIterator >::difference_type count(InputIterator first, InputIterator last, const T &value)