Sierra Toolkit  Version of the Day
FieldParallel.hpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 
10 #ifndef stk_mesh_FieldParallel_hpp
11 #define stk_mesh_FieldParallel_hpp
12 
13 //----------------------------------------------------------------------
14 
15 #include <stk_util/util/SimpleArrayOps.hpp>
16 #include <stk_util/parallel/Parallel.hpp>
17 #include <stk_util/parallel/ParallelComm.hpp>
18 
19 #include <stk_mesh/base/Types.hpp>
20 #include <stk_mesh/base/Field.hpp>
21 #include <stk_mesh/base/Entity.hpp>
22 #include <stk_mesh/base/BulkData.hpp>
23 
24 namespace stk_classic {
25 namespace mesh {
26 
37 void communicate_field_data(
38  ParallelMachine machine,
39  const std::vector<EntityProc> & domain ,
40  const std::vector<EntityProc> & range ,
41  const std::vector< const FieldBase *> & fields );
42 
43 void communicate_field_data(
44  const Ghosting & ghosts ,
45  const std::vector< const FieldBase *> & fields );
46 
48 void communicate_field_data(
49  const BulkData & mesh ,
50  const unsigned field_count ,
51  const FieldBase * fields[] ,
52  CommAll & sparse );
53 
54 void communicate_field_data_verify_read( CommAll & );
55 
56 //----------------------------------------------------------------------
57 
58 namespace {
59 
60 //----------------------------------------------------------------------
69 template< class OpField >
70 void parallel_reduce( const BulkData & mesh ,
71  const OpField & op )
72 {
73  const FieldBase * fields[1] = { & op.field };
74 
75  CommAll sparse ;
76 
77  communicate_field_data( mesh, 1, fields, sparse );
78 
79  op( mesh.entity_comm() , sparse );
80 
81  // For debugging:
82  // communicate_field_data_verify_read( sparse );
83 }
84 
91 template< class OpField1 , class OpField2 >
92 void parallel_reduce( const BulkData & mesh ,
93  const OpField1 & op1 ,
94  const OpField2 & op2 )
95 {
96  const FieldBase * fields[2] = { & op1.field , & op2.field };
97 
98  CommAll sparse ;
99 
100  communicate_field_data( mesh, 2, fields, sparse );
101 
102  op1( mesh.entity_comm() , sparse );
103  op2( mesh.entity_comm() , sparse );
104 
105  // For debugging:
106  // communicate_field_data_verify_read( sparse );
107 }
108 
109 //----------------------------------------------------------------------
110 
111 template< class ReduceOp ,
112  class Type , class Tag1, class Tag2, class Tag3 ,
113  class Tag4 , class Tag5, class Tag6, class Tag7 >
114 struct ParallelReduceField {
115  typedef Field<Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7> field_type ;
116 
117  const field_type & field ;
118 
119  ParallelReduceField( const field_type & f ) : field(f) {}
120  ParallelReduceField( const ParallelReduceField & p ) : field(p.field) {}
121 
122  void operator()( const std::vector<Entity*> & entity_comm ,
123  CommAll & sparse ) const ;
124 
125 private:
126  ParallelReduceField & operator = ( const ParallelReduceField & );
127 };
128 
129 template< class ReduceOp ,
130  class Type , class Tag1, class Tag2, class Tag3 ,
131  class Tag4 , class Tag5, class Tag6, class Tag7 >
132 void ParallelReduceField< ReduceOp , Type , Tag1, Tag2, Tag3 ,
133  Tag4 , Tag5, Tag6, Tag7 >::
134  operator()( const std::vector<Entity*> & entity_comm ,
135  CommAll & sparse ) const
136 {
137  typedef EntityArray< field_type > array_type ;
138 
139  for ( std::vector<Entity*>::const_iterator
140  i = entity_comm.begin(); i != entity_comm.end() ; ++i ) {
141  Entity & entity = **i ;
142  array_type array( field , entity );
143  Type * const ptr_beg = array.contiguous_data();
144  Type * const ptr_end = ptr_beg + array.size();
145 
146  if (ptr_beg == NULL || ptr_end == NULL) continue;
147 
148  for ( PairIterEntityComm
149  ec = entity.comm() ; ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
150 
151  CommBuffer & b = sparse.recv_buffer( ec->proc );
152 
153  for ( Type * ptr = ptr_beg ; ptr < ptr_end ; ++ptr ) {
154  Type tmp ;
155  b.template unpack<unsigned char>( (unsigned char *)(&tmp), sizeof(Type) );
156  ReduceOp( ptr , & tmp );
157  }
158  }
159  }
160 }
161 
162 }
163 
164 //----------------------------------------------------------------------
165 
166 template< class Type , class Tag1, class Tag2, class Tag3 ,
167  class Tag4 , class Tag5, class Tag6, class Tag7 >
168 ParallelReduceField<Sum<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>
169 inline
170 sum( const Field<Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7> & f )
171 {
172  return ParallelReduceField<Sum<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>( f );
173 }
174 
175 template< class Type , class Tag1, class Tag2, class Tag3 ,
176  class Tag4 , class Tag5, class Tag6, class Tag7 >
177 ParallelReduceField<Max<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>
178 inline
179 max( const Field<Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7> & f )
180 {
181  return ParallelReduceField<Max<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>( f );
182 }
183 
184 template< class Type , class Tag1, class Tag2, class Tag3 ,
185  class Tag4 , class Tag5, class Tag6, class Tag7 >
186 ParallelReduceField<Min<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>
187 inline
188 min( const Field<Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7> & f )
189 {
190  return ParallelReduceField<Min<1>,Type,Tag1,Tag2,Tag3,Tag4,Tag5,Tag6,Tag7>( f );
191 }
192 
193 } // namespace mesh
194 } // namespace stk_classic
195 
196 #endif
197 
PairIterEntityComm comm() const
Complete communicaiton list for this entity.
Definition: Entity.hpp:181
Sierra Toolkit.
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
PairIter< std::vector< EntityCommInfo >::const_iterator > PairIterEntityComm
Span of ( communication-subset-ordinal , process-rank ) pairs for the communication of an entity...
Definition: Types.hpp:128