| Geant4 Cross Reference |
1 // Copyright (C) 2010, Guy Barrand. All rights 1 // Copyright (C) 2010, Guy Barrand. All rights reserved.
2 // See the file tools.license for terms. 2 // See the file tools.license for terms.
3 3
4 #ifndef toolx_mpi_hmpi 4 #ifndef toolx_mpi_hmpi
5 #define toolx_mpi_hmpi 5 #define toolx_mpi_hmpi
6 6
7 // code to send, receive histos through MPI. 7 // code to send, receive histos through MPI.
8 8
9 #include <tools/histo/hmpi> 9 #include <tools/histo/hmpi>
10 #include <tools/histo/hd2mpi> 10 #include <tools/histo/hd2mpi>
11 11
12 #include <tools/histo/h1d> 12 #include <tools/histo/h1d>
13 #include <tools/histo/h2d> 13 #include <tools/histo/h2d>
14 #include <tools/histo/h3d> 14 #include <tools/histo/h3d>
15 #include <tools/histo/p1d> 15 #include <tools/histo/p1d>
16 #include <tools/histo/p2d> 16 #include <tools/histo/p2d>
17 17
18 #include "wrmpi" 18 #include "wrmpi"
19 19
20 namespace toolx { 20 namespace toolx {
21 namespace mpi { 21 namespace mpi {
22 22
23 class hmpi : public virtual tools::histo::hmpi 23 class hmpi : public virtual tools::histo::hmpi {
24 typedef tools::histo::hmpi parent; 24 typedef tools::histo::hmpi parent;
25 protected: 25 protected:
26 typedef unsigned int num_t; 26 typedef unsigned int num_t;
27 static const std::string& s_class() { 27 static const std::string& s_class() {
28 static const std::string s_v("toolx::mpi:: 28 static const std::string s_v("toolx::mpi::hmpi");
29 return s_v; 29 return s_v;
30 } 30 }
31 public: 31 public:
32 virtual bool pack(const tools::histo::h1d& a 32 virtual bool pack(const tools::histo::h1d& a_h) {
33 if(!m_wrmpi.spack(a_h.s_cls())) return fal 33 if(!m_wrmpi.spack(a_h.s_cls())) return false;
34 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac 34 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac())) return false;
35 return true; 35 return true;
36 } 36 }
37 virtual bool pack(const tools::histo::h2d& a 37 virtual bool pack(const tools::histo::h2d& a_h) {
38 if(!m_wrmpi.spack(a_h.s_cls())) return fal 38 if(!m_wrmpi.spack(a_h.s_cls())) return false;
39 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac 39 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac())) return false;
40 return true; 40 return true;
41 } 41 }
42 virtual bool pack(const tools::histo::h3d& a 42 virtual bool pack(const tools::histo::h3d& a_h) {
43 if(!m_wrmpi.spack(a_h.s_cls())) return fal 43 if(!m_wrmpi.spack(a_h.s_cls())) return false;
44 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac 44 if(!histo_data_duiuid_pack(m_wrmpi,a_h.dac())) return false;
45 return true; 45 return true;
46 } 46 }
47 virtual bool pack(const tools::histo::p1d& a 47 virtual bool pack(const tools::histo::p1d& a_h) {
48 if(!m_wrmpi.spack(a_h.s_cls())) return fal 48 if(!m_wrmpi.spack(a_h.s_cls())) return false;
49 if(!profile_data_duiuidd_pack(m_wrmpi,a_h. 49 if(!profile_data_duiuidd_pack(m_wrmpi,a_h.get_histo_data())) return false;
50 return true; 50 return true;
51 } 51 }
52 virtual bool pack(const tools::histo::p2d& a 52 virtual bool pack(const tools::histo::p2d& a_h) {
53 if(!m_wrmpi.spack(a_h.s_cls())) return fal 53 if(!m_wrmpi.spack(a_h.s_cls())) return false;
54 if(!profile_data_duiuidd_pack(m_wrmpi,a_h. 54 if(!profile_data_duiuidd_pack(m_wrmpi,a_h.get_histo_data())) return false;
55 return true; 55 return true;
56 } 56 }
57 public: 57 public:
58 virtual bool beg_send(unsigned int a_nhist) 58 virtual bool beg_send(unsigned int a_nhist) {
59 m_wrmpi.pack_reset(); 59 m_wrmpi.pack_reset();
60 return m_wrmpi.pack(a_nhist); 60 return m_wrmpi.pack(a_nhist);
61 } 61 }
62 virtual bool send(int a_dest) { 62 virtual bool send(int a_dest) {
63 if(::MPI_Send(m_wrmpi.buffer(),m_wrmpi.ipo 63 if(::MPI_Send(m_wrmpi.buffer(),m_wrmpi.ipos(),MPI_CHAR,a_dest,m_tag,m_comm)!=MPI_SUCCESS) {
64 m_out << "toolx::mpi::hmpi::send : rank 64 m_out << "toolx::mpi::hmpi::send : rank " << m_rank << " : MPI_Send failed." << std::endl;
65 return false; 65 return false;
66 } 66 }
67 m_wrmpi.pack_reset(); 67 m_wrmpi.pack_reset();
68 return true; 68 return true;
69 } 69 }
70 public: 70 public:
71 virtual bool wait_histos(int a_src,std::vect 71 virtual bool wait_histos(int a_src,std::vector< std::pair<std::string,void*> >& a_hists) {
72 a_hists.clear(); 72 a_hists.clear();
73 73
74 typedef std::pair<std::string,void*> class 74 typedef std::pair<std::string,void*> class_pointer;
75 75
76 MPI_Status status; 76 MPI_Status status;
77 if(::MPI_Probe(a_src,m_tag,m_comm,&status) 77 if(::MPI_Probe(a_src,m_tag,m_comm,&status)!=MPI_SUCCESS) {
78 m_out << "toolx::mpi::hmpi::wait_histos 78 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : MPI_Probe : failed." << std::endl;
79 return false; 79 return false;
80 } 80 }
81 81
82 int buffer_size = 0; 82 int buffer_size = 0;
83 if(::MPI_Get_count(&status,MPI_CHAR,&buffe 83 if(::MPI_Get_count(&status,MPI_CHAR,&buffer_size)!=MPI_SUCCESS) {
84 m_out << "toolx::mpi::hmpi::wait_histos 84 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : MPI_Get_count : failed." << std::endl;
85 return false; 85 return false;
86 } 86 }
87 87
88 if(!buffer_size) { 88 if(!buffer_size) {
89 m_out << "exlb::mpi::wait_histos : MPI_G 89 m_out << "exlb::mpi::wait_histos : MPI_Get_count returns zero data." << std::endl;
90 return false; 90 return false;
91 } 91 }
92 92
93 if(m_verbose) m_out << "rank " << m_rank < 93 if(m_verbose) m_out << "rank " << m_rank << " : get_count " << buffer_size << std::endl;
94 94
95 char* buffer = new char[buffer_size]; 95 char* buffer = new char[buffer_size];
96 if(!buffer) { 96 if(!buffer) {
97 m_out << "toolx::mpi::hmpi::wait_histos 97 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : can't alloc buffer of size " << buffer_size << std::endl;
98 return false; 98 return false;
99 } 99 }
100 100
101 if(::MPI_Recv(buffer,buffer_size,MPI_CHAR, 101 if(::MPI_Recv(buffer,buffer_size,MPI_CHAR,a_src,m_tag,m_comm,&status)!=MPI_SUCCESS) {
102 m_out << "toolx::mpi::hmpi::wait_histos 102 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : MPI_Recv : failed." << std::endl;
103 delete [] buffer; 103 delete [] buffer;
104 return false; 104 return false;
105 } 105 }
106 106
107 if(m_verbose) m_out << "toolx::mpi::hmpi:: 107 if(m_verbose) m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : unpack data ..." << std::endl;
108 108
109 wrmpi _mpi(m_out,m_comm,buffer_size,buffer 109 wrmpi _mpi(m_out,m_comm,buffer_size,buffer); //give ownership of buffer to _mpi.
110 110
111 num_t nhist; 111 num_t nhist;
112 if(!_mpi.unpack(nhist)) return false; 112 if(!_mpi.unpack(nhist)) return false;
113 113
114 if(m_verbose) 114 if(m_verbose)
115 m_out << "toolx::mpi::hmpi::wait_histos 115 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank << " : number of histos to unpack " << nhist << std::endl;
116 116
117 {for(num_t ihist=0;ihist<nhist;ihist++) { 117 {for(num_t ihist=0;ihist<nhist;ihist++) {
118 118
119 std::string scls; 119 std::string scls;
120 if(!_mpi.sunpack(scls)) return false; 120 if(!_mpi.sunpack(scls)) return false;
121 121
122 if(scls==tools::histo::h1d::s_class()) { 122 if(scls==tools::histo::h1d::s_class()) {
123 tools::histo::histo_data<double,unsign 123 tools::histo::histo_data<double,unsigned int,unsigned int,double> hdata;
124 if(!histo_data_duiuid_unpack(_mpi,hdat 124 if(!histo_data_duiuid_unpack(_mpi,hdata)) return false;
125 tools::histo::h1d* h = new tools::hist 125 tools::histo::h1d* h = new tools::histo::h1d("",10,0,1);
126 h->copy_from_data(hdata); 126 h->copy_from_data(hdata);
127 if(m_verbose) { 127 if(m_verbose) {
128 m_out << "toolx::mpi::hmpi::wait_his 128 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
129 << " : got a " << scls 129 << " : got a " << scls
130 << ", title " << h->title() 130 << ", title " << h->title()
131 << ", mean_x " << h->mean() << 131 << ", mean_x " << h->mean() << ", rms " << h->rms()
132 << std::endl; 132 << std::endl;
133 } 133 }
134 a_hists.push_back(class_pointer(h->s_c 134 a_hists.push_back(class_pointer(h->s_cls(),h)); //give ownership of h.
135 135
136 } else if(scls==tools::histo::h2d::s_cla 136 } else if(scls==tools::histo::h2d::s_class()) {
137 tools::histo::histo_data<double,unsign 137 tools::histo::histo_data<double,unsigned int,unsigned int,double> hdata;
138 if(!histo_data_duiuid_unpack(_mpi,hdat 138 if(!histo_data_duiuid_unpack(_mpi,hdata)) return false;
139 tools::histo::h2d* h = new tools::hist 139 tools::histo::h2d* h = new tools::histo::h2d("",10,0,1,10,0,1);
140 h->copy_from_data(hdata); 140 h->copy_from_data(hdata);
141 if(m_verbose) { 141 if(m_verbose) {
142 m_out << "toolx::mpi::hmpi::wait_his 142 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
143 << " : got a " << scls 143 << " : got a " << scls
144 << ", title " << h->title() 144 << ", title " << h->title()
145 << ", mean_x " << h->mean_x() 145 << ", mean_x " << h->mean_x() << ", rms_x " << h->rms_x()
146 << ", mean_y " << h->mean_y() 146 << ", mean_y " << h->mean_y() << ", rms_y " << h->rms_y()
147 << std::endl; 147 << std::endl;
148 } 148 }
149 a_hists.push_back(class_pointer(h->s_c 149 a_hists.push_back(class_pointer(h->s_cls(),h)); //give ownership of h.
150 150
151 } else if(scls==tools::histo::h3d::s_cla 151 } else if(scls==tools::histo::h3d::s_class()) {
152 tools::histo::histo_data<double,unsign 152 tools::histo::histo_data<double,unsigned int,unsigned int,double> hdata;
153 if(!histo_data_duiuid_unpack(_mpi,hdat 153 if(!histo_data_duiuid_unpack(_mpi,hdata)) return false;
154 tools::histo::h3d* h = new tools::hist 154 tools::histo::h3d* h = new tools::histo::h3d("",10,0,1,10,0,1,10,0,1);
155 h->copy_from_data(hdata); 155 h->copy_from_data(hdata);
156 if(m_verbose) { 156 if(m_verbose) {
157 m_out << "toolx::mpi::hmpi::wait_his 157 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
158 << " : got a " << scls 158 << " : got a " << scls
159 << ", title " << h->title() 159 << ", title " << h->title()
160 << ", mean_x " << h->mean_x() 160 << ", mean_x " << h->mean_x() << ", rms_x " << h->rms_x()
161 << ", mean_y " << h->mean_y() 161 << ", mean_y " << h->mean_y() << ", rms_y " << h->rms_y()
162 << ", mean_z " << h->mean_z() 162 << ", mean_z " << h->mean_z() << ", rms_z " << h->rms_z()
163 << std::endl; 163 << std::endl;
164 } 164 }
165 a_hists.push_back(class_pointer(h->s_c 165 a_hists.push_back(class_pointer(h->s_cls(),h)); //give ownership of h.
166 166
167 } else if(scls==tools::histo::p1d::s_cla 167 } else if(scls==tools::histo::p1d::s_class()) {
168 tools::histo::profile_data<double,unsi 168 tools::histo::profile_data<double,unsigned int,unsigned int,double,double> pdata;
169 if(!profile_data_duiuidd_unpack(_mpi,p 169 if(!profile_data_duiuidd_unpack(_mpi,pdata)) return false;
170 tools::histo::p1d* h = new tools::hist 170 tools::histo::p1d* h = new tools::histo::p1d("",10,0,1);
171 h->copy_from_data(pdata); 171 h->copy_from_data(pdata);
172 if(m_verbose) { 172 if(m_verbose) {
173 m_out << "toolx::mpi::hmpi::wait_his 173 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
174 << " : got a " << scls 174 << " : got a " << scls
175 << ", title " << h->title() 175 << ", title " << h->title()
176 << ", mean_x " << h->mean() << 176 << ", mean_x " << h->mean() << ", rms " << h->rms()
177 << std::endl; 177 << std::endl;
178 } 178 }
179 a_hists.push_back(class_pointer(h->s_c 179 a_hists.push_back(class_pointer(h->s_cls(),h)); //give ownership of h.
180 180
181 } else if(scls==tools::histo::p2d::s_cla 181 } else if(scls==tools::histo::p2d::s_class()) {
182 tools::histo::profile_data<double,unsi 182 tools::histo::profile_data<double,unsigned int,unsigned int,double,double> pdata;
183 if(!profile_data_duiuidd_unpack(_mpi,p 183 if(!profile_data_duiuidd_unpack(_mpi,pdata)) return false;
184 tools::histo::p2d* h = new tools::hist 184 tools::histo::p2d* h = new tools::histo::p2d("",10,0,1,10,0,1);
185 h->copy_from_data(pdata); 185 h->copy_from_data(pdata);
186 if(m_verbose) { 186 if(m_verbose) {
187 m_out << "toolx::mpi::hmpi::wait_his 187 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
188 << " : got a " << scls 188 << " : got a " << scls
189 << ", title " << h->title() 189 << ", title " << h->title()
190 << ", mean_x " << h->mean_x() 190 << ", mean_x " << h->mean_x() << ", rms_x " << h->rms_x()
191 << ", mean_y " << h->mean_y() 191 << ", mean_y " << h->mean_y() << ", rms_y " << h->rms_y()
192 << std::endl; 192 << std::endl;
193 } 193 }
194 a_hists.push_back(class_pointer(h->s_c 194 a_hists.push_back(class_pointer(h->s_cls(),h)); //give ownership of h.
195 195
196 } else { 196 } else {
197 m_out << "toolx::mpi::hmpi::wait_histo 197 m_out << "toolx::mpi::hmpi::wait_histos : rank " << m_rank
198 << " : got not treated class " < 198 << " : got not treated class " << scls
199 << std::endl; 199 << std::endl;
200 } 200 }
201 201
202 }} //ihist 202 }} //ihist
203 203
204 return true; 204 return true;
205 } 205 }
206 206
207 public: 207 public:
208 virtual int rank() const { return m_rank;} 208 virtual int rank() const { return m_rank;}
209 virtual bool comm_rank(int& a_rank) const { 209 virtual bool comm_rank(int& a_rank) const {
210 if(::MPI_Comm_rank(m_comm,&a_rank)!=MPI_SU 210 if(::MPI_Comm_rank(m_comm,&a_rank)!=MPI_SUCCESS) {a_rank=-1;return false;}
211 return true; 211 return true;
212 } 212 }
213 virtual bool comm_size(int& a_size) const { 213 virtual bool comm_size(int& a_size) const {
214 if(::MPI_Comm_size(m_comm,&a_size)!=MPI_SU 214 if(::MPI_Comm_size(m_comm,&a_size)!=MPI_SUCCESS) {a_size=0;return false;}
215 return true; 215 return true;
216 } 216 }
217 public: 217 public:
218 hmpi(std::ostream& a_out,int a_rank,int a_ta 218 hmpi(std::ostream& a_out,int a_rank,int a_tag,const MPI_Comm& a_comm,bool a_verbose = false)
219 :m_out(a_out) 219 :m_out(a_out)
220 ,m_rank(a_rank) 220 ,m_rank(a_rank)
221 ,m_tag(a_tag) 221 ,m_tag(a_tag)
222 ,m_comm(a_comm) 222 ,m_comm(a_comm)
223 ,m_verbose(a_verbose) 223 ,m_verbose(a_verbose)
224 ,m_wrmpi(a_out,a_comm) 224 ,m_wrmpi(a_out,a_comm)
225 { 225 {
226 #ifdef TOOLS_MEM 226 #ifdef TOOLS_MEM
227 tools::mem::increment(s_class().c_str()); 227 tools::mem::increment(s_class().c_str());
228 #endif 228 #endif
229 } 229 }
230 virtual ~hmpi(){ 230 virtual ~hmpi(){
231 #ifdef TOOLS_MEM 231 #ifdef TOOLS_MEM
232 tools::mem::decrement(s_class().c_str()); 232 tools::mem::decrement(s_class().c_str());
233 #endif 233 #endif
234 } 234 }
235 protected: 235 protected:
236 hmpi(const hmpi& a_from) 236 hmpi(const hmpi& a_from)
237 :parent(a_from) 237 :parent(a_from)
238 ,m_out(a_from.m_out) 238 ,m_out(a_from.m_out)
239 ,m_rank(a_from.m_rank) 239 ,m_rank(a_from.m_rank)
240 ,m_tag(a_from.m_tag) 240 ,m_tag(a_from.m_tag)
241 ,m_comm(a_from.m_comm) 241 ,m_comm(a_from.m_comm)
242 ,m_verbose(a_from.m_verbose) 242 ,m_verbose(a_from.m_verbose)
243 ,m_wrmpi(a_from.m_out,a_from.m_comm) 243 ,m_wrmpi(a_from.m_out,a_from.m_comm)
244 { 244 {
245 #ifdef TOOLS_MEM 245 #ifdef TOOLS_MEM
246 tools::mem::increment(s_class().c_str()); 246 tools::mem::increment(s_class().c_str());
247 #endif 247 #endif
248 } 248 }
249 hmpi& operator=(const hmpi& a_from){ 249 hmpi& operator=(const hmpi& a_from){
250 m_rank = a_from.m_rank; 250 m_rank = a_from.m_rank;
251 m_tag = a_from.m_tag; 251 m_tag = a_from.m_tag;
252 m_verbose = a_from.m_verbose; 252 m_verbose = a_from.m_verbose;
253 return *this; 253 return *this;
254 } 254 }
255 protected: 255 protected:
256 std::ostream& m_out; 256 std::ostream& m_out;
257 int m_rank; 257 int m_rank;
258 int m_tag; 258 int m_tag;
259 const MPI_Comm& m_comm; 259 const MPI_Comm& m_comm;
260 bool m_verbose; 260 bool m_verbose;
261 wrmpi m_wrmpi; 261 wrmpi m_wrmpi;
262 }; 262 };
263 263
264 }} 264 }}
265 265
266 #endif 266 #endif