chombo-discharge
CD_ParallelOpsImplem.H
Go to the documentation of this file.
1 /* chombo-discharge
2  * Copyright © 2021 SINTEF Energy Research.
3  * Please refer to Copyright.txt and LICENSE in the chombo-discharge root directory.
4  */
5 
12 #ifndef CD_ParallelOpsImplem_H
13 #define CD_ParallelOpsImplem_H
14 
15 // Std includes
16 #include <limits>
17 
18 // Chombo includes
19 #include <SPMD.H>
20 
21 // Our includes
22 #include <CD_NamespaceHeader.H>
23 
24 inline void
25 ParallelOps::barrier() noexcept
26 {
27  CH_TIME("ParallelOps::barrier");
28 
29 #ifdef CH_MPI
30  MPI_Barrier(Chombo_MPI::comm);
31 #endif
32 }
33 
34 inline std::pair<int, int>
35 ParallelOps::partition(const int a_N) noexcept
36 {
37  CH_TIME("ParallelOps::partition");
38 
39 #ifdef CH_MPI
40  const int equalChunk = a_N / numProc();
41  const int remainder = a_N % numProc();
42 
43  int begin = procID() * equalChunk;
44  int end = begin + equalChunk - 1;
45 
46  // Remainder goes to last rank.
47  if (procID() == numProc() - 1) {
48  end += remainder;
49  }
50 
51  return std::make_pair(begin, end);
52 #else
53  return std::make_pair(0, a_N - 1);
54 #endif
55 }
56 
57 inline Real
58 ParallelOps::min(const Real& a_input) noexcept
59 {
60  CH_TIME("ParallelOps::min(Real)");
61 
62 #ifdef CH_MPI
63  Real ret = a_input;
64 
65  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_CH_REAL, MPI_MIN, Chombo_MPI::comm);
66 
67  if (result != MPI_SUCCESS) {
68  MayDay::Error("In file ParallelOps::min -- MPI communication error");
69  }
70 
71  return ret;
72 #else
73  return a_input;
74 #endif
75 }
76 
77 inline int
78 ParallelOps::min(const int& a_input) noexcept
79 {
80  CH_TIME("ParallelOps::min(int)");
81 
82 #ifdef CH_MPI
83  int ret = a_input;
84 
85  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_INT, MPI_MIN, Chombo_MPI::comm);
86 
87  if (result != MPI_SUCCESS) {
88  MayDay::Error("In file ParallelOps::min -- MPI communication error");
89  }
90  return ret;
91 #else
92  return a_input;
93 #endif
94 }
95 
96 inline long long int
97 ParallelOps::min(const long long int& a_input) noexcept
98 {
99  CH_TIME("ParallelOps::min(long long int)");
100 
101 #ifdef CH_MPI
102  long long int ret = a_input;
103 
104  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_LONG_LONG, MPI_MIN, Chombo_MPI::comm);
105 
106  if (result != MPI_SUCCESS) {
107  MayDay::Error("In file ParallelOps::min -- MPI communication error");
108  }
109  return ret;
110 #else
111  return a_input;
112 #endif
113 }
114 
115 inline std::pair<Real, RealVect>
116 ParallelOps::min(const Real& a_val, const RealVect& a_pos) noexcept
117 {
118  CH_TIME("ParallelOps::min(Real, RealVect)");
119 
120 #ifdef CH_MPI
121  const int nProc = numProc();
122 
123  Real d = a_val;
124  Real* D = new Real[nProc];
125 
126  Real x = a_pos[0];
127  Real* X = new Real[nProc];
128 
129  Real y = a_pos[1];
130  Real* Y = new Real[nProc];
131 
132 #if CH_SPACEDIM == 3
133  Real z = a_pos[2];
134  Real* Z = new Real[nProc];
135 #endif
136 
137  MPI_Allgather(&d, 1, MPI_CH_REAL, D, 1, MPI_CH_REAL, Chombo_MPI::comm);
138  MPI_Allgather(&x, 1, MPI_CH_REAL, X, 1, MPI_CH_REAL, Chombo_MPI::comm);
139  MPI_Allgather(&y, 1, MPI_CH_REAL, Y, 1, MPI_CH_REAL, Chombo_MPI::comm);
140 #if CH_SPACEDIM == 3
141  MPI_Allgather(&z, 1, MPI_CH_REAL, Z, 1, MPI_CH_REAL, Chombo_MPI::comm);
142 #endif
143 
144  Real dRet = D[0];
145  Real xRet = X[0];
146  Real yRet = Y[0];
147 #if CH_SPACEDIM == 3
148  Real zRet = Z[0];
149 #endif
150 
151  for (int i = 1; i < nProc; i++) {
152  if (D[i] < dRet) {
153  dRet = D[i];
154  xRet = X[i];
155  yRet = Y[i];
156 #if CH_SPACEDIM == 3
157  zRet = Z[i];
158 #endif
159  }
160  }
161 
162  delete[] D;
163  delete[] X;
164  delete[] Y;
165 #if CH_SPACEDIM == 3
166  delete[] Z;
167 #endif
168 
169  return std::make_pair(dRet, RealVect(D_DECL(xRet, yRet, zRet)));
170 #else
171  return std::make_pair(a_val, a_pos);
172 #endif
173 }
174 
175 inline Real
176 ParallelOps::max(const Real& a_input) noexcept
177 {
178  CH_TIME("ParallelOps::max(Real)");
179 
180 #ifdef CH_MPI
181  Real ret = a_input;
182 
183  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_CH_REAL, MPI_MAX, Chombo_MPI::comm);
184  if (result != MPI_SUCCESS) {
185  MayDay::Error("In file ParallelOps::max -- MPI communication error");
186  }
187 
188  return ret;
189 #else
190  return a_input;
191 #endif
192 }
193 
194 inline int
195 ParallelOps::max(const int& a_input) noexcept
196 {
197  CH_TIME("ParallelOps::max(int)");
198 
199 #ifdef CH_MPI
200  int ret = a_input;
201 
202  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_INT, MPI_MAX, Chombo_MPI::comm);
203 
204  if (result != MPI_SUCCESS) {
205  MayDay::Error("In file ParallelOps::max -- MPI communication error");
206  }
207  return ret;
208 #else
209  return a_input;
210 #endif
211 }
212 
213 inline long long
214 ParallelOps::max(const long long& a_input) noexcept
215 {
216  CH_TIME("ParallelOps::max(long long)");
217 
218 #ifdef CH_MPI
219  long long ret = a_input;
220 
221  const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_LONG_LONG, MPI_MAX, Chombo_MPI::comm);
222 
223  if (result != MPI_SUCCESS) {
224  MayDay::Error("In file ParallelOps::max -- MPI communication error");
225  }
226  return ret;
227 #else
228  return a_input;
229 #endif
230 }
231 
232 inline std::pair<Real, RealVect>
233 ParallelOps::max(const Real& a_val, const RealVect& a_pos) noexcept
234 {
235  CH_TIME("ParallelOps::max(Real, RealVect)");
236 
237 #ifdef CH_MPI
238  const int nProc = numProc();
239 
240  Real d = a_val;
241  Real* D = new Real[nProc];
242 
243  Real x = a_pos[0];
244  Real* X = new Real[nProc];
245 
246  Real y = a_pos[1];
247  Real* Y = new Real[nProc];
248 
249 #if CH_SPACEDIM == 3
250  Real z = a_pos[2];
251  Real* Z = new Real[nProc];
252 #endif
253 
254  MPI_Allgather(&d, 1, MPI_CH_REAL, D, 1, MPI_CH_REAL, Chombo_MPI::comm);
255  MPI_Allgather(&x, 1, MPI_CH_REAL, X, 1, MPI_CH_REAL, Chombo_MPI::comm);
256  MPI_Allgather(&y, 1, MPI_CH_REAL, Y, 1, MPI_CH_REAL, Chombo_MPI::comm);
257 #if CH_SPACEDIM == 3
258  MPI_Allgather(&z, 1, MPI_CH_REAL, Z, 1, MPI_CH_REAL, Chombo_MPI::comm);
259 #endif
260 
261  Real dRet = D[0];
262  Real xRet = X[0];
263  Real yRet = Y[0];
264 #if CH_SPACEDIM == 3
265  Real zRet = Z[0];
266 #endif
267 
268  for (int i = 1; i < nProc; i++) {
269  if (D[i] > dRet) {
270  dRet = D[i];
271  xRet = X[i];
272  yRet = Y[i];
273 #if CH_SPACEDIM == 3
274  zRet = Z[i];
275 #endif
276  }
277  }
278 
279  delete[] D;
280  delete[] X;
281  delete[] Y;
282 #if CH_SPACEDIM == 3
283  delete[] Z;
284 #endif
285 
286  return std::make_pair(dRet, RealVect(D_DECL(xRet, yRet, zRet)));
287 #else
288  return std::make_pair(a_val, a_pos);
289 #endif
290 }
291 
292 inline std::pair<Real, int>
293 ParallelOps::maxRank(const Real& a_val) noexcept
294 {
295  CH_TIME("ParallelOps::maxRank(Real)");
296 
297 #ifdef CH_MPI
298  const int nProc = numProc();
299 
300  Real* values = new Real[nProc];
301 
302  MPI_Allgather(&a_val, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
303 
304  int irank = -1;
305  Real maxVal = -std::numeric_limits<Real>::max();
306 
307  for (int i = 0; i < nProc; i++) {
308  if (values[i] > maxVal) {
309  irank = i;
310  maxVal = values[i];
311  }
312  }
313 
314  delete[] values;
315 
316  return std::make_pair(maxVal, irank);
317 #else
318  return std::make_pair(a_val, 0);
319 #endif
320 }
321 
322 inline std::pair<Real, int>
323 ParallelOps::minRank(const Real& a_val) noexcept
324 {
325  CH_TIME("ParallelOps::minRank(Real)");
326 
327 #ifdef CH_MPI
328  const int nProc = numProc();
329 
330  Real* values = new Real[nProc];
331 
332  MPI_Allgather(&a_val, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
333 
334  int irank = -1;
335  Real maxVal = std::numeric_limits<Real>::max();
336 
337  for (int i = 0; i < nProc; i++) {
338  if (values[i] < maxVal) {
339  irank = i;
340  maxVal = values[i];
341  }
342  }
343 
344  delete[] values;
345 
346  return std::make_pair(maxVal, irank);
347 #else
348  return std::make_pair(a_val, 0);
349 #endif
350 }
351 
352 inline Real
353 ParallelOps::sum(const Real& a_value) noexcept
354 {
355  CH_TIME("ParallelOps::sum(Real)");
356 
357 #ifdef CH_MPI
358  Real ret = 0.0;
359 
360  const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_CH_REAL, MPI_SUM, Chombo_MPI::comm);
361  if (result != MPI_SUCCESS) {
362  MayDay::Error("In file ParallelOps::sum -- MPI communication error");
363  }
364 
365  return ret;
366 #else
367  return a_value;
368 #endif
369 }
370 
371 inline int
372 ParallelOps::sum(const int& a_value) noexcept
373 {
374  CH_TIME("ParallelOps::sum(Real)");
375 
376 #ifdef CH_MPI
377  int ret = 0;
378 
379  const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_INT, MPI_SUM, Chombo_MPI::comm);
380  if (result != MPI_SUCCESS) {
381  MayDay::Error("In file ParallelOps::sum -- MPI communication error");
382  }
383 
384  return ret;
385 #else
386  return a_value;
387 #endif
388 }
389 
390 inline long
391 ParallelOps::sum(const long& a_value) noexcept
392 {
393  CH_TIME("ParallelOps::sum(long)");
394 
395 #ifdef CH_MPI
396  long ret = 0;
397 
398  const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_LONG, MPI_SUM, Chombo_MPI::comm);
399  if (result != MPI_SUCCESS) {
400  MayDay::Error("In file ParallelOps::sum -- MPI communication error");
401  }
402 
403  return ret;
404 #else
405  return a_value;
406 #endif
407 }
408 
409 inline long long
410 ParallelOps::sum(const long long& a_value) noexcept
411 {
412  CH_TIME("ParallelOps::sum(long long)");
413 
414 #ifdef CH_MPI
415  long long ret = 0;
416 
417  const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_LONG_LONG, MPI_SUM, Chombo_MPI::comm);
418  if (result != MPI_SUCCESS) {
419  MayDay::Error("In file ParallelOps::sum -- MPI communication error");
420  }
421 
422  return ret;
423 #else
424  return a_value;
425 #endif
426 }
427 
428 inline unsigned long long
429 ParallelOps::sum(const unsigned long long& a_value) noexcept
430 {
431  CH_TIME("ParallelOps::sum(unsigned long long)");
432 
433 #ifdef CH_MPI
434  unsigned long long ret = 0;
435 
436  const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, Chombo_MPI::comm);
437  if (result != MPI_SUCCESS) {
438  MayDay::Error("In file ParallelOps::sum -- MPI communication error");
439  }
440 
441  return ret;
442 #else
443  return a_value;
444 #endif
445 }
446 
447 inline void
448 ParallelOps::vectorSum(Vector<Real>& a_data) noexcept
449 {
450  CH_TIME("ParallelOps::vectorSum(Real)");
451 
452 #ifdef CH_MPI
453  const int result = MPI_Allreduce(MPI_IN_PLACE, &(a_data[0]), a_data.size(), MPI_CH_REAL, MPI_SUM, Chombo_MPI::comm);
454  if (result != MPI_SUCCESS) {
455  MayDay::Error("In file ParallelOps::vectorSum -- MPI communication error");
456  }
457 #endif
458 }
459 
460 inline void
461 ParallelOps::vectorSum(Vector<int>& a_data) noexcept
462 {
463  CH_TIME("ParallelOps::vectorSum(int)");
464 
465 #ifdef CH_MPI
466  const int result = MPI_Allreduce(MPI_IN_PLACE, &(a_data[0]), a_data.size(), MPI_INT, MPI_SUM, Chombo_MPI::comm);
467  if (result != MPI_SUCCESS) {
468  MayDay::Error("In file ParallelOps::vectorSum -- MPI communication error");
469  }
470 #endif
471 }
472 
473 inline void
474 ParallelOps::vectorSum(Vector<long int>& a_data) noexcept
475 {
476  CH_TIME("ParallelOps::vectorSum(long int)");
477 
478 #ifdef CH_MPI
479  const int result = MPI_Allreduce(MPI_IN_PLACE, &(a_data[0]), a_data.size(), MPI_LONG, MPI_SUM, Chombo_MPI::comm);
480  if (result != MPI_SUCCESS) {
481  MayDay::Error("In file ParallelOps::vectorSum -- MPI communication error");
482  }
483 #endif
484 }
485 
486 inline void
487 ParallelOps::vectorSum(Vector<long long int>& a_data) noexcept
488 {
489  CH_TIME("ParallelOps::vectorSum(long long int)");
490 
491 #ifdef CH_MPI
492  const int result = MPI_Allreduce(MPI_IN_PLACE, &(a_data[0]), a_data.size(), MPI_LONG_LONG, MPI_SUM, Chombo_MPI::comm);
493  if (result != MPI_SUCCESS) {
494  MayDay::Error("In file ParallelOps::vectorSum -- MPI communication error");
495  }
496 #endif
497 }
498 
499 inline Real
500 ParallelOps::average(const Real& a_val) noexcept
501 {
502  CH_TIME("ParallelOps::average(Real)");
503 
504 #ifdef CH_MPI
505  return ParallelOps::sum(a_val) / numProc();
506 #else
507  return a_val;
508 #endif
509 }
510 
511 inline Real
512 ParallelOps::standardDeviation(const Real& a_value) noexcept
513 {
514  CH_TIME("ParallelOps::standardDeviation(Real)");
515 
516 #ifdef CH_MPI
517  const int nProc = numProc();
518 
519  const Real average = ParallelOps::average(a_value);
520 
521  Real* values = new Real[nProc];
522 
523  MPI_Allgather(&a_value, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
524 
525  Real sigma = 0.0;
526  for (int i = 0; i < nProc; i++) {
527  sigma += std::pow(values[i] - average, 2);
528  }
529 
530  delete[] values;
531 
532  return sqrt(sigma / nProc);
533 #else
534  return 0.0;
535 #endif
536 }
537 
538 #include <CD_NamespaceFooter.H>
539 
540 #endif
ParallelOps::max
Real max(const Real &a_input) noexcept
Get the maximum of the input, reduced over MPI ranks (in the Chombo communicator)
Definition: CD_ParallelOpsImplem.H:176
ParallelOps::maxRank
std::pair< Real, int > maxRank(const Real &a_val) noexcept
Get the maximum value and the rank having the maximum value.
Definition: CD_ParallelOpsImplem.H:293
ParallelOps::average
Real average(const Real &a_val) noexcept
Compute the average (across MPI ranks) of the input value.
Definition: CD_ParallelOpsImplem.H:500
ParallelOps::standardDeviation
Real standardDeviation(const Real &a_value) noexcept
Compute the standard deviation of the input value.
Definition: CD_ParallelOpsImplem.H:512
ParallelOps::minRank
std::pair< Real, int > minRank(const Real &a_val) noexcept
Get the minimum value and the rank having the minimum value.
Definition: CD_ParallelOpsImplem.H:323
ParallelOps::vectorSum
void vectorSum(Vector< Real > &a_data) noexcept
Perform a summation of all the MPI ranks's input data.
Definition: CD_ParallelOpsImplem.H:448
ParallelOps::min
Real min(const Real &a_input) noexcept
Get the minimum of the input, reduced over MPI ranks (in the Chombo communicator)
Definition: CD_ParallelOpsImplem.H:58
ParallelOps::sum
Real sum(const Real &a_value) noexcept
Compute the sum across all MPI ranks.
Definition: CD_ParallelOpsImplem.H:353
ParallelOps::barrier
void barrier() noexcept
MPI barrier.
Definition: CD_ParallelOpsImplem.H:25
ParallelOps::partition
std::pair< int, int > partition(const int a_N) noexcept
Partition the input number such that each MPI rank gets an equal chunk. We assume that N is the lengt...
Definition: CD_ParallelOpsImplem.H:35
Generated by doxygen 1.9.1