chombo-discharge
Loading...
Searching...
No Matches
CD_ParallelOpsImplem.H
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: 2021-2026 SINTEF Energy Research
3 *
4 * SPDX-License-Identifier: GPL-3.0-or-later
5 */
6
13#ifndef CD_PARALLELOPSIMPLEM_H
14#define CD_PARALLELOPSIMPLEM_H
15
16// Std includes
17#include <limits>
18
19// Chombo includes
20#include <SPMD.H>
21
22// Our includes
23#include <CD_NamespaceHeader.H>
24
25inline void
26ParallelOps::barrier() noexcept
27{
28 CH_TIME("ParallelOps::barrier");
29
30#ifdef CH_MPI
31 MPI_Barrier(Chombo_MPI::comm);
32#endif
33}
34
35inline std::pair<int, int>
36ParallelOps::partition(const int a_N) noexcept
37{
38 CH_TIME("ParallelOps::partition");
39
40#ifdef CH_MPI
41 const int equalChunk = a_N / numProc();
42 const int remainder = a_N % numProc();
43
44 int begin = procID() * equalChunk;
45 int end = begin + equalChunk - 1;
46
47 // Remainder goes to last rank.
48 if (procID() == numProc() - 1) {
49 end += remainder;
50 }
51
52 return std::make_pair(begin, end);
53#else
54 return std::make_pair(0, a_N - 1);
55#endif
56}
57
58inline Real
59ParallelOps::min(const Real& a_input) noexcept
60{
61 CH_TIME("ParallelOps::min(Real)");
62
63#ifdef CH_MPI
64 Real ret = a_input;
65
66 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_CH_REAL, MPI_MIN, Chombo_MPI::comm);
67
68 if (result != MPI_SUCCESS) {
69 MayDay::Error("In file ParallelOps::min -- MPI communication error");
70 }
71
72 return ret;
73#else
74 return a_input;
75#endif
76}
77
78inline int
79ParallelOps::min(const int& a_input) noexcept
80{
81 CH_TIME("ParallelOps::min(int)");
82
83#ifdef CH_MPI
84 int ret = a_input;
85
86 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_INT, MPI_MIN, Chombo_MPI::comm);
87
88 if (result != MPI_SUCCESS) {
89 MayDay::Error("In file ParallelOps::min -- MPI communication error");
90 }
91 return ret;
92#else
93 return a_input;
94#endif
95}
96
97inline long long int
98ParallelOps::min(const long long int& a_input) noexcept
99{
100 CH_TIME("ParallelOps::min(long long int)");
101
102#ifdef CH_MPI
103 long long int ret = a_input;
104
105 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_LONG_LONG, MPI_MIN, Chombo_MPI::comm);
106
107 if (result != MPI_SUCCESS) {
108 MayDay::Error("In file ParallelOps::min -- MPI communication error");
109 }
110 return ret;
111#else
112 return a_input;
113#endif
114}
115
116inline std::pair<Real, RealVect>
117ParallelOps::min(const Real& a_val, const RealVect& a_pos) noexcept
118{
119 CH_TIME("ParallelOps::min(Real, RealVect)");
120
121#ifdef CH_MPI
122 const int nProc = numProc();
123
124 Real d = a_val;
125 Real* D = new Real[nProc];
126
127 Real x = a_pos[0];
128 Real* X = new Real[nProc];
129
130 Real y = a_pos[1];
131 Real* Y = new Real[nProc];
132
133#if CH_SPACEDIM == 3
134 Real z = a_pos[2];
135 Real* Z = new Real[nProc];
136#endif
137
138 MPI_Allgather(&d, 1, MPI_CH_REAL, D, 1, MPI_CH_REAL, Chombo_MPI::comm);
139 MPI_Allgather(&x, 1, MPI_CH_REAL, X, 1, MPI_CH_REAL, Chombo_MPI::comm);
140 MPI_Allgather(&y, 1, MPI_CH_REAL, Y, 1, MPI_CH_REAL, Chombo_MPI::comm);
141#if CH_SPACEDIM == 3
142 MPI_Allgather(&z, 1, MPI_CH_REAL, Z, 1, MPI_CH_REAL, Chombo_MPI::comm);
143#endif
144
145 Real dRet = D[0];
146 Real xRet = X[0];
147 Real yRet = Y[0];
148#if CH_SPACEDIM == 3
149 Real zRet = Z[0];
150#endif
151
152 for (int i = 1; i < nProc; i++) {
153 if (D[i] < dRet) {
154 dRet = D[i];
155 xRet = X[i];
156 yRet = Y[i];
157#if CH_SPACEDIM == 3
158 zRet = Z[i];
159#endif
160 }
161 }
162
163 delete[] D;
164 delete[] X;
165 delete[] Y;
166#if CH_SPACEDIM == 3
167 delete[] Z;
168#endif
169
170 return std::make_pair(dRet, RealVect(D_DECL(xRet, yRet, zRet)));
171#else
172 return std::make_pair(a_val, a_pos);
173#endif
174}
175
176inline Real
177ParallelOps::max(const Real& a_input) noexcept
178{
179 CH_TIME("ParallelOps::max(Real)");
180
181#ifdef CH_MPI
182 Real ret = a_input;
183
184 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_CH_REAL, MPI_MAX, Chombo_MPI::comm);
185 if (result != MPI_SUCCESS) {
186 MayDay::Error("In file ParallelOps::max -- MPI communication error");
187 }
188
189 return ret;
190#else
191 return a_input;
192#endif
193}
194
195inline int
196ParallelOps::max(const int& a_input) noexcept
197{
198 CH_TIME("ParallelOps::max(int)");
199
200#ifdef CH_MPI
201 int ret = a_input;
202
203 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_INT, MPI_MAX, Chombo_MPI::comm);
204
205 if (result != MPI_SUCCESS) {
206 MayDay::Error("In file ParallelOps::max -- MPI communication error");
207 }
208 return ret;
209#else
210 return a_input;
211#endif
212}
213
214inline long long
215ParallelOps::max(const long long& a_input) noexcept
216{
217 CH_TIME("ParallelOps::max(long long)");
218
219#ifdef CH_MPI
220 long long ret = a_input;
221
222 const int result = MPI_Allreduce(MPI_IN_PLACE, &ret, 1, MPI_LONG_LONG, MPI_MAX, Chombo_MPI::comm);
223
224 if (result != MPI_SUCCESS) {
225 MayDay::Error("In file ParallelOps::max -- MPI communication error");
226 }
227 return ret;
228#else
229 return a_input;
230#endif
231}
232
233inline std::pair<Real, RealVect>
234ParallelOps::max(const Real& a_val, const RealVect& a_pos) noexcept
235{
236 CH_TIME("ParallelOps::max(Real, RealVect)");
237
238#ifdef CH_MPI
239 const int nProc = numProc();
240
241 Real d = a_val;
242 Real* D = new Real[nProc];
243
244 Real x = a_pos[0];
245 Real* X = new Real[nProc];
246
247 Real y = a_pos[1];
248 Real* Y = new Real[nProc];
249
250#if CH_SPACEDIM == 3
251 Real z = a_pos[2];
252 Real* Z = new Real[nProc];
253#endif
254
255 MPI_Allgather(&d, 1, MPI_CH_REAL, D, 1, MPI_CH_REAL, Chombo_MPI::comm);
256 MPI_Allgather(&x, 1, MPI_CH_REAL, X, 1, MPI_CH_REAL, Chombo_MPI::comm);
257 MPI_Allgather(&y, 1, MPI_CH_REAL, Y, 1, MPI_CH_REAL, Chombo_MPI::comm);
258#if CH_SPACEDIM == 3
259 MPI_Allgather(&z, 1, MPI_CH_REAL, Z, 1, MPI_CH_REAL, Chombo_MPI::comm);
260#endif
261
262 Real dRet = D[0];
263 Real xRet = X[0];
264 Real yRet = Y[0];
265#if CH_SPACEDIM == 3
266 Real zRet = Z[0];
267#endif
268
269 for (int i = 1; i < nProc; i++) {
270 if (D[i] > dRet) {
271 dRet = D[i];
272 xRet = X[i];
273 yRet = Y[i];
274#if CH_SPACEDIM == 3
275 zRet = Z[i];
276#endif
277 }
278 }
279
280 delete[] D;
281 delete[] X;
282 delete[] Y;
283#if CH_SPACEDIM == 3
284 delete[] Z;
285#endif
286
287 return std::make_pair(dRet, RealVect(D_DECL(xRet, yRet, zRet)));
288#else
289 return std::make_pair(a_val, a_pos);
290#endif
291}
292
293inline std::pair<Real, int>
294ParallelOps::maxRank(const Real& a_val) noexcept
295{
296 CH_TIME("ParallelOps::maxRank(Real)");
297
298#ifdef CH_MPI
299 const int nProc = numProc();
300
301 Real* values = new Real[nProc];
302
303 MPI_Allgather(&a_val, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
304
305 int irank = -1;
306 Real maxVal = -std::numeric_limits<Real>::max();
307
308 for (int i = 0; i < nProc; i++) {
309 if (values[i] > maxVal) {
310 irank = i;
311 maxVal = values[i];
312 }
313 }
314
315 delete[] values;
316
317 return std::make_pair(maxVal, irank);
318#else
319 return std::make_pair(a_val, 0);
320#endif
321}
322
323inline std::pair<Real, int>
324ParallelOps::minRank(const Real& a_val) noexcept
325{
326 CH_TIME("ParallelOps::minRank(Real)");
327
328#ifdef CH_MPI
329 const int nProc = numProc();
330
331 Real* values = new Real[nProc];
332
333 MPI_Allgather(&a_val, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
334
335 int irank = -1;
336 Real maxVal = std::numeric_limits<Real>::max();
337
338 for (int i = 0; i < nProc; i++) {
339 if (values[i] < maxVal) {
340 irank = i;
341 maxVal = values[i];
342 }
343 }
344
345 delete[] values;
346
347 return std::make_pair(maxVal, irank);
348#else
349 return std::make_pair(a_val, 0);
350#endif
351}
352
353inline Real
354ParallelOps::sum(const Real& a_value) noexcept
355{
356 CH_TIME("ParallelOps::sum(Real)");
357
358#ifdef CH_MPI
359 Real ret = 0.0;
360
361 const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_CH_REAL, MPI_SUM, Chombo_MPI::comm);
362 if (result != MPI_SUCCESS) {
363 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
364 }
365
366 return ret;
367#else
368 return a_value;
369#endif
370}
371
372inline int
373ParallelOps::sum(const int& a_value) noexcept
374{
375 CH_TIME("ParallelOps::sum(Real)");
376
377#ifdef CH_MPI
378 int ret = 0;
379
380 const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_INT, MPI_SUM, Chombo_MPI::comm);
381 if (result != MPI_SUCCESS) {
382 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
383 }
384
385 return ret;
386#else
387 return a_value;
388#endif
389}
390
391inline long
392ParallelOps::sum(const long& a_value) noexcept
393{
394 CH_TIME("ParallelOps::sum(long)");
395
396#ifdef CH_MPI
397 long ret = 0;
398
399 const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_LONG, MPI_SUM, Chombo_MPI::comm);
400 if (result != MPI_SUCCESS) {
401 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
402 }
403
404 return ret;
405#else
406 return a_value;
407#endif
408}
409
410inline long long
411ParallelOps::sum(const long long& a_value) noexcept
412{
413 CH_TIME("ParallelOps::sum(long long)");
414
415#ifdef CH_MPI
416 long long ret = 0;
417
418 const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_LONG_LONG, MPI_SUM, Chombo_MPI::comm);
419 if (result != MPI_SUCCESS) {
420 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
421 }
422
423 return ret;
424#else
425 return a_value;
426#endif
427}
428
429inline unsigned long long
430ParallelOps::sum(const unsigned long long& a_value) noexcept
431{
432 CH_TIME("ParallelOps::sum(unsigned long long)");
433
434#ifdef CH_MPI
435 unsigned long long ret = 0;
436
437 const int result = MPI_Allreduce(&a_value, &ret, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, Chombo_MPI::comm);
438 if (result != MPI_SUCCESS) {
439 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
440 }
441
442 return ret;
443#else
444 return a_value;
445#endif
446}
447
448inline void
449ParallelOps::sum(Vector<Real>& a_data) noexcept
450{
451 CH_TIME("ParallelOps::sum(Real)");
452
453#ifdef CH_MPI
454 const int result = MPI_Allreduce(MPI_IN_PLACE,
455 a_data.stdVector().data(),
456 a_data.size(),
457 MPI_CH_REAL,
458 MPI_SUM,
459 Chombo_MPI::comm);
460 if (result != MPI_SUCCESS) {
461 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
462 }
463#endif
464}
465
466inline void
467ParallelOps::sum(Vector<int>& a_data) noexcept
468{
469 CH_TIME("ParallelOps::sum(int)");
470
471#ifdef CH_MPI
472 const int result = MPI_Allreduce(MPI_IN_PLACE,
473 a_data.stdVector().data(),
474 a_data.size(),
475 MPI_INT,
476 MPI_SUM,
477 Chombo_MPI::comm);
478 if (result != MPI_SUCCESS) {
479 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
480 }
481#endif
482}
483
484inline void
485ParallelOps::sum(Vector<long int>& a_data) noexcept
486{
487 CH_TIME("ParallelOps::sum(long int)");
488
489#ifdef CH_MPI
490 const int result = MPI_Allreduce(MPI_IN_PLACE,
491 a_data.stdVector().data(),
492 a_data.size(),
493 MPI_LONG,
494 MPI_SUM,
495 Chombo_MPI::comm);
496 if (result != MPI_SUCCESS) {
497 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
498 }
499#endif
500}
501
502inline void
503ParallelOps::sum(Vector<long long int>& a_data) noexcept
504{
505 CH_TIME("ParallelOps::sum(long long int)");
506
507#ifdef CH_MPI
508 const int result = MPI_Allreduce(MPI_IN_PLACE,
509 a_data.stdVector().data(),
510 a_data.size(),
511 MPI_LONG_LONG,
512 MPI_SUM,
513 Chombo_MPI::comm);
514 if (result != MPI_SUCCESS) {
515 MayDay::Error("In file ParallelOps::sum -- MPI communication error");
516 }
517#endif
518}
519
520inline Real
521ParallelOps::average(const Real& a_val) noexcept
522{
523 CH_TIME("ParallelOps::average(Real)");
524
525#ifdef CH_MPI
526 return ParallelOps::sum(a_val) / numProc();
527#else
528 return a_val;
529#endif
530}
531
532inline Real
533ParallelOps::standardDeviation(const Real& a_value) noexcept
534{
535 CH_TIME("ParallelOps::standardDeviation(Real)");
536
537#ifdef CH_MPI
538 const int nProc = numProc();
539
540 const Real average = ParallelOps::average(a_value);
541
542 Real* values = new Real[nProc];
543
544 MPI_Allgather(&a_value, 1, MPI_CH_REAL, values, 1, MPI_CH_REAL, Chombo_MPI::comm);
545
546 Real sigma = 0.0;
547 for (int i = 0; i < nProc; i++) {
548 sigma += std::pow(values[i] - average, 2);
549 }
550
551 delete[] values;
552
553 return sqrt(sigma / nProc);
554#else
555 return 0.0;
556#endif
557}
558
559inline Vector<int>
560ParallelOps::gather(const int& a_localValue) noexcept
561{
562 CH_TIME("ParallelOps::gather(int)");
563
564#ifdef CH_MPI
565 Vector<int> ret(numProc());
566
567 MPI_Allgather(&a_localValue, 1, MPI_INT, ret.stdVector().data(), 1, MPI_INT, Chombo_MPI::comm);
568
569 return ret;
570#else
571 return Vector<int>(1, a_localValue);
572#endif
573}
574
575inline Vector<Real>
576ParallelOps::gather(const Real& a_localValue) noexcept
577{
578 CH_TIME("ParallelOps::gather(Real)");
579
580#ifdef CH_MPI
581 Vector<Real> ret(numProc());
582
583 MPI_Allgather(&a_localValue, 1, MPI_CH_REAL, ret.stdVector().data(), 1, MPI_CH_REAL, Chombo_MPI::comm);
584
585 return ret;
586#else
587 return Vector<Real>(1, a_localValue);
588#endif
589}
590
591inline Vector<long long>
592ParallelOps::gather(const long long& a_localValue) noexcept
593{
594 CH_TIME("ParallelOps::gather(long long)");
595
596#ifdef CH_MPI
597 Vector<long long> ret(numProc());
598
599 MPI_Allgather(&a_localValue, 1, MPI_LONG_LONG, ret.stdVector().data(), 1, MPI_LONG_LONG, Chombo_MPI::comm);
600
601 return ret;
602#else
603 return Vector<long long>(1, a_localValue);
604#endif
605}
606
607#include <CD_NamespaceFooter.H>
608
609#endif
ParallelOps::max
Real max(const Real &a_input) noexcept
Get the maximum of the input, reduced over MPI ranks (in the Chombo communicator)
Definition CD_ParallelOpsImplem.H:177
ParallelOps::maxRank
std::pair< Real, int > maxRank(const Real &a_val) noexcept
Get the maximum value and the rank having the maximum value.
Definition CD_ParallelOpsImplem.H:294
ParallelOps::average
Real average(const Real &a_val) noexcept
Compute the average (across MPI ranks) of the input value.
Definition CD_ParallelOpsImplem.H:521
ParallelOps::standardDeviation
Real standardDeviation(const Real &a_value) noexcept
Compute the standard deviation of the input value.
Definition CD_ParallelOpsImplem.H:533
ParallelOps::gather
Vector< int > gather(const int &a_localValue) noexcept
Gather local values – return a vector of local values.
Definition CD_ParallelOpsImplem.H:560
ParallelOps::minRank
std::pair< Real, int > minRank(const Real &a_val) noexcept
Get the minimum value and the rank having the minimum value.
Definition CD_ParallelOpsImplem.H:324
ParallelOps::min
Real min(const Real &a_input) noexcept
Get the minimum of the input, reduced over MPI ranks (in the Chombo communicator)
Definition CD_ParallelOpsImplem.H:59
ParallelOps::sum
Real sum(const Real &a_value) noexcept
Compute the sum across all MPI ranks.
Definition CD_ParallelOpsImplem.H:354
ParallelOps::partition
std::pair< int, int > partition(int a_N) noexcept
Partition the input number such that each MPI rank gets an equal chunk. We assume that N is the lengt...
Definition CD_ParallelOpsImplem.H:36
ParallelOps::barrier
void barrier() noexcept
MPI barrier.
Definition CD_ParallelOpsImplem.H:26
Generated by doxygen 1.9.8