Line data Source code
1 : /*----------------------------------------------------------------------------*/
2 : /* CP2K: A general program to perform molecular dynamics simulations */
3 : /* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4 : /* */
5 : /* SPDX-License-Identifier: BSD-3-Clause */
6 : /*----------------------------------------------------------------------------*/
7 : #include "dbm_library.h"
8 : #include "../mpiwrap/cp_mpi.h"
9 : #include "../offload/offload_library.h"
10 : #include "../offload/offload_mempool.h"
11 :
12 : #include <assert.h>
13 : #include <inttypes.h>
14 : #include <omp.h>
15 : #include <stdio.h>
16 : #include <stdlib.h>
17 : #include <string.h>
18 :
19 : #define DBM_LIBRARY_PRINT(FN, MSG, OUTPUT_UNIT) \
20 : ((FN)(MSG, (int)strlen(MSG), OUTPUT_UNIT))
21 :
22 : static int64_t **per_thread_counters = NULL;
23 : static double phase_seconds[DBM_NUM_PHASES] = {0};
24 : static int64_t phase_bytes[DBM_NUM_PHASES] = {0};
25 : static bool library_initialized = false;
26 : static int max_threads = 0;
27 :
28 : #if !defined(_OPENMP)
29 : #error "OpenMP is required. Please add -fopenmp to your C compiler flags."
30 : #endif
31 :
32 : /*******************************************************************************
33 : * \brief Initializes the DBM library.
34 : * \author Ole Schuett
35 : ******************************************************************************/
36 1476 : void dbm_library_init(void) {
37 1476 : assert(omp_get_num_threads() == 1);
38 :
39 1476 : if (library_initialized) {
40 0 : fprintf(stderr, "DBM library was already initialized.\n");
41 0 : abort();
42 : }
43 :
44 1476 : offload_init();
45 :
46 1476 : max_threads = omp_get_max_threads();
47 1476 : memset(phase_seconds, 0, sizeof(phase_seconds));
48 1476 : memset(phase_bytes, 0, sizeof(phase_bytes));
49 1476 : per_thread_counters = malloc(max_threads * sizeof(int64_t *));
50 1476 : assert(per_thread_counters != NULL);
51 :
52 : // Using parallel regions to ensure memory is allocated near a thread's core.
53 : #pragma omp parallel default(none) shared(per_thread_counters) \
54 : num_threads(max_threads)
55 : {
56 : const int ithread = omp_get_thread_num();
57 : const size_t counters_size = DBM_NUM_COUNTERS * sizeof(int64_t);
58 : per_thread_counters[ithread] = malloc(counters_size);
59 : assert(per_thread_counters[ithread] != NULL);
60 : memset(per_thread_counters[ithread], 0, counters_size);
61 : }
62 :
63 1476 : library_initialized = true;
64 1476 : }
65 :
66 : /*******************************************************************************
67 : * \brief Finalizes the DBM library.
68 : * \author Ole Schuett
69 : ******************************************************************************/
70 1476 : void dbm_library_finalize(void) {
71 1476 : assert(omp_get_num_threads() == 1);
72 :
73 1476 : if (!library_initialized) {
74 0 : fprintf(stderr, "Error: DBM library is not initialized.\n");
75 0 : abort();
76 : }
77 :
78 2952 : for (int i = 0; i < max_threads; i++) {
79 1476 : free(per_thread_counters[i]);
80 : }
81 1476 : free(per_thread_counters);
82 1476 : per_thread_counters = NULL;
83 :
84 1476 : offload_mempool_clear();
85 1476 : library_initialized = false;
86 1476 : }
87 :
88 : /*******************************************************************************
89 : * \brief Add the calling thread's counts to the stats. This routine is
90 : * thread-safe.
91 : * \author Hans Pabst
92 : ******************************************************************************/
93 384922 : void dbm_library_counters_add(const int64_t counters[DBM_NUM_COUNTERS]) {
94 384922 : const int ithread = omp_get_thread_num();
95 384922 : assert(ithread < max_threads);
96 25019930 : for (int i = 0; i < DBM_NUM_COUNTERS; i++) {
97 24635008 : per_thread_counters[ithread][i] += counters[i];
98 : }
99 384922 : }
100 :
101 : /*******************************************************************************
102 : * \brief Add the durations (seconds) of a multiplication's phases to the
103 : * stats. Called once per dbm_multiply outside of parallel regions.
104 : * \author Hans Pabst
105 : ******************************************************************************/
106 349938 : void dbm_library_phases_add(const double seconds[DBM_NUM_PHASES],
107 : const int64_t bytes[DBM_NUM_PHASES]) {
108 349938 : assert(omp_get_num_threads() == 1);
109 3149442 : for (int i = 0; i < DBM_NUM_PHASES; i++) {
110 2799504 : phase_seconds[i] += seconds[i];
111 2799504 : phase_bytes[i] += bytes[i];
112 : }
113 349938 : }
114 :
115 : /*******************************************************************************
116 : * \brief Comperator passed to qsort to compare two counters.
117 : * \author Ole Schuett
118 : ******************************************************************************/
119 2176620 : static int compare_counters(const void *a, const void *b) {
120 : // Descending order: the difference of two counts may overflow int.
121 2176620 : const int64_t x = *(const int64_t *)a, y = *(const int64_t *)b;
122 2176620 : return (x < y) - (x > y);
123 : }
124 :
125 : /*******************************************************************************
126 : * \brief Prints statistics gathered by the DBM library.
127 : * \author Ole Schuett
128 : ******************************************************************************/
129 11308 : void dbm_library_print_stats(const int fortran_comm,
130 : void (*print_func)(const char *, int, int),
131 : const int output_unit) {
132 11308 : assert(omp_get_num_threads() == 1);
133 :
134 11308 : if (!library_initialized) {
135 0 : fprintf(stderr, "Error: DBM library is not initialized.\n");
136 0 : abort();
137 : }
138 :
139 11308 : const cp_mpi_comm_t comm = cp_mpi_comm_f2c(fortran_comm);
140 : // Sum all counters across threads and mpi ranks.
141 11308 : int64_t counters[DBM_NUM_COUNTERS][2] = {{0}};
142 11308 : double total = 0.0;
143 735020 : for (int i = 0; i < DBM_NUM_COUNTERS; i++) {
144 723712 : counters[i][1] = i; // needed as inverse index after qsort
145 1447424 : for (int j = 0; j < max_threads; j++) {
146 723712 : counters[i][0] += per_thread_counters[j][i];
147 : }
148 723712 : cp_mpi_sum_int64(&counters[i][0], 1, comm);
149 723712 : total += counters[i][0];
150 : }
151 :
152 : // Phases per rank: minimum, average, and maximum over ranks (last: total).
153 11308 : double phase_min[DBM_NUM_PHASES + 1], phase_avg[DBM_NUM_PHASES + 1];
154 11308 : double phase_max[DBM_NUM_PHASES + 1];
155 11308 : phase_max[DBM_NUM_PHASES] = 0.0;
156 101772 : for (int i = 0; i < DBM_NUM_PHASES; i++) {
157 90464 : phase_max[i] = phase_seconds[i];
158 90464 : phase_max[DBM_NUM_PHASES] += phase_seconds[i];
159 : }
160 113080 : for (int i = 0; i <= DBM_NUM_PHASES; i++) {
161 101772 : phase_min[i] = -phase_max[i]; // minimum as maximum of the negated values
162 101772 : phase_avg[i] = phase_max[i];
163 : }
164 11308 : cp_mpi_max_double(phase_max, DBM_NUM_PHASES + 1, comm);
165 11308 : cp_mpi_max_double(phase_min, DBM_NUM_PHASES + 1, comm);
166 11308 : cp_mpi_sum_double(phase_avg, DBM_NUM_PHASES + 1, comm);
167 11308 : int64_t bytes[DBM_NUM_PHASES];
168 11308 : memcpy(bytes, phase_bytes, sizeof(bytes));
169 11308 : cp_mpi_sum_int64(bytes, DBM_NUM_PHASES, comm);
170 11308 : const int nranks = cp_mpi_comm_size(comm);
171 :
172 : // Sort counters.
173 11308 : qsort(counters, DBM_NUM_COUNTERS, 2 * sizeof(int64_t), &compare_counters);
174 :
175 : // Determine if anything needs to be printed.
176 11308 : bool print = false;
177 701630 : for (int i = 0; i < DBM_NUM_COUNTERS && !print; i++) {
178 690322 : if (counters[i][0] != 0) {
179 530 : print = true;
180 : }
181 : }
182 11308 : if (!print) {
183 10778 : return; // nothing to be printed
184 : }
185 :
186 : // Print counters.
187 530 : DBM_LIBRARY_PRINT(print_func, "\n", output_unit);
188 530 : DBM_LIBRARY_PRINT(
189 : print_func,
190 : " ----------------------------------------------------------------"
191 : "---------------\n",
192 : output_unit);
193 530 : DBM_LIBRARY_PRINT(
194 : print_func,
195 : " - "
196 : " -\n",
197 : output_unit);
198 530 : DBM_LIBRARY_PRINT(
199 : print_func,
200 : " - DBM STATISTICS "
201 : " -\n",
202 : output_unit);
203 530 : DBM_LIBRARY_PRINT(
204 : print_func,
205 : " - "
206 : " -\n",
207 : output_unit);
208 530 : DBM_LIBRARY_PRINT(
209 : print_func,
210 : " ----------------------------------------------------------------"
211 : "---------------\n",
212 : output_unit);
213 530 : DBM_LIBRARY_PRINT(
214 : print_func,
215 : " M x N x K "
216 : "COUNT PERCENT\n",
217 : output_unit);
218 :
219 530 : const char *labels[] = {"?", "??", "???", ">999"};
220 530 : char buffer[100];
221 34450 : for (int i = 0; i < DBM_NUM_COUNTERS; i++) {
222 33920 : if (counters[i][0] == 0) {
223 28848 : continue; // skip empty counters
224 : }
225 5072 : const double percent = 100.0 * counters[i][0] / total;
226 5072 : const int idx = counters[i][1];
227 5072 : const int m = (idx % 64) / 16;
228 5072 : const int n = (idx % 16) / 4;
229 5072 : const int k = (idx % 4) / 1;
230 5072 : snprintf(buffer, sizeof(buffer),
231 : " %4s x %4s x %4s %46" PRId64 " %10.2f%%\n", labels[m],
232 : labels[n], labels[k], counters[i][0], percent);
233 5072 : DBM_LIBRARY_PRINT(print_func, buffer, output_unit);
234 : }
235 :
236 : // Print phases, i.e., where a rank spends the time of dbm_multiply.
237 530 : if (0.0 < phase_max[DBM_NUM_PHASES]) {
238 530 : const char *const phases[] = {"setup", "pack", "alltoall",
239 : "sort", "shift", "upload",
240 : "multiply", "finish", "total"};
241 530 : DBM_LIBRARY_PRINT(
242 : print_func,
243 : " ----------------------------------------------------------------"
244 : "---------------\n",
245 : output_unit);
246 530 : snprintf(buffer, sizeof(buffer), " %-24s %12s %12s %12s %12s\n",
247 : "PHASE PER RANK", "MIN [s]", "AVG [s]", "MAX [s]", "GB/s");
248 530 : DBM_LIBRARY_PRINT(print_func, buffer, output_unit);
249 5300 : for (int i = 0; i <= DBM_NUM_PHASES; i++) {
250 : // Bytes sent over the time all ranks spent in the phase.
251 9540 : const bool rate =
252 4770 : (i < DBM_NUM_PHASES && 0 < bytes[i] && 0.0 < phase_avg[i]);
253 4770 : const int n = snprintf(
254 4770 : buffer, sizeof(buffer), " %-24s %12.3f %12.3f %12.3f", phases[i],
255 4770 : -phase_min[i], phase_avg[i] / nranks, phase_max[i]);
256 4770 : if (rate) {
257 948 : snprintf(buffer + n, sizeof(buffer) - n, " %12.1f\n",
258 948 : 1E-9 * bytes[i] / phase_avg[i]);
259 : } else {
260 3822 : snprintf(buffer + n, sizeof(buffer) - n, "\n");
261 : }
262 4770 : DBM_LIBRARY_PRINT(print_func, buffer, output_unit);
263 : }
264 : }
265 :
266 530 : DBM_LIBRARY_PRINT(
267 : print_func,
268 : " ----------------------------------------------------------------"
269 : "---------------\n",
270 : output_unit);
271 : }
272 :
273 : // EOF
|
