1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
/* -*- C -*-
*
* Copyright (c) 2008 Los Alamos National Security, LLC. All rights reserved.
*
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#include <stdio.h>
#include <stdbool.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <time.h>
#include <sys/time.h>
#include <mpi.h>
int main(int argc, char* argv[])
{
int msg;
int rank, size, my_twin;
int ppn, my_node;
struct timeval tv;
unsigned long my_timestamp[2];
long *timestamps;
int i, maxrank;
unsigned long maxsec, maxusec, minutes, seconds;
unsigned long start_sec, start_usec;
float fsecs;
int nnodes;
bool odd_nnodes;
bool recvit;
char *ppnstr;
if (argc < 3) {
fprintf(stderr, "start times must be provided\n");
return 1;
}
ppnstr = getenv("OMPI_COMM_WORLD_LOCAL_SIZE");
ppn = strtol(ppnstr, NULL, 10);
start_sec = strtol(argv[1], NULL, 10);
start_usec = strtol(argv[2], NULL, 10);
MPI_Init(NULL, NULL);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
/* this program requires that the size be an integer multiple of ppn */
if (0 != (size % ppn)) {
if (0 == rank) {
fprintf(stderr, "The number of procs must be an integer multiple of the ppn\n"
"Given: num_procs %d ppn %d\n", size, ppn);
MPI_Abort(MPI_COMM_WORLD, 1);
} else {
goto cleanup;
}
}
/* see how many nodes we have */
nnodes = size / ppn;
odd_nnodes = false;
if (0 != (nnodes % 2)) {
/* we have an odd # of nodes */
odd_nnodes = true;
}
/* compute the rank of the rank with which I am to exchange a message.
* Per requirements, this proc must be on another node. To accomplish
* this with max efficiency, we take advantage of knowing that the ppn
* on every node will be the same. We therefore pair up the nodes, and
* pair up the procs on each node, so that only one connection is setup
* for each proc. We also want to ensure that the node pairs are
* "neighboring" - i.e., that they hopefully share a switch so that the
* hop count of sending the messages is minimized.
*/
/* first, determine if my node is odd or even */
my_node = rank / ppn;
if (0 != (my_node % 2)) {
/* compute my twin's rank - as I am an odd numbered node, my
* twin will be on the node below me. Thus, its rank will be
* my rank - ppn
*/
my_twin = rank - ppn;
/* if I am an odd numbered node, then I will receive first */
MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
/* receive the return message so that we meet the stated requirement
* that -every- proc send a message
*/
MPI_Send(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD);
} else {
/* compute my twin's rank - as I am an even numbered node, my
* twin will be on the node above me. Thus, its rank will be
* my rank + ppn
*/
my_twin = rank + ppn;
/* if we have an odd number of nodes, then the last node will be
* even and will have no one above them. In this case, we wrap around
* and ask that node=0 take the additional connections
*/
recvit = true;
if (my_twin >= size) {
my_twin = my_twin - size;
recvit = false;
}
/* I am an even numbered node, so I send first */
MPI_Send(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD);
/* now receive the reply so my twin also meets the requirement - but only
* if we don't have an odd number of nodes. If we have an odd number of
* nodes, then the node=0 procs will already have met their requirement
*/
if (recvit) {
MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
}
/* if we have an odd number of nodes and I am on node=0, then I have
* to take the extra recv
*/
if (odd_nnodes && 0 == my_node) {
my_twin = size - ppn + rank;
MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
/* get a completion time stamp */
gettimeofday(&tv, NULL);
my_timestamp[0] = tv.tv_sec;
my_timestamp[1] = tv.tv_usec;
/* THIS COMPLETES THE OFFICIAL TIMING POINT */
/* Gather to get all the timestamps to rank 0 */
timestamps = NULL;
if (0 == rank) {
timestamps = malloc(2 * size * sizeof(unsigned long));
if (NULL == timestamps) {
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
MPI_Gather(&my_timestamp, 2, MPI_LONG,
timestamps, 2, MPI_LONG, 0, MPI_COMM_WORLD);
if (0 == rank) {
/* The "timestamps" array will now have everyone's timestamp
(i.e., rank 0's timestamp will be in pos 0 & 1,, rank 1's timestamp
will be in 2 & 3, ...etc. */
/* find the maximum timestamp */
maxsec = start_sec;
maxusec = start_usec;
maxrank = -1;
for (i=0; i < 2*size; i+=2) {
if (timestamps[i] < maxsec) {
continue;
}
if (timestamps[i] == maxsec &&
timestamps[i+1] < maxusec) {
continue;
}
maxsec = timestamps[i];
maxusec = timestamps[i+1];
maxrank = i/2;
}
free(timestamps);
/* subtract starting time to get time in microsecs for test */
maxsec = maxsec - start_sec;
if (maxusec >= start_usec) {
maxusec = maxusec - start_usec;
} else {
maxsec--;
maxusec = 1000000 - start_usec + maxusec;
}
/* pretty-print the result */
seconds = maxsec + (maxusec / 1000000l);
minutes = seconds / 60l;
seconds = seconds % 60l;
if (0 == minutes && 0 == seconds) {
fsecs = ((float)(maxsec)*1000000.0 + (float)maxusec) / 1000.0;
fprintf(stderr, "Time test was completed in %8.2f millisecs\nSlowest rank: %d\n",
fsecs, maxrank);
} else {
fprintf(stderr, "Time test was completed in %3lu:%02lu min:sec\nSlowest rank: %d\n",
minutes, seconds, maxrank);
}
}
cleanup:
/* this completes the test */
MPI_Finalize();
return 0;
}
|
