1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
/*
* Copyright (C) by Argonne National Laboratory
* See COPYRIGHT in top-level directory
*/
/* This test checks that the nemesis code correctly exposes statistics related
* to "fbox" handling. It also attempts to verify that it accurately maintains
* these statistics.
*
* Originally written by Ralf Gunter Correa Carvalho. */
#include <mpi.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include "mpitest.h"
#define TRY(func) \
do { \
err = (func); \
if (err != MPI_SUCCESS) \
MPI_Abort(MPI_COMM_WORLD, err); \
} while (0)
#define STR_LEN 100
#define BUF_COUNT 10
uint64_t null_fbox[2] = { 0 };
int err, rank;
MPI_T_pvar_session session;
MPI_T_pvar_handle fbox_handle;
/* Check that we can successfully write to the variable.
* Question: Do we really want to write pvars other than reset?
*/
void blank_test(void);
void blank_test()
{
uint64_t temp[2] = { -1 };
temp[0] = 0x1234;
temp[1] = 0xABCD;
TRY(MPI_T_pvar_write(session, fbox_handle, temp));
temp[0] = 0xCD34;
temp[1] = 0x12AB;
TRY(MPI_T_pvar_read(session, fbox_handle, temp));
assert(temp[0] == 0x1234);
assert(temp[1] == 0xABCD);
}
/* Nemesis' fastbox falls back to regular queues when more than one message
* is yet to be delivered.
* Here, the sender posts all sends before the receiver has a chance to
* acknowledge any of them; this should force the sender to fall_back to the
* queue every time. */
static void send_first_test(void)
{
uint64_t nem_fbox_fall_back_to_queue_count[2] = { -1 };
/* Reset the fbox variable. */
MPI_T_pvar_reset(session, fbox_handle);
if (rank == 0) {
char send_buf[BUF_COUNT] = { 0x12 };
/* Check that the variable has been correctly initialized. */
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 1);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 2);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 3);
/* Make sure we've posted the sends before the receiver gets a chance
* to receive them.
* FIXME: Ideally this should use a barrier, but that uses messages
* internally and hence will sometimes screw up the asserts above.
*/
MTestSleep(1);
} else if (rank == 1) {
char recv_buf[BUF_COUNT];
MPI_Status status;
MTestSleep(1); /* see above */
MPI_Recv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
MPI_Recv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
MPI_Recv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
MPI_Recv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
}
MPI_Barrier(MPI_COMM_WORLD); /* ensure we've finished this test before
* moving on to the next */
}
/* By posting receives ahead of time, messages should be taken out of the
* fastbox as soon as they are delivered. Hence, the counter should remain 0
* throughout.
* FIXME: This doesn't quite work yet, in part because of the barrier (which
* also uses messages). May want to 'sleep' between sends as a
* workaround.
*/
void recv_first_test(void);
void recv_first_test()
{
uint64_t nem_fbox_fall_back_to_queue_count[2] = { -1 };
/* Reset the fbox variable. */
MPI_T_pvar_reset(session, fbox_handle);
if (rank == 0) {
char send_buf[BUF_COUNT] = { 0x12 };
MPI_Barrier(MPI_COMM_WORLD); /* see below */
/* Check that the variable has been correctly initialized. */
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
MPI_Send(send_buf, BUF_COUNT, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
TRY(MPI_T_pvar_read(session, fbox_handle, nem_fbox_fall_back_to_queue_count));
assert(nem_fbox_fall_back_to_queue_count[1] == 0);
} else if (rank == 1) {
char recv_buf[BUF_COUNT];
MPI_Request reqs[4];
MPI_Irecv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &reqs[0]);
MPI_Irecv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &reqs[1]);
MPI_Irecv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &reqs[2]);
MPI_Irecv(recv_buf, BUF_COUNT, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &reqs[3]);
MPI_Barrier(MPI_COMM_WORLD); /* make sure we've posted the receives
* before the sender gets a chance
* to send them */
MPI_Status status[4];
MPI_Waitall(4, reqs, status);
}
MPI_Barrier(MPI_COMM_WORLD); /* ensure we've finished this test before
* moving on to the next */
}
int main(int argc, char *argv[])
{
int i, size, num, name_len, desc_len, count, verb, thread_support;
int varclass, bind, readonly, continuous, atomic;
int fbox_idx = -1;
char name[STR_LEN], desc[STR_LEN];
MPI_Datatype dtype;
MPI_T_enum enumtype;
MPI_Init(NULL, NULL);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0) {
printf("MPIT pvar test: nem_fbox_fall_back_to_queue_count\n");
fflush(stdout);
}
/* Ensure we're using exactly two ranks. */
assert(size == 2);
/* Standard MPIT initialization. */
TRY(MPI_T_init_thread(MPI_THREAD_SINGLE, &thread_support));
TRY(MPI_T_pvar_get_num(&num));
/* Locate desired MPIT variable. */
for (i = 0; i < num; i++) {
name_len = desc_len = STR_LEN;
TRY(MPI_T_pvar_get_info(i, name, &name_len, &verb, &varclass, &dtype,
&enumtype, desc, &desc_len, &bind, &readonly,
&continuous, &atomic));
if (strcmp(name, "nem_fbox_fall_back_to_queue_count") == 0)
fbox_idx = i;
}
/* Ensure variable was registered by the runtime */
assert(fbox_idx != -1);
/* Initialize MPIT session & variable handle. */
MPI_T_pvar_session_create(&session);
MPI_T_pvar_handle_alloc(session, fbox_idx, NULL, &fbox_handle, &count);
/* Ensure the variable is of the correct size. */
assert(count == 2);
/* Run a batch of tests. */
/* blank_test(); */
send_first_test();
/* recv_first_test(); */
/* Cleanup. */
MPI_T_pvar_handle_free(session, &fbox_handle);
MPI_T_pvar_session_free(&session);
if (rank == 0) {
printf("finished\n");
fflush(stdout);
}
TRY(MPI_T_finalize());
MPI_Finalize();
return 0;
}
|
