1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
/*
* Copyright (C) by Argonne National Laboratory
* See COPYRIGHT in top-level directory
*/
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <mpi.h>
#include "mpitest.h"
#define ELEM_PER_PROC 4
int local_buf[ELEM_PER_PROC];
const int verbose = 0;
int main(int argc, char **argv)
{
int i, rank, nproc;
int shm_rank, shm_nproc;
MPI_Aint size;
int errors = 0;
int **bases = NULL, *my_base = NULL;
int disp_unit;
MPI_Win shm_win = MPI_WIN_NULL, win = MPI_WIN_NULL;
MPI_Comm shm_comm = MPI_COMM_NULL;
MPI_Group shm_group = MPI_GROUP_NULL, world_group = MPI_GROUP_NULL;
int dst_shm_rank, dst_world_rank;
MPI_Info create_info = MPI_INFO_NULL;
MTest_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shm_comm);
MPI_Comm_rank(shm_comm, &shm_rank);
MPI_Comm_size(shm_comm, &shm_nproc);
/* Platform does not support shared memory, just return. */
if (shm_nproc < 2) {
goto exit;
}
/* Specify the last process in the node as the target process */
dst_shm_rank = shm_nproc - 1;
MPI_Comm_group(shm_comm, &shm_group);
MPI_Comm_group(MPI_COMM_WORLD, &world_group);
MPI_Group_translate_ranks(shm_group, 1, &dst_shm_rank, world_group, &dst_world_rank);
bases = calloc(shm_nproc, sizeof(int *));
/* Allocate shm window among local processes, then create a global window with
* those shm window buffers */
MPI_Win_allocate_shared(sizeof(int) * ELEM_PER_PROC, sizeof(int), MPI_INFO_NULL,
shm_comm, &my_base, &shm_win);
if (verbose)
printf("%d -- allocate shared: my_base = %p, absolute base\n", shm_rank, my_base);
for (i = 0; i < shm_nproc; i++) {
MPI_Win_shared_query(shm_win, i, &size, &disp_unit, &bases[i]);
if (verbose)
printf("%d -- shared query: base[%d]=%p, size %ld, unit %d\n",
shm_rank, i, bases[i], size, disp_unit);
}
#ifdef USE_INFO_ALLOC_SHM
MPI_Info_create(&create_info);
MPI_Info_set(create_info, "alloc_shm", "true");
#else
create_info = MPI_INFO_NULL;
#endif
/* Reset data */
for (i = 0; i < ELEM_PER_PROC; i++) {
my_base[i] = 0;
local_buf[i] = i + 1;
}
MPI_Win_create(my_base, sizeof(int) * ELEM_PER_PROC, sizeof(int), create_info, MPI_COMM_WORLD,
&win);
/* Do RMA through global window, then check value through shared window */
MPI_Win_lock_all(0, win);
MPI_Win_lock_all(0, shm_win);
if (shm_rank == 0) {
MPI_Put(&local_buf[0], 1, MPI_INT, dst_world_rank, 0, 1, MPI_INT, win);
MPI_Put(&local_buf[ELEM_PER_PROC - 1], 1, MPI_INT, dst_world_rank, ELEM_PER_PROC - 1, 1,
MPI_INT, win);
MPI_Win_flush(dst_world_rank, win);
}
MPI_Win_sync(shm_win);
MPI_Barrier(shm_comm);
MPI_Win_sync(shm_win);
if (bases[dst_shm_rank][0] != local_buf[0]) {
errors++;
printf("%d -- Got %d at rank %d index %d, expected %d\n", rank,
bases[dst_shm_rank][0], dst_shm_rank, 0, local_buf[0]);
}
if (bases[dst_shm_rank][ELEM_PER_PROC - 1] != local_buf[ELEM_PER_PROC - 1]) {
errors++;
printf("%d -- Got %d at rank %d index %d, expected %d\n", rank,
bases[dst_shm_rank][ELEM_PER_PROC - 1], dst_shm_rank,
ELEM_PER_PROC - 1, local_buf[ELEM_PER_PROC - 1]);
}
MPI_Win_unlock_all(shm_win);
MPI_Win_unlock_all(win);
MPI_Win_free(&win);
MPI_Win_free(&shm_win);
exit:
if (create_info != MPI_INFO_NULL)
MPI_Info_free(&create_info);
if (shm_comm != MPI_COMM_NULL)
MPI_Comm_free(&shm_comm);
if (shm_group != MPI_GROUP_NULL)
MPI_Group_free(&shm_group);
if (world_group != MPI_GROUP_NULL)
MPI_Group_free(&world_group);
MTest_Finalize(errors);
if (bases)
free(bases);
return MTestReturnValue(errors);
}
|
