1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
/*
* Copyright (c) 2021-2022 Triad National Security, LLC.
* All rights reserved.
*
* Copyright (c) 2021 Nanook Consulting All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/* Timed test of basic fence with no data exchange (barrier) */
#include "pmix.h"
#include "test_common.h"
static int parse_fence_client(int *, int, char **, test_params *, validation_params *);
static void client_help(char *binary) {
fprintf(stderr, "Usage: %s [OPTION...]\n", binary);
fprintf(stderr, " -h Display this message\n");
fprintf(stderr, " --time-fence Time the fence calls (best on a quiet system)\n");
fprintf(stderr, " (Default: off, specifying -m or -r also enables timing)\n");
fprintf(stderr, " -m num Fence time multiplier (similar to an error bar, default: 100)\n");
fprintf(stderr, " note: specifying -m activates fence timing (--time-fence)\n");
fprintf(stderr, " -r num Fence timeout ratio (used to calculate sleep time before fence calls, default: 100)\n");
fprintf(stderr, " note: specifying -r activates fence timing (--time-fence)\n");
/*
fprintf(stderr, " -c fence[_nb] callback shall include all collected data\n");
fprintf(stderr, " -nb use non-blocking fence\n");
*/
}
// client-specific command parser logic
static int parse_fence_client(int *index, int argc, char **argv, test_params *lparams, validation_params *v_params)
{
PMIX_HIDE_UNUSED_PARAMS(argc, v_params);
if (0 == strcmp(argv[*index], "-h") || 0 == strcmp(argv[*index], "--help")) {
// produce client-specific help
client_help(argv[0]);
exit(0);
} else if (0 == strcmp(argv[*index], "--time-fence")) {
lparams->time_fence = true;
// multiplier for fence timeout threshold called 'fence timeout ratio' (1 of 2)
} else if (0 == strcmp(argv[*index], "--fence-timeout-ratio") || 0 == strcmp(argv[*index], "-r") ) {
(*index)++;
lparams->time_fence = true;
if (NULL != argv[*index]) {
lparams->fence_timeout_ratio = strtod(argv[*index], NULL);
}
// multiplier for fence timeout threshold called 'fence time multiplier' (2 of 2)
} else if (0 == strcmp(argv[*index], "--fence-time-multiplier") || 0 == strcmp(argv[*index], "-m") ) {
(*index)++;
lparams->time_fence = true;
if (NULL != argv[*index]) {
lparams->fence_time_multiplier = strtod(argv[*index], NULL);
}
}
else {
return PMIX_ERR_NOT_SUPPORTED;
}
return PMIX_SUCCESS;
}
int main(int argc, char *argv[]) {
int rc;
size_t ninfo = 0;
test_params l_params;
validation_params v_params;
pmix_proc_t job_proc, this_proc;
struct timeval start, end;
unsigned long usecs_elapsed, sleep_time_ms;
double secs_elapsed, fence_time, sleep_time, padded_fence_time;
// pass in function pointer for custom argument processing, if no custom processing, will be null
pmixt_pre_init(argc, argv, &l_params, &v_params, &parse_fence_client);
/* initialization */
PMIXT_CHECK(PMIx_Init(&this_proc, NULL, ninfo), l_params, v_params);
/* Handles everything that needs to happen after PMIx_Init() */
pmixt_post_init(&this_proc, &l_params, &v_params);
PMIX_PROC_CONSTRUCT(&job_proc);
PMIX_LOAD_NSPACE(job_proc.nspace, this_proc.nspace);
job_proc.rank = PMIX_RANK_WILDCARD;
if (l_params.time_fence) {
// establishes baseline fence time before entering loop
fence_time = avg_fence_time();
// padded_fence_time is the time permitted just for the fence call
padded_fence_time = l_params.fence_time_multiplier * fence_time;
sleep_time = l_params.fence_timeout_ratio * padded_fence_time;
sleep_time_ms = (long)(sleep_time * 1000.0);
TEST_VERBOSE(("Rank %d fence timeout ratio: %lf fence time multiplier: %lf",
this_proc.rank, l_params.fence_timeout_ratio, l_params.fence_time_multiplier));
// Synchronize before timing
rc = PMIx_Fence(NULL, 0, NULL, 0);
if (0 != rc) {
TEST_ERROR_EXIT(("PMIx_Fence Problem: ret val = %d", rc));
}
if (0 == this_proc.rank){
sleep_ms(sleep_time_ms);
}
gettimeofday(&start, NULL);
rc = PMIx_Fence(&job_proc, 1, NULL, 0);
gettimeofday(&end, NULL);
if (0 != rc) {
TEST_ERROR_EXIT(("PMIx_Fence Problem: ret val = %d", rc));
}
usecs_elapsed =
((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec));
secs_elapsed = (double) usecs_elapsed / 1E6;
TEST_VERBOSE(("Rank %d fence time: %lf padded_fence_time: %lf sleep_time: %lf secs_elapsed: %lf",
this_proc.rank, fence_time, padded_fence_time, sleep_time, secs_elapsed));
// secs_elapsed for rank 0 must be less than padded_fence_time
if (0 == this_proc.rank) {
if (secs_elapsed > padded_fence_time) {
TEST_ERROR(("%s: PMIx_Fence timeout: Rank 0 elapsed fence time: %lf exceeded cutoff of: %lf",
this_proc.nspace, secs_elapsed, padded_fence_time));
pmixt_exit(PMIX_ERR_TIMEOUT);
}
}
else {
// secs_elapsed for other ranks must be within sleep_time +/- padded_fence_time
if (secs_elapsed < (sleep_time - padded_fence_time) ) {
TEST_ERROR(("%s: PMIx_Fence failed: Rank %d did not wait for Rank 0,"
" elapsed time: %lf",
this_proc.nspace, this_proc.rank, secs_elapsed));
pmixt_exit(PMIX_ERR_TIMEOUT);
}
else if (secs_elapsed > (sleep_time + padded_fence_time) ) {
TEST_ERROR(("%s: PMIx_Fence timeout: Rank %d elapsed fence time: %lf exceeded cutoff of: %lf",
this_proc.nspace, this_proc.rank, secs_elapsed, sleep_time + padded_fence_time));
pmixt_exit(PMIX_ERR_TIMEOUT);
}
}
}
else {
rc = PMIx_Fence(&job_proc, 1, NULL, 0);
if (0 != rc) {
TEST_ERROR_EXIT(("PMIx_Fence Problem: ret val = %d", rc));
}
}
PMIX_PROC_DESTRUCT(&job_proc);
/* finalize */
PMIXT_CHECK(PMIx_Finalize(NULL, 0), l_params, v_params);
/* Handles cleanup */
pmixt_post_finalize(&this_proc, &l_params, &v_params);
}
|
