1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2020 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2021 Triad National Security, LLC. All rights
* reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "ompi/request/request.h"
#include "ompi/errhandler/errcode.h"
#include "ompi/runtime/params.h"
#include "ompi/mca/coll/base/coll_tags.h"
#include "ompi/mca/pml/base/pml_base_request.h"
#include "ompi/mca/pml/pml.h"
#include "opal/mca/backtrace/backtrace.h"
/**************************
* Support Routines
**************************/
bool ompi_request_is_failed_fn(ompi_request_t *req)
{
mca_pml_base_request_t* pml_req = NULL;
if( !ompi_ftmpi_enabled ) {
return false;
}
/* Pick our behavior depending upon the type of request */
switch(req->req_type) {
case OMPI_REQUEST_PML:
pml_req = (mca_pml_base_request_t*)req;
/*
* Toggle 'off' the MPI_ANY_SOURCE MPI_ERR_PROC_FAILED_PENDING flag
* We will recheck, but in the case the request is complete we
* need to remove the error code.
*/
if( MPI_ERR_PROC_FAILED_PENDING == req->req_status.MPI_ERROR ) {
req->req_status.MPI_ERROR = MPI_SUCCESS;
}
break;
case OMPI_REQUEST_COLL:
case OMPI_REQUEST_COMM:
/* Supported by bubbling up errors from REQUEST_PML subrequests
* thus this type of requests never need to be interrupted */
return false;
default:
/* not supported yet, pretend everything is always fine */
return false;
}
/*
* Sanity check
*/
assert( NULL != req->req_mpi_object.comm );
/*
* If the request is complete, then just skip it
*/
if( REQUEST_COMPLETE(req) ) {
return false;
}
/*
* Has this communicator been 'revoked'?
*
* If so unless we are in the FT part (propagate revoke, agreement or
* shrink) this should fail.
*/
if( OPAL_UNLIKELY(ompi_comm_is_revoked(req->req_mpi_object.comm) &&
req->req_type == OMPI_REQUEST_PML && !ompi_request_tag_is_ft(pml_req->req_tag)) ) {
/* Do not set req->req_status.MPI_SOURCE */
req->req_status.MPI_ERROR = MPI_ERR_REVOKED;
opal_output_verbose(10, ompi_ftmpi_output_handle,
"%s ompi_request_is_failed: %p (peer %d, tag %d) is on communicator %s(%s) that has been revoked!",
OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm));
goto return_with_error;
}
/*
* Collectives Check:
* If the request is part of a collective, then the whole communicator
* must be ok to continue. If not then return first failed process
*/
if(OPAL_UNLIKELY( ompi_comm_coll_revoked(req->req_mpi_object.comm) &&
(req->req_type == OMPI_REQUEST_PML && ompi_request_tag_is_collective(pml_req->req_tag)) )) {
/* Do not set req->req_status.MPI_SOURCE */
req->req_status.MPI_ERROR = MPI_ERR_PROC_FAILED;
opal_output_verbose(10, ompi_ftmpi_output_handle,
"%s ompi_request_is_failed: Request %p (peer %d) is part of a collective (tag %d), and some process died. (mpi_source %3d)",
OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag, req->req_status.MPI_SOURCE );
goto return_with_error;
}
/* Corner-cases: two processes that can't fail (NULL and myself) */
if((pml_req->req_peer == MPI_PROC_NULL) ||
(OMPI_COMM_IS_INTRA(req->req_mpi_object.comm) && pml_req->req_peer == req->req_mpi_object.comm->c_local_group->grp_my_rank)) {
return false;
}
/* If any_source but not FT related then the request is always marked for return */
if( OPAL_UNLIKELY(MPI_ANY_SOURCE == pml_req->req_peer && !ompi_comm_is_any_source_enabled(req->req_mpi_object.comm)) ) {
if( !ompi_request_tag_is_ft(pml_req->req_tag) ) {
req->req_status.MPI_ERROR = MPI_ERR_PROC_FAILED_PENDING;
/* If it is a probe/mprobe, escalate the error */
if( (MCA_PML_REQUEST_MPROBE == pml_req->req_type) ||
(MCA_PML_REQUEST_PROBE == pml_req->req_type) ) {
req->req_status.MPI_ERROR = MPI_ERR_PROC_FAILED;
}
opal_output_verbose(10, ompi_ftmpi_output_handle,
"%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%s) peer ANY_SOURCE %s!",
OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm),
ompi_mpi_errnum_get_string(req->req_status.MPI_ERROR));
goto return_with_error;
}
}
/* Any type of request with a dead process must be terminated with error */
if( OPAL_UNLIKELY(!ompi_comm_is_proc_active(req->req_mpi_object.comm, pml_req->req_peer,
OMPI_COMM_IS_INTER(req->req_mpi_object.comm))) ) {
req->req_status.MPI_SOURCE = pml_req->req_peer;
req->req_status.MPI_ERROR = MPI_ERR_PROC_FAILED;
assert(MPI_ANY_SOURCE != pml_req->req_peer); /* this case is handled above, so... */
opal_output_verbose(10, ompi_ftmpi_output_handle,
"%s ompi_request_is_failed: Request %p (peer %d, tag %d) in comm %s(%s) mpi_source %3d failed - Ret %s",
OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag,
req->req_mpi_object.comm->c_name, ompi_comm_print_cid(req->req_mpi_object.comm),
req->req_status.MPI_SOURCE,
ompi_mpi_errnum_get_string(req->req_status.MPI_ERROR));
goto return_with_error;
}
OPAL_OUTPUT_VERBOSE((100, ompi_ftmpi_output_handle,
"%s ompi_request_is_failed: Request %p (peer %d tag %d) is still ok. (mpi_source %3d)",
OMPI_NAME_PRINT(OMPI_PROC_MY_NAME), (void*)req, pml_req->req_peer, pml_req->req_tag, req->req_status.MPI_SOURCE ));
return false;
return_with_error:
if( MPI_ERR_PROC_FAILED_PENDING != req->req_status.MPI_ERROR ) {
int cancelled = req->req_status._cancelled;
#if OPAL_ENABLE_DEBUG
int verbose = opal_output_get_verbosity(ompi_ftmpi_output_handle);
if( verbose > 90 ) {
opal_backtrace_print(stderr, NULL, 1);
}
if( verbose > 50 ) {
MCA_PML_CALL(dump(req->req_mpi_object.comm, ompi_ftmpi_output_handle));
}
#endif /* OPAL_ENABLE_DEBUG */
/* Cancel and force completion immediately
* However, for Revoked and Collective error we can't complete
* with an error before the buffer is unpinned (i.e. the request gets
* wire cancelled).
*/
ompi_request_cancel(req);
req->req_status._cancelled = cancelled; /* This request is not user cancelled here, it is completed in error */
return REQUEST_COMPLETE(req); /* If this request is not complete yet, it is still ok and needs more spinning */
}
return (MPI_SUCCESS != req->req_status.MPI_ERROR);
}
|
