1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2022, Intel Corporation */
/*
* server.c -- a server of the receive-completion-queue example
*
* Please see README.md for a detailed description of this example.
*/
#include <inttypes.h>
#include <librpma.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#define USAGE_STR "usage: %s <server_address> <port>\n"
#include "common-conn.h"
#include "receive-completion-queue-common.h"
int
main(int argc, char *argv[])
{
/* validate parameters */
if (argc < 3) {
fprintf(stderr, USAGE_STR, argv[0]);
exit(-1);
}
/* configure logging thresholds to see more details */
rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO);
rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO);
/* read common parameters */
char *addr = argv[1];
char *port = argv[2];
int ret;
/* prepare memory */
struct rpma_mr_local *recv_mr, *send_mr;
uint64_t *recv = malloc_aligned(MSG_SIZE);
if (recv == NULL)
return -1;
uint64_t *send = malloc_aligned(MSG_SIZE);
if (send == NULL) {
free(recv);
return -1;
}
/* RPMA resources */
struct rpma_peer *peer = NULL;
struct rpma_ep *ep = NULL;
struct rpma_conn_req *req = NULL;
enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED;
struct rpma_conn *conn = NULL;
/*
* lookup an ibv_context via the address and create a new peer using it
*/
ret = server_peer_via_address(addr, &peer);
if (ret)
goto err_free;
/* start a listening endpoint at addr:port */
ret = rpma_ep_listen(peer, addr, port, &ep);
if (ret)
goto err_peer_delete;
/* register the memory */
ret = rpma_mr_reg(peer, recv, MSG_SIZE, RPMA_MR_USAGE_RECV, &recv_mr);
if (ret)
goto err_ep_shutdown;
ret = rpma_mr_reg(peer, send, MSG_SIZE, RPMA_MR_USAGE_SEND, &send_mr);
if (ret) {
(void) rpma_mr_dereg(&recv_mr);
goto err_ep_shutdown;
}
/* create a new connection configuration and set RCQ size */
struct rpma_conn_cfg *cfg = NULL;
ret = rpma_conn_cfg_new(&cfg);
if (ret)
goto err_mr_dereg;
ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE);
if (ret)
goto err_cfg_delete;
/* receive an incoming connection request */
ret = rpma_ep_next_conn_req(ep, cfg, &req);
if (ret)
goto err_cfg_delete;
/*
* Post an initial receive to be prepared for the first message of the client's ping-pong.
*/
ret = rpma_conn_req_recv(req, recv_mr, 0, MSG_SIZE, recv);
if (ret) {
(void) rpma_conn_req_delete(&req);
goto err_cfg_delete;
}
/* accept the connection request and obtain the connection object */
ret = rpma_conn_req_connect(&req, NULL, &conn);
if (ret)
goto err_cfg_delete;
/* wait for the connection to be established */
ret = rpma_conn_next_event(conn, &conn_event);
if (ret)
goto err_conn_disconnect;
if (conn_event != RPMA_CONN_ESTABLISHED) {
fprintf(stderr, "rpma_conn_next_event returned an unexpected event: %s\n",
rpma_utils_conn_event_2str(conn_event));
goto err_conn_disconnect;
}
/* get the connection's main CQ */
struct rpma_cq *cq = NULL;
ret = rpma_conn_get_cq(conn, &cq);
if (ret)
goto err_conn_disconnect;
/* get the connection's RCQ */
struct rpma_cq *rcq = NULL;
ret = rpma_conn_get_rcq(conn, &rcq);
if (ret)
goto err_conn_disconnect;
while (1) {
/* get one receive completion and validate it */
ret = get_wc_and_validate(rcq, IBV_WC_RECV, "rpma_recv()");
if (ret)
break;
if (*recv == I_M_DONE)
break;
/* print the received old value of the client's counter */
(void) printf("SERVER: Value received: %" PRIu64 "\n", *recv);
/* calculate a new counter's value */
*send = *recv + 1;
/* prepare a receive for the client's response */
ret = rpma_recv(conn, recv_mr, 0, MSG_SIZE, recv);
if (ret)
break;
/* send the new value to the client */
(void) printf("SERVER: Value sent: %" PRIu64 "\n", *send);
/*
* XXX when using RPMA_F_COMPLETION_ON_ERROR
* after few rounds rpma_send() returns ENOMEM.
*/
ret = rpma_send(conn, send_mr, 0, MSG_SIZE, RPMA_F_COMPLETION_ALWAYS, NULL);
if (ret)
break;
/* get one send completion and validate it */
ret = get_wc_and_validate(cq, IBV_WC_SEND, "rpma_send()");
if (ret)
break;
}
err_conn_disconnect:
ret |= common_disconnect_and_wait_for_conn_close(&conn);
err_cfg_delete:
ret |= rpma_conn_cfg_delete(&cfg);
err_mr_dereg:
/* deregister the memory regions */
ret |= rpma_mr_dereg(&send_mr);
ret |= rpma_mr_dereg(&recv_mr);
err_ep_shutdown:
ret |= rpma_ep_shutdown(&ep);
err_peer_delete:
/* delete the peer object */
ret |= rpma_peer_delete(&peer);
err_free:
free(send);
free(recv);
return ret ? -1 : 0;
}
|
