1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright 2020-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
import unittest
import errno
from pyverbs.cq import CQ, CompChannel
from pyverbs.libibverbs_enums import ibv_qp_create_send_ops_flags, ibv_wr_opcode, ibv_qp_attr_mask
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
import pyverbs.providers.efa.efa_enums as efa_enums
import pyverbs.providers.efa.efadv as efa
from tests.efa_base import EfaRDMATestCase
from tests.efa_base import SRDResources
import tests.utils as u
class CqEventsSRD(SRDResources):
def __init__(self, dev_name, ib_port, gid_index):
super().__init__(dev_name, ib_port, gid_index, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
def create_cq(self):
self.comp_channel = CompChannel(self.ctx)
self.cq = CQ(self.ctx, self.num_msgs, None, self.comp_channel)
self.cq.req_notify()
class CqEventsSRDTestCase(EfaRDMATestCase):
def setUp(self):
super().setUp()
self.iters = 100
def test_cq_events_srd(self):
for use_new_send in [False, True]:
with self.subTest():
super().create_players(CqEventsSRD)
u.traffic(**self.traffic_args, new_send=use_new_send)
class QPSRDTestCase(EfaRDMATestCase):
def setUp(self):
super().setUp()
self.iters = 100
self.server = None
self.client = None
def create_players(self, send_ops_flags=0, qp_count=8, dev_cap=None, wc_flags=0, qp_flags=0):
super().create_players(SRDResources, send_ops_flags=send_ops_flags, qp_count=qp_count,
required_dev_cap=dev_cap, wc_flags=wc_flags, qp_flags=qp_flags)
def full_sq_bad_flow(self):
"""
Check post_send while qp's sq is full.
- Find qp's sq length
- Fill the qp with work requests until overflow
"""
qp_idx = 0
send_op = ibv_wr_opcode.IBV_WR_SEND
ah = u.get_global_ah(self.client, self.gid_index, self.ib_port)
qp_attr, _ = self.client.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_CAP)
max_send_wr = qp_attr.cap.max_send_wr
with self.assertRaises(PyverbsRDMAError) as ex:
for _ in range (max_send_wr + 1):
_, c_sg = u.get_send_elements(self.client, False)
u.send(self.client, c_sg, send_op, new_send=True, qp_idx=qp_idx, ah=ah)
self.assertEqual(ex.exception.error_code, errno.ENOMEM)
def unsolicited_rdma_write_traffic(self, client, server, iters, gid_idx, port):
"""
Runs unsolicited rdma write traffic between two sides
"""
ah_client = u.get_global_ah(client, gid_idx, port)
ah_server = u.get_global_ah(server, gid_idx, port)
poll = u.poll_cq_ex
imm_data = u.IMM_DATA
send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False, send_op)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client, is_imm=True)
poll(client.cq)
poll(server.cq, data=imm_data)
# Validate that the CQE is marked as unsolicited
if not server.cq.is_unsolicited():
raise PyverbsError('Completion was not marked as unsolicited')
msg_received_list = u.get_msg_received(server, read_offset)
for msg in msg_received_list:
u.validate(msg, True, server.msg_size)
def test_qp_ex_srd_send(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
def test_qp_ex_srd_send_imm(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM)
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM)
def test_qp_ex_srd_rdma_read(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ)
self.server.mr.write('s' * self.server.msg_size, self.server.msg_size)
u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_READ)
def test_qp_ex_srd_rdma_write(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE)
u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
def test_qp_ex_srd_rdma_write_with_imm(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM)
def test_qp_ex_srd_rdma_unsolicited_write_with_imm(self):
wc_flag = efa_enums.EFADV_WC_EX_WITH_IS_UNSOLICITED
dev_cap = efa_enums.EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV
qp_flags = efa_enums.EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM,
dev_cap=dev_cap, wc_flags=wc_flag, qp_flags=qp_flags)
self.unsolicited_rdma_write_traffic(**self.traffic_args)
def test_qp_ex_srd_old_send(self):
self.create_players()
u.traffic(**self.traffic_args, new_send=False)
def test_qp_ex_srd_old_send_imm(self):
self.create_players()
u.traffic(**self.traffic_args, new_send=False, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM)
def test_qp_ex_srd_zero_size(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
self.client.msg_size = 0
self.server.msg_size = 0
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
def test_post_receive_qp_state_bad_flow(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1)
u.post_rq_state_bad_flow(self)
def test_post_send_qp_state_bad_flow(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1)
u.post_sq_state_bad_flow(self)
def test_full_rq_bad_flow(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1)
u.full_rq_bad_flow(self)
def test_full_sq_bad_flow(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1)
self.full_sq_bad_flow()
def test_rq_with_larger_sgl_bad_flow(self):
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1)
u.create_rq_with_larger_sgl_bad_flow(self)
|
