1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
"""
Test module for Pyverbs' ParentDomain.
"""
from pyverbs.pd import ParentDomainInitAttr, ParentDomain, ParentDomainContext
from tests.base import RCResources, UDResources, RDMATestCase
from pyverbs.pyverbs_error import PyverbsRDMAError
from pyverbs.cq import CqInitAttrEx, CQEX
import pyverbs.mem_alloc as mem
from pyverbs.libibverbs_enums import _IBV_ALLOCATOR_USE_DEFAULT, IBV_WC_STANDARD_FLAGS, ibv_cq_init_attr_mask
import tests.utils as u
import unittest
import errno
HUGE_PAGE_SIZE = 0x200000
def default_allocator(pd, context, size, alignment, resource_type):
return _IBV_ALLOCATOR_USE_DEFAULT
def default_free(pd, context, ptr, resource_type):
return _IBV_ALLOCATOR_USE_DEFAULT
def mem_align_allocator(pd, context, size, alignment, resource_type):
p = mem.posix_memalign(size, alignment)
return p
def free_func(pd, context, ptr, resource_type):
mem.free(ptr)
def huge_page_alloc(pd, context, size, alignment, resource_type):
ptr = context.user_data
remainder = ptr % alignment
ptr += 0 if remainder == 0 else (alignment - remainder)
context.user_data += size
return ptr
def huge_page_free(pd, context, ptr, resource_type):
"""
No need to free memory, since this allocator assumes the huge page was
externally mapped (and will be externally un-mapped).
"""
pass
def create_parent_domain_with_allocators(res):
"""
Creates parent domain for res instance. The allocators themselves are taken
from res.allocator_func and res.free_func.
:param res: The resources instance to work on (an instance of BaseResources)
"""
if res.allocator_func and res.free_func:
res.pd_ctx = ParentDomainContext(res.pd, res.allocator_func,
res.free_func, res.user_data)
pd_attr = ParentDomainInitAttr(pd=res.pd, pd_context=res.pd_ctx)
try:
res.pd = ParentDomain(res.ctx, attr=pd_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Parent Domain is not supported on this device')
raise ex
def parent_domain_res_cls(base_class):
"""
This is a factory function which creates a class that inherits base_class of
any BaseResources type. Its purpose is to behave exactly as base_class does,
except for creating a parent domain with custom allocators.
Hence the returned class must be initialized with (alloc_func, free_func,
user_data, **kwargs), while kwargs are the arguments needed (if any) for
base_class.
:param base_class: The base resources class to inherit from
:return: ParentDomainRes(alloc_func=None, free_func=None, **kwargs) class
"""
class ParentDomainRes(base_class):
def __init__(self, alloc_func=None, free_func=None, user_data=None, **kwargs):
self.pd_ctx = None
self.protection_domain = None
self.allocator_func = alloc_func
self.free_func = free_func
self.user_data = user_data
super().__init__(**kwargs)
def create_pd(self):
super().create_pd()
self.protection_domain = self.pd
create_parent_domain_with_allocators(self)
return ParentDomainRes
class ParentDomainHugePageRcRes(parent_domain_res_cls(RCResources)):
def __init__(self, alloc_func=None, free_func=None, **kwargs):
user_data = mem.mmap(length=HUGE_PAGE_SIZE,
flags=mem.MAP_ANONYMOUS_ | mem.MAP_PRIVATE_ | mem.MAP_HUGETLB_)
super().__init__(alloc_func=alloc_func, free_func=free_func,
user_data=user_data, **kwargs)
def __del__(self):
mem.munmap(self.user_data, HUGE_PAGE_SIZE)
class ParentDomainCqExSrqRes(parent_domain_res_cls(RCResources)):
"""
Parent domain resources. Based on RCResources.
This includes a parent domain created with the given allocators, in addition
it creates an extended CQ and a SRQ for RC traffic.
:param dev_name: Device name to be used
:param ib_port: IB port of the device to use
:param gid_index: Which GID index to use
:param alloc_func: Custom allocator function
:param free_func: Custom free function
"""
def __init__(self, dev_name, ib_port=None, gid_index=None, alloc_func=None,
free_func=None):
super().__init__(dev_name=dev_name, ib_port=ib_port,
gid_index=gid_index, alloc_func=alloc_func,
free_func=free_func, with_srq=True)
def create_cq(self):
wc_flags = IBV_WC_STANDARD_FLAGS
cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags, parent_domain=self.pd,
comp_mask=ibv_cq_init_attr_mask.IBV_CQ_INIT_ATTR_MASK_FLAGS |
ibv_cq_init_attr_mask.IBV_CQ_INIT_ATTR_MASK_PD)
try:
self.cq = CQEX(self.ctx, cia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Extended CQ with Parent Domain is not supported')
raise ex
class ParentDomainTrafficTest(RDMATestCase):
def setUp(self):
super().setUp()
self.iters = 10
self.server = None
self.client = None
def test_without_allocators_rc_traffic(self):
parent_domain_rc_res = parent_domain_res_cls(RCResources)
self.create_players(parent_domain_rc_res)
u.traffic(**self.traffic_args)
def test_default_allocators_rc_traffic(self):
parent_domain_rc_res = parent_domain_res_cls(RCResources)
self.create_players(parent_domain_rc_res, alloc_func=default_allocator,
free_func=default_free)
u.traffic(**self.traffic_args)
def test_mem_align_rc_traffic(self):
parent_domain_rc_res = parent_domain_res_cls(RCResources)
self.create_players(parent_domain_rc_res,
alloc_func=mem_align_allocator, free_func=free_func)
u.traffic(**self.traffic_args)
def test_mem_align_ud_traffic(self):
parent_domain_ud_res = parent_domain_res_cls(UDResources)
self.create_players(parent_domain_ud_res,
alloc_func=mem_align_allocator, free_func=free_func)
u.traffic(**self.traffic_args)
def test_mem_align_srq_excq_rc_traffic(self):
self.create_players(ParentDomainCqExSrqRes,
alloc_func=mem_align_allocator, free_func=free_func)
u.traffic(**self.traffic_args, is_cq_ex=True)
@u.requires_huge_pages()
def test_huge_page_traffic(self):
self.create_players(ParentDomainHugePageRcRes,
alloc_func=huge_page_alloc, free_func=huge_page_free)
u.traffic(**self.traffic_args)
|
