1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#include <linux/mman.h>
#include <linux/dma-mapping.h>
#include "ionic_fw.h"
#include "ionic_ibdev.h"
__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va)
{
u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;
u64 dma;
if (!buf->tbl_pages)
return cpu_to_le64(0);
if (buf->tbl_pages > 1)
return cpu_to_le64(buf->tbl_dma);
if (buf->tbl_buf)
dma = le64_to_cpu(buf->tbl_buf[0]);
else
dma = buf->tbl_dma;
return cpu_to_le64(dma + (va & pg_mask));
}
__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va)
{
if (buf->tbl_pages > 1) {
u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1;
return cpu_to_be64(va & pg_mask);
}
return 0;
}
int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma)
{
if (unlikely(buf->tbl_pages == buf->tbl_limit))
return -ENOMEM;
if (buf->tbl_buf)
buf->tbl_buf[buf->tbl_pages] = cpu_to_le64(dma);
else
buf->tbl_dma = dma;
++buf->tbl_pages;
return 0;
}
static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev,
struct ionic_tbl_buf *buf)
{
int rc;
buf->tbl_size = buf->tbl_limit * sizeof(*buf->tbl_buf);
buf->tbl_buf = kmalloc(buf->tbl_size, GFP_KERNEL);
if (!buf->tbl_buf)
return -ENOMEM;
buf->tbl_dma = dma_map_single(dev->lif_cfg.hwdev, buf->tbl_buf,
buf->tbl_size, DMA_TO_DEVICE);
rc = dma_mapping_error(dev->lif_cfg.hwdev, buf->tbl_dma);
if (rc) {
kfree(buf->tbl_buf);
return rc;
}
return 0;
}
static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem)
{
struct ib_block_iter biter;
u64 page_dma;
int rc;
rdma_umem_for_each_dma_block(umem, &biter, BIT_ULL(buf->page_size_log2)) {
page_dma = rdma_block_iter_dma_address(&biter);
rc = ionic_pgtbl_page(buf, page_dma);
if (rc)
return rc;
}
return 0;
}
void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf)
{
if (buf->tbl_buf)
dma_unmap_single(dev->lif_cfg.hwdev, buf->tbl_dma,
buf->tbl_size, DMA_TO_DEVICE);
kfree(buf->tbl_buf);
memset(buf, 0, sizeof(*buf));
}
int ionic_pgtbl_init(struct ionic_ibdev *dev,
struct ionic_tbl_buf *buf,
struct ib_umem *umem,
dma_addr_t dma,
int limit,
u64 page_size)
{
int rc;
memset(buf, 0, sizeof(*buf));
if (umem) {
limit = ib_umem_num_dma_blocks(umem, page_size);
buf->page_size_log2 = order_base_2(page_size);
}
if (limit < 1)
return -EINVAL;
buf->tbl_limit = limit;
/* skip pgtbl if contiguous / direct translation */
if (limit > 1) {
rc = ionic_tbl_buf_alloc(dev, buf);
if (rc)
return rc;
}
if (umem)
rc = ionic_pgtbl_umem(buf, umem);
else
rc = ionic_pgtbl_page(buf, dma);
if (rc)
goto err_unbuf;
return 0;
err_unbuf:
ionic_pgtbl_unbuf(dev, buf);
return rc;
}
|
