1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
// SPDX-License-Identifier: GPL-2.0
/*
* Generic support for Memory System Cache Maintenance operations.
*
* Coherency maintenance drivers register with this simple framework that will
* iterate over each registered instance to first kick off invalidation and
* then to wait until it is complete.
*
* If no implementations are registered yet cpu_cache_has_invalidate_memregion()
* will return false. If this runs concurrently with unregistration then a
* race exists but this is no worse than the case where the operations instance
* responsible for a given memory region has not yet registered.
*/
#include <linux/cache_coherency.h>
#include <linux/cleanup.h>
#include <linux/container_of.h>
#include <linux/export.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/memregion.h>
#include <linux/module.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
static LIST_HEAD(cache_ops_instance_list);
static DECLARE_RWSEM(cache_ops_instance_list_lock);
static void __cache_coherency_ops_instance_free(struct kref *kref)
{
struct cache_coherency_ops_inst *cci =
container_of(kref, struct cache_coherency_ops_inst, kref);
kfree(cci);
}
void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci)
{
kref_put(&cci->kref, __cache_coherency_ops_instance_free);
}
EXPORT_SYMBOL_GPL(cache_coherency_ops_instance_put);
static int cache_inval_one(struct cache_coherency_ops_inst *cci, void *data)
{
if (!cci->ops)
return -EINVAL;
return cci->ops->wbinv(cci, data);
}
static int cache_inval_done_one(struct cache_coherency_ops_inst *cci)
{
if (!cci->ops)
return -EINVAL;
if (!cci->ops->done)
return 0;
return cci->ops->done(cci);
}
static int cache_invalidate_memregion(phys_addr_t addr, size_t size)
{
int ret;
struct cache_coherency_ops_inst *cci;
struct cc_inval_params params = {
.addr = addr,
.size = size,
};
guard(rwsem_read)(&cache_ops_instance_list_lock);
list_for_each_entry(cci, &cache_ops_instance_list, node) {
ret = cache_inval_one(cci, ¶ms);
if (ret)
return ret;
}
list_for_each_entry(cci, &cache_ops_instance_list, node) {
ret = cache_inval_done_one(cci);
if (ret)
return ret;
}
return 0;
}
struct cache_coherency_ops_inst *
_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops,
size_t size)
{
struct cache_coherency_ops_inst *cci;
if (!ops || !ops->wbinv)
return NULL;
cci = kzalloc(size, GFP_KERNEL);
if (!cci)
return NULL;
cci->ops = ops;
INIT_LIST_HEAD(&cci->node);
kref_init(&cci->kref);
return cci;
}
EXPORT_SYMBOL_NS_GPL(_cache_coherency_ops_instance_alloc, "CACHE_COHERENCY");
int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci)
{
guard(rwsem_write)(&cache_ops_instance_list_lock);
list_add(&cci->node, &cache_ops_instance_list);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_register, "CACHE_COHERENCY");
void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci)
{
guard(rwsem_write)(&cache_ops_instance_list_lock);
list_del(&cci->node);
}
EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_unregister, "CACHE_COHERENCY");
int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len)
{
return cache_invalidate_memregion(start, len);
}
EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, "DEVMEM");
/*
* Used for optimization / debug purposes only as removal can race
*
* Machines that do not support invalidation, e.g. VMs, will not have any
* operations instance to register and so this will always return false.
*/
bool cpu_cache_has_invalidate_memregion(void)
{
guard(rwsem_read)(&cache_ops_instance_list_lock);
return !list_empty(&cache_ops_instance_list);
}
EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM");
|
