1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
|
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
// Copyright (c) 2022 Google
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
/* maximum stack trace depth */
#define MAX_STACKS 8
/* default buffer size */
#define MAX_ENTRIES 10240
struct contention_key {
__s32 stack_id;
};
struct contention_data {
__u64 total_time;
__u64 min_time;
__u64 max_time;
__u32 count;
__u32 flags;
};
struct tstamp_data {
__u64 timestamp;
__u64 lock;
__u32 flags;
__s32 stack_id;
};
/* callstack storage */
struct {
__uint(type, BPF_MAP_TYPE_STACK_TRACE);
__uint(key_size, sizeof(__u32));
__uint(value_size, MAX_STACKS * sizeof(__u64));
__uint(max_entries, MAX_ENTRIES);
} stacks SEC(".maps");
/* maintain timestamp at the beginning of contention */
struct {
__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct tstamp_data);
} tstamp SEC(".maps");
/* actual lock contention statistics */
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(struct contention_key));
__uint(value_size, sizeof(struct contention_data));
__uint(max_entries, MAX_ENTRIES);
} lock_stat SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(__u8));
__uint(max_entries, 1);
} cpu_filter SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(__u8));
__uint(max_entries, 1);
} task_filter SEC(".maps");
/* control flags */
int enabled;
int has_cpu;
int has_task;
int stack_skip;
/* error stat */
int lost;
static inline int can_record(void)
{
if (has_cpu) {
__u32 cpu = bpf_get_smp_processor_id();
__u8 *ok;
ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
if (!ok)
return 0;
}
if (has_task) {
__u8 *ok;
__u32 pid = bpf_get_current_pid_tgid();
ok = bpf_map_lookup_elem(&task_filter, &pid);
if (!ok)
return 0;
}
return 1;
}
SEC("tp_btf/contention_begin")
int contention_begin(u64 *ctx)
{
struct task_struct *curr;
struct tstamp_data *pelem;
if (!enabled || !can_record())
return 0;
curr = bpf_get_current_task_btf();
pelem = bpf_task_storage_get(&tstamp, curr, NULL,
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (!pelem || pelem->lock)
return 0;
pelem->timestamp = bpf_ktime_get_ns();
pelem->lock = (__u64)ctx[0];
pelem->flags = (__u32)ctx[1];
pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP | stack_skip);
if (pelem->stack_id < 0)
lost++;
return 0;
}
SEC("tp_btf/contention_end")
int contention_end(u64 *ctx)
{
struct task_struct *curr;
struct tstamp_data *pelem;
struct contention_key key;
struct contention_data *data;
__u64 duration;
if (!enabled)
return 0;
curr = bpf_get_current_task_btf();
pelem = bpf_task_storage_get(&tstamp, curr, NULL, 0);
if (!pelem || pelem->lock != ctx[0])
return 0;
duration = bpf_ktime_get_ns() - pelem->timestamp;
key.stack_id = pelem->stack_id;
data = bpf_map_lookup_elem(&lock_stat, &key);
if (!data) {
struct contention_data first = {
.total_time = duration,
.max_time = duration,
.min_time = duration,
.count = 1,
.flags = pelem->flags,
};
bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);
pelem->lock = 0;
return 0;
}
__sync_fetch_and_add(&data->total_time, duration);
__sync_fetch_and_add(&data->count, 1);
/* FIXME: need atomic operations */
if (data->max_time < duration)
data->max_time = duration;
if (data->min_time > duration)
data->min_time = duration;
pelem->lock = 0;
return 0;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
|