|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
| 2 | +/* Copyright (c) 2021 Facebook */ |
| 3 | + |
| 4 | +#include <linux/bitmap.h> |
| 5 | +#include <linux/bpf.h> |
| 6 | +#include <linux/btf.h> |
| 7 | +#include <linux/err.h> |
| 8 | +#include <linux/jhash.h> |
| 9 | +#include <linux/random.h> |
| 10 | + |
| 11 | +#define BLOOM_CREATE_FLAG_MASK \ |
| 12 | + (BPF_F_NUMA_NODE | BPF_F_ZERO_SEED | BPF_F_ACCESS_MASK) |
| 13 | + |
| 14 | +struct bpf_bloom_filter { |
| 15 | + struct bpf_map map; |
| 16 | + u32 bitset_mask; |
| 17 | + u32 hash_seed; |
| 18 | + /* If the size of the values in the bloom filter is u32 aligned, |
| 19 | + * then it is more performant to use jhash2 as the underlying hash |
| 20 | + * function, else we use jhash. This tracks the number of u32s |
| 21 | + * in an u32-aligned value size. If the value size is not u32 aligned, |
| 22 | + * this will be 0. |
| 23 | + */ |
| 24 | + u32 aligned_u32_count; |
| 25 | + u32 nr_hash_funcs; |
| 26 | + unsigned long bitset[]; |
| 27 | +}; |
| 28 | + |
| 29 | +static u32 hash(struct bpf_bloom_filter *bloom, void *value, |
| 30 | + u32 value_size, u32 index) |
| 31 | +{ |
| 32 | + u32 h; |
| 33 | + |
| 34 | + if (bloom->aligned_u32_count) |
| 35 | + h = jhash2(value, bloom->aligned_u32_count, |
| 36 | + bloom->hash_seed + index); |
| 37 | + else |
| 38 | + h = jhash(value, value_size, bloom->hash_seed + index); |
| 39 | + |
| 40 | + return h & bloom->bitset_mask; |
| 41 | +} |
| 42 | + |
| 43 | +static int peek_elem(struct bpf_map *map, void *value) |
| 44 | +{ |
| 45 | + struct bpf_bloom_filter *bloom = |
| 46 | + container_of(map, struct bpf_bloom_filter, map); |
| 47 | + u32 i, h; |
| 48 | + |
| 49 | + for (i = 0; i < bloom->nr_hash_funcs; i++) { |
| 50 | + h = hash(bloom, value, map->value_size, i); |
| 51 | + if (!test_bit(h, bloom->bitset)) |
| 52 | + return -ENOENT; |
| 53 | + } |
| 54 | + |
| 55 | + return 0; |
| 56 | +} |
| 57 | + |
| 58 | +static int push_elem(struct bpf_map *map, void *value, u64 flags) |
| 59 | +{ |
| 60 | + struct bpf_bloom_filter *bloom = |
| 61 | + container_of(map, struct bpf_bloom_filter, map); |
| 62 | + u32 i, h; |
| 63 | + |
| 64 | + if (flags != BPF_ANY) |
| 65 | + return -EINVAL; |
| 66 | + |
| 67 | + for (i = 0; i < bloom->nr_hash_funcs; i++) { |
| 68 | + h = hash(bloom, value, map->value_size, i); |
| 69 | + set_bit(h, bloom->bitset); |
| 70 | + } |
| 71 | + |
| 72 | + return 0; |
| 73 | +} |
| 74 | + |
| 75 | +static int pop_elem(struct bpf_map *map, void *value) |
| 76 | +{ |
| 77 | + return -EOPNOTSUPP; |
| 78 | +} |
| 79 | + |
| 80 | +static struct bpf_map *map_alloc(union bpf_attr *attr) |
| 81 | +{ |
| 82 | + u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; |
| 83 | + int numa_node = bpf_map_attr_numa_node(attr); |
| 84 | + struct bpf_bloom_filter *bloom; |
| 85 | + |
| 86 | + if (!bpf_capable()) |
| 87 | + return ERR_PTR(-EPERM); |
| 88 | + |
| 89 | + if (attr->key_size != 0 || attr->value_size == 0 || |
| 90 | + attr->max_entries == 0 || |
| 91 | + attr->map_flags & ~BLOOM_CREATE_FLAG_MASK || |
| 92 | + !bpf_map_flags_access_ok(attr->map_flags) || |
| 93 | + (attr->map_extra & ~0xF)) |
| 94 | + return ERR_PTR(-EINVAL); |
| 95 | + |
| 96 | + /* The lower 4 bits of map_extra specify the number of hash functions */ |
| 97 | + nr_hash_funcs = attr->map_extra & 0xF; |
| 98 | + if (nr_hash_funcs == 0) |
| 99 | + /* Default to using 5 hash functions if unspecified */ |
| 100 | + nr_hash_funcs = 5; |
| 101 | + |
| 102 | + /* For the bloom filter, the optimal bit array size that minimizes the |
| 103 | + * false positive probability is n * k / ln(2) where n is the number of |
| 104 | + * expected entries in the bloom filter and k is the number of hash |
| 105 | + * functions. We use 7 / 5 to approximate 1 / ln(2). |
| 106 | + * |
| 107 | + * We round this up to the nearest power of two to enable more efficient |
| 108 | + * hashing using bitmasks. The bitmask will be the bit array size - 1. |
| 109 | + * |
| 110 | + * If this overflows a u32, the bit array size will have 2^32 (4 |
| 111 | + * GB) bits. |
| 112 | + */ |
| 113 | + if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || |
| 114 | + check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || |
| 115 | + nr_bits > (1UL << 31)) { |
| 116 | + /* The bit array size is 2^32 bits but to avoid overflowing the |
| 117 | + * u32, we use U32_MAX, which will round up to the equivalent |
| 118 | + * number of bytes |
| 119 | + */ |
| 120 | + bitset_bytes = BITS_TO_BYTES(U32_MAX); |
| 121 | + bitset_mask = U32_MAX; |
| 122 | + } else { |
| 123 | + if (nr_bits <= BITS_PER_LONG) |
| 124 | + nr_bits = BITS_PER_LONG; |
| 125 | + else |
| 126 | + nr_bits = roundup_pow_of_two(nr_bits); |
| 127 | + bitset_bytes = BITS_TO_BYTES(nr_bits); |
| 128 | + bitset_mask = nr_bits - 1; |
| 129 | + } |
| 130 | + |
| 131 | + bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); |
| 132 | + bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); |
| 133 | + |
| 134 | + if (!bloom) |
| 135 | + return ERR_PTR(-ENOMEM); |
| 136 | + |
| 137 | + bpf_map_init_from_attr(&bloom->map, attr); |
| 138 | + |
| 139 | + bloom->nr_hash_funcs = nr_hash_funcs; |
| 140 | + bloom->bitset_mask = bitset_mask; |
| 141 | + |
| 142 | + /* Check whether the value size is u32-aligned */ |
| 143 | + if ((attr->value_size & (sizeof(u32) - 1)) == 0) |
| 144 | + bloom->aligned_u32_count = |
| 145 | + attr->value_size / sizeof(u32); |
| 146 | + |
| 147 | + if (!(attr->map_flags & BPF_F_ZERO_SEED)) |
| 148 | + bloom->hash_seed = get_random_int(); |
| 149 | + |
| 150 | + return &bloom->map; |
| 151 | +} |
| 152 | + |
| 153 | +static void map_free(struct bpf_map *map) |
| 154 | +{ |
| 155 | + struct bpf_bloom_filter *bloom = |
| 156 | + container_of(map, struct bpf_bloom_filter, map); |
| 157 | + |
| 158 | + bpf_map_area_free(bloom); |
| 159 | +} |
| 160 | + |
| 161 | +static void *lookup_elem(struct bpf_map *map, void *key) |
| 162 | +{ |
| 163 | + /* The eBPF program should use map_peek_elem instead */ |
| 164 | + return ERR_PTR(-EINVAL); |
| 165 | +} |
| 166 | + |
| 167 | +static int update_elem(struct bpf_map *map, void *key, |
| 168 | + void *value, u64 flags) |
| 169 | +{ |
| 170 | + /* The eBPF program should use map_push_elem instead */ |
| 171 | + return -EINVAL; |
| 172 | +} |
| 173 | + |
| 174 | +static int check_btf(const struct bpf_map *map, const struct btf *btf, |
| 175 | + const struct btf_type *key_type, |
| 176 | + const struct btf_type *value_type) |
| 177 | +{ |
| 178 | + /* Bloom filter maps are keyless */ |
| 179 | + return btf_type_is_void(key_type) ? 0 : -EINVAL; |
| 180 | +} |
| 181 | + |
| 182 | +static int bpf_bloom_btf_id; |
| 183 | +const struct bpf_map_ops bloom_filter_map_ops = { |
| 184 | + .map_meta_equal = bpf_map_meta_equal, |
| 185 | + .map_alloc = map_alloc, |
| 186 | + .map_free = map_free, |
| 187 | + .map_push_elem = push_elem, |
| 188 | + .map_peek_elem = peek_elem, |
| 189 | + .map_pop_elem = pop_elem, |
| 190 | + .map_lookup_elem = lookup_elem, |
| 191 | + .map_update_elem = update_elem, |
| 192 | + .map_check_btf = check_btf, |
| 193 | + .map_btf_name = "bpf_bloom_filter", |
| 194 | + .map_btf_id = &bpf_bloom_btf_id, |
| 195 | +}; |
0 commit comments