Skip to content

BPF task work WIP #9311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
+582 −18
Closed

BPF task work WIP #9311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions include/linux/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ enum btf_field_type {
BPF_WORKQUEUE = (1 << 10),
BPF_UPTR = (1 << 11),
BPF_RES_SPIN_LOCK = (1 << 12),
BPF_TASK_WORK = (1 << 13),
};

typedef void (*btf_dtor_kfunc_t)(void *);
Expand Down Expand Up @@ -245,6 +246,7 @@ struct btf_record {
int timer_off;
int wq_off;
int refcount_off;
int task_work_off;
struct btf_field fields[];
};

Expand Down Expand Up @@ -340,6 +342,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type)
return "bpf_rb_node";
case BPF_REFCOUNT:
return "bpf_refcount";
case BPF_TASK_WORK:
return "bpf_task_work";
default:
WARN_ON_ONCE(1);
return "unknown";
Expand Down Expand Up @@ -378,6 +382,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type)
return sizeof(struct bpf_rb_node);
case BPF_REFCOUNT:
return sizeof(struct bpf_refcount);
case BPF_TASK_WORK:
return sizeof(struct bpf_task_work);
default:
WARN_ON_ONCE(1);
return 0;
Expand Down Expand Up @@ -410,6 +416,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type)
return __alignof__(struct bpf_rb_node);
case BPF_REFCOUNT:
return __alignof__(struct bpf_refcount);
case BPF_TASK_WORK:
return __alignof__(struct bpf_task_work);
default:
WARN_ON_ONCE(1);
return 0;
Expand Down Expand Up @@ -441,6 +449,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
case BPF_KPTR_REF:
case BPF_KPTR_PERCPU:
case BPF_UPTR:
case BPF_TASK_WORK:
break;
default:
WARN_ON_ONCE(1);
Expand Down Expand Up @@ -577,6 +586,7 @@ void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
bool lock_src);
void bpf_timer_cancel_and_free(void *timer);
void bpf_wq_cancel_and_free(void *timer);
void bpf_task_work_cancel_and_free(void *timer);
void bpf_list_head_free(const struct btf_field *field, void *list_head,
struct bpf_spin_lock *spin_lock);
void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
Expand Down Expand Up @@ -2390,6 +2400,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);
bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);

Expand Down
4 changes: 4 additions & 0 deletions include/uapi/linux/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7410,6 +7410,10 @@ struct bpf_timer {
__u64 __opaque[2];
} __attribute__((aligned(8)));

struct bpf_task_work {
__u64 __opaque[8];
} __attribute__((aligned(8)));

struct bpf_wq {
__u64 __opaque[2];
} __attribute__((aligned(8)));
Expand Down
8 changes: 5 additions & 3 deletions kernel/bpf/arraymap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,20 +431,22 @@ static void *array_map_vmalloc_addr(struct bpf_array *array)
return (void *)round_down((unsigned long)array, PAGE_SIZE);
}

static void array_map_free_timers_wq(struct bpf_map *map)
static void array_map_free_internal_structs(struct bpf_map *map)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);
int i;

/* We don't reset or free fields other than timer and workqueue
* on uref dropping to zero.
*/
if (btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE)) {
if (btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) {
for (i = 0; i < array->map.max_entries; i++) {
if (btf_record_has_field(map->record, BPF_TIMER))
bpf_obj_free_timer(map->record, array_map_elem_ptr(array, i));
if (btf_record_has_field(map->record, BPF_WORKQUEUE))
bpf_obj_free_workqueue(map->record, array_map_elem_ptr(array, i));
if (btf_record_has_field(map->record, BPF_TASK_WORK))
bpf_obj_free_task_work(map->record, array_map_elem_ptr(array, i));
}
}
}
Expand Down Expand Up @@ -783,7 +785,7 @@ const struct bpf_map_ops array_map_ops = {
.map_alloc = array_map_alloc,
.map_free = array_map_free,
.map_get_next_key = array_map_get_next_key,
.map_release_uref = array_map_free_timers_wq,
.map_release_uref = array_map_free_internal_structs,
.map_lookup_elem = array_map_lookup_elem,
.map_update_elem = array_map_update_elem,
.map_delete_elem = array_map_delete_elem,
Expand Down
15 changes: 15 additions & 0 deletions kernel/bpf/btf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3527,6 +3527,15 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_
goto end;
}
}
if (field_mask & BPF_TASK_WORK) {
if (!strcmp(name, "bpf_task_work")) {
if (*seen_mask & BPF_TASK_WORK)
return -E2BIG;
*seen_mask |= BPF_TASK_WORK;
type = BPF_TASK_WORK;
goto end;
}
}
field_mask_test_name(BPF_LIST_HEAD, "bpf_list_head");
field_mask_test_name(BPF_LIST_NODE, "bpf_list_node");
field_mask_test_name(BPF_RB_ROOT, "bpf_rb_root");
Expand Down Expand Up @@ -3693,6 +3702,7 @@ static int btf_find_field_one(const struct btf *btf,
case BPF_LIST_NODE:
case BPF_RB_NODE:
case BPF_REFCOUNT:
case BPF_TASK_WORK:
ret = btf_find_struct(btf, var_type, off, sz, field_type,
info_cnt ? &info[0] : &tmp);
if (ret < 0)
Expand Down Expand Up @@ -3985,6 +3995,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
rec->timer_off = -EINVAL;
rec->wq_off = -EINVAL;
rec->refcount_off = -EINVAL;
rec->task_work_off = -EINVAL;
for (i = 0; i < cnt; i++) {
field_type_size = btf_field_type_size(info_arr[i].type);
if (info_arr[i].off + field_type_size > value_size) {
Expand Down Expand Up @@ -4050,6 +4061,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
case BPF_LIST_NODE:
case BPF_RB_NODE:
break;
case BPF_TASK_WORK:
WARN_ON_ONCE(rec->task_work_off >= 0);
rec->task_work_off = rec->fields[i].offset;
break;
default:
ret = -EFAULT;
goto end;
Expand Down
22 changes: 14 additions & 8 deletions kernel/bpf/hashtab.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ static bool htab_has_extra_elems(struct bpf_htab *htab)
return !htab_is_percpu(htab) && !htab_is_lru(htab) && !is_fd_htab(htab);
}

static void htab_free_prealloced_timers_and_wq(struct bpf_htab *htab)
static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)
{
u32 num_entries = htab->map.max_entries;
int i;
Expand All @@ -233,6 +233,9 @@ static void htab_free_prealloced_timers_and_wq(struct bpf_htab *htab)
if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE))
bpf_obj_free_workqueue(htab->map.record,
htab_elem_value(elem, htab->map.key_size));
if (btf_record_has_field(htab->map.record, BPF_TASK_WORK))
bpf_obj_free_task_work(htab->map.record,
htab_elem_value(elem, htab->map.key_size));
cond_resched();
}
}
Expand Down Expand Up @@ -1490,7 +1493,7 @@ static void delete_all_elements(struct bpf_htab *htab)
}
}

static void htab_free_malloced_timers_and_wq(struct bpf_htab *htab)
static void htab_free_malloced_internal_structs(struct bpf_htab *htab)
{
int i;

Expand All @@ -1508,22 +1511,25 @@ static void htab_free_malloced_timers_and_wq(struct bpf_htab *htab)
if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE))
bpf_obj_free_workqueue(htab->map.record,
htab_elem_value(l, htab->map.key_size));
if (btf_record_has_field(htab->map.record, BPF_TASK_WORK))
bpf_obj_free_task_work(htab->map.record,
htab_elem_value(l, htab->map.key_size));
}
cond_resched_rcu();
}
rcu_read_unlock();
}

static void htab_map_free_timers_and_wq(struct bpf_map *map)
static void htab_map_free_internal_structs(struct bpf_map *map)
{
struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

/* We only free timer and workqueue on uref dropping to zero */
if (btf_record_has_field(htab->map.record, BPF_TIMER | BPF_WORKQUEUE)) {
if (btf_record_has_field(htab->map.record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) {
if (!htab_is_prealloc(htab))
htab_free_malloced_timers_and_wq(htab);
htab_free_malloced_internal_structs(htab);
else
htab_free_prealloced_timers_and_wq(htab);
htab_free_prealloced_internal_structs(htab);
}
}

Expand Down Expand Up @@ -2255,7 +2261,7 @@ const struct bpf_map_ops htab_map_ops = {
.map_alloc = htab_map_alloc,
.map_free = htab_map_free,
.map_get_next_key = htab_map_get_next_key,
.map_release_uref = htab_map_free_timers_and_wq,
.map_release_uref = htab_map_free_internal_structs,
.map_lookup_elem = htab_map_lookup_elem,
.map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem,
.map_update_elem = htab_map_update_elem,
Expand All @@ -2276,7 +2282,7 @@ const struct bpf_map_ops htab_lru_map_ops = {
.map_alloc = htab_map_alloc,
.map_free = htab_map_free,
.map_get_next_key = htab_map_get_next_key,
.map_release_uref = htab_map_free_timers_and_wq,
.map_release_uref = htab_map_free_internal_structs,
.map_lookup_elem = htab_lru_map_lookup_elem,
.map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem,
.map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys,
Expand Down
Loading
Loading