mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

sched_ext: Compact struct bpf_iter_scx_dsq_kern

Browse source 
struct scx_iter_scx_dsq is defined as 6 u64's and scx_dsq_iter_kern was
using 5 of them. We want to add two more u64 fields but it's better if we do
so while staying within scx_iter_scx_dsq to maintain binary compatibility.

The way scx_iter_scx_dsq_kern is laid out is rather inefficient - the node
field takes up three u64's but only one bit of the last u64 is used. Turn
the bool into u32 flags and only use the lower 16 bits freeing up 48 bits -
16 bits for flags, 32 bits for a u32 - for use by struct
bpf_iter_scx_dsq_kern.

This allows moving the dsq_seq and flags fields of bpf_iter_scx_dsq_kern
into the cursor field reducing the struct size by a full u64.

No behavior changes intended.

Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Tejun Heo 2024-09-09 13:42:47 -10:00
parent cf3e94430d
commit 6462dd53a2
2 changed files with 21 additions and 12 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

10
include/linux/sched/ext.h
View file

@ -119,9 +119,17 @@ enum scx_kf_mask {
__SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST,
}; };
enum scx_dsq_lnode_flags {
SCX_DSQ_LNODE_ITER_CURSOR = 1 << 0,
/* high 16 bits can be for iter cursor flags */
__SCX_DSQ_LNODE_PRIV_SHIFT = 16,
};
struct scx_dsq_list_node { struct scx_dsq_list_node {
struct list_head node; struct list_head node;
bool is_bpf_iter_cursor; u32 flags;
u32 priv; /* can be used by iter cursor */
}; };
/* /*

23
kernel/sched/ext.c
View file

@ -1191,7 +1191,7 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
dsq_lnode = container_of(list_node, struct scx_dsq_list_node, dsq_lnode = container_of(list_node, struct scx_dsq_list_node,
node); node);
} while (dsq_lnode->is_bpf_iter_cursor); } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR);
return container_of(dsq_lnode, struct task_struct, scx.dsq_list); return container_of(dsq_lnode, struct task_struct, scx.dsq_list);
} }
@ -1209,16 +1209,15 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
*/ */
enum scx_dsq_iter_flags { enum scx_dsq_iter_flags {
/* iterate in the reverse dispatch order */ /* iterate in the reverse dispatch order */
SCX_DSQ_ITER_REV = 1U << 0, SCX_DSQ_ITER_REV = 1U << 16,
__SCX_DSQ_ITER_ALL_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
__SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS,
}; };
struct bpf_iter_scx_dsq_kern { struct bpf_iter_scx_dsq_kern {
struct scx_dsq_list_node cursor; struct scx_dsq_list_node cursor;
struct scx_dispatch_q *dsq; struct scx_dispatch_q *dsq;
u32 dsq_seq;
u32 flags;
} __attribute__((aligned(8))); } __attribute__((aligned(8)));
struct bpf_iter_scx_dsq { struct bpf_iter_scx_dsq {
@ -1256,6 +1255,9 @@ static void scx_task_iter_init(struct scx_task_iter *iter)
{ {
lockdep_assert_held(&scx_tasks_lock); lockdep_assert_held(&scx_tasks_lock);
BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS &
((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1));
iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
list_add(&iter->cursor.tasks_node, &scx_tasks); list_add(&iter->cursor.tasks_node, &scx_tasks);
iter->locked = NULL; iter->locked = NULL;
@ -6285,7 +6287,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) !=
__alignof__(struct bpf_iter_scx_dsq)); __alignof__(struct bpf_iter_scx_dsq));
if (flags & ~__SCX_DSQ_ITER_ALL_FLAGS) if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
return -EINVAL; return -EINVAL;
kit->dsq = find_non_local_dsq(dsq_id); kit->dsq = find_non_local_dsq(dsq_id);
@ -6293,9 +6295,8 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
return -ENOENT; return -ENOENT;
INIT_LIST_HEAD(&kit->cursor.node); INIT_LIST_HEAD(&kit->cursor.node);
kit->cursor.is_bpf_iter_cursor = true; kit->cursor.flags |= SCX_DSQ_LNODE_ITER_CURSOR | flags;
kit->dsq_seq = READ_ONCE(kit->dsq->seq); kit->cursor.priv = READ_ONCE(kit->dsq->seq);
kit->flags = flags;
return 0; return 0;
} }
@ -6309,7 +6310,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
{ {
struct bpf_iter_scx_dsq_kern *kit = (void *)it; struct bpf_iter_scx_dsq_kern *kit = (void *)it;
bool rev = kit->flags & SCX_DSQ_ITER_REV; bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;
struct task_struct *p; struct task_struct *p;
unsigned long flags; unsigned long flags;
@ -6330,7 +6331,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i
*/ */
do { do {
p = nldsq_next_task(kit->dsq, p, rev); p = nldsq_next_task(kit->dsq, p, rev);
} while (p && unlikely(u32_before(kit->dsq_seq, p->scx.dsq_seq))); } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));
if (p) { if (p) {
if (rev) if (rev)