diff --git a/scheds/rust/scx_p2dq/src/bpf/intf.h b/scheds/rust/scx_p2dq/src/bpf/intf.h index a5a214d44..b67d3569d 100644 --- a/scheds/rust/scx_p2dq/src/bpf/intf.h +++ b/scheds/rust/scx_p2dq/src/bpf/intf.h @@ -62,6 +62,7 @@ enum stat_idx { P2DQ_STAT_LLC_MIGRATION, P2DQ_STAT_NODE_MIGRATION, P2DQ_STAT_KEEP, + P2DQ_STAT_ENQ_CPU, P2DQ_STAT_ENQ_LLC, P2DQ_STAT_ENQ_INTR, P2DQ_STAT_ENQ_MIG, diff --git a/scheds/rust/scx_p2dq/src/bpf/main.bpf.c b/scheds/rust/scx_p2dq/src/bpf/main.bpf.c index 4563a5bc3..b50581092 100644 --- a/scheds/rust/scx_p2dq/src/bpf/main.bpf.c +++ b/scheds/rust/scx_p2dq/src/bpf/main.bpf.c @@ -870,7 +870,6 @@ static __always_inline void async_p2dq_enqueue(struct enqueue_promise *ret, ret->kind = P2DQ_ENQUEUE_PROMISE_COMPLETE; return; } - if (cpuc->nice_task) enq_flags |= SCX_ENQ_PREEMPT; @@ -890,6 +889,21 @@ static __always_inline void async_p2dq_enqueue(struct enqueue_promise *ret, return; } + // If the last CPU is running an interactive task then reenque to the + // CPUs local DSQ. This should reduce the number of migrations. + if (taskc->last_cpu_dsq_id != SCX_DSQ_INVALID && + taskc->interactive && + scx_bpf_dsq_nr_queued(taskc->last_cpu_dsq_id) == 0 && + scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON|taskc->last_cpu) == 0) { + ret->kind = P2DQ_ENQUEUE_PROMISE_VTIME; + ret->vtime.dsq_id = taskc->last_cpu_dsq_id; + ret->vtime.enq_flags = enq_flags; + ret->vtime.slice_ns = taskc->slice_ns; + ret->vtime.vtime = p->scx.dsq_vtime; + stat_inc(P2DQ_STAT_ENQ_CPU); + return; + } + bool migrate = can_migrate(taskc, llcx); if (interactive_dsq && taskc->interactive && !migrate) { taskc->dsq_id = llcx->intr_dsq; @@ -967,6 +981,8 @@ static __always_inline int p2dq_running_impl(struct task_struct *p) taskc->llc_id = llcx->id; taskc->node_id = llcx->node_id; taskc->was_nice = p->scx.weight < 100; + taskc->last_cpu = task_cpu; + taskc->last_cpu_dsq_id = cpuc->affn_dsq; cpuc->interactive = taskc->interactive; cpuc->dsq_index = taskc->dsq_index; cpuc->nice_task = p->scx.weight < 100; @@ -1335,6 +1351,7 @@ static __always_inline s32 p2dq_init_task_impl(struct task_struct *p, } else if (p->scx.weight > 100) { taskc->dsq_index = nr_dsqs_per_llc - 1; } + taskc->last_cpu_dsq_id = SCX_DSQ_INVALID; taskc->last_dsq_index = taskc->dsq_index; taskc->slice_ns = slice_ns; taskc->all_cpus = p->cpus_ptr == &p->cpus_mask && diff --git a/scheds/rust/scx_p2dq/src/bpf/types.h b/scheds/rust/scx_p2dq/src/bpf/types.h index 0f2fee382..799126ba6 100644 --- a/scheds/rust/scx_p2dq/src/bpf/types.h +++ b/scheds/rust/scx_p2dq/src/bpf/types.h @@ -10,6 +10,7 @@ struct p2dq_timer { struct cpu_ctx { int id; u32 llc_id; + u64 dsq_id; u64 affn_dsq; u32 dsq_index; u64 slice_ns; @@ -61,10 +62,12 @@ struct task_p2dq { u32 node_id; u64 used; u64 last_dsq_id; + u64 last_cpu_dsq_id; u64 last_run_started; u64 last_run_at; u64 llc_runs; /* how many runs on the current LLC */ int last_dsq_index; + s32 last_cpu; bool interactive; bool was_nice; diff --git a/scheds/rust/scx_p2dq/src/main.rs b/scheds/rust/scx_p2dq/src/main.rs index 9eaa1e4dd..4875476e1 100644 --- a/scheds/rust/scx_p2dq/src/main.rs +++ b/scheds/rust/scx_p2dq/src/main.rs @@ -42,6 +42,7 @@ use bpf_intf::stat_idx_P2DQ_STAT_DIRECT; use bpf_intf::stat_idx_P2DQ_STAT_DISPATCH_PICK2; use bpf_intf::stat_idx_P2DQ_STAT_DSQ_CHANGE; use bpf_intf::stat_idx_P2DQ_STAT_DSQ_SAME; +use bpf_intf::stat_idx_P2DQ_STAT_ENQ_CPU; use bpf_intf::stat_idx_P2DQ_STAT_ENQ_INTR; use bpf_intf::stat_idx_P2DQ_STAT_ENQ_LLC; use bpf_intf::stat_idx_P2DQ_STAT_ENQ_MIG; @@ -152,6 +153,7 @@ impl<'a> Scheduler<'a> { dsq_change: stats[stat_idx_P2DQ_STAT_DSQ_CHANGE as usize], same_dsq: stats[stat_idx_P2DQ_STAT_DSQ_SAME as usize], keep: stats[stat_idx_P2DQ_STAT_KEEP as usize], + enq_cpu: stats[stat_idx_P2DQ_STAT_ENQ_CPU as usize], enq_llc: stats[stat_idx_P2DQ_STAT_ENQ_LLC as usize], enq_intr: stats[stat_idx_P2DQ_STAT_ENQ_INTR as usize], enq_mig: stats[stat_idx_P2DQ_STAT_ENQ_MIG as usize], diff --git a/scheds/rust/scx_p2dq/src/stats.rs b/scheds/rust/scx_p2dq/src/stats.rs index c7ecd249a..7ae569cc4 100644 --- a/scheds/rust/scx_p2dq/src/stats.rs +++ b/scheds/rust/scx_p2dq/src/stats.rs @@ -23,6 +23,8 @@ pub struct Metrics { pub same_dsq: u64, #[stat(desc = "Number of times a task kept running")] pub keep: u64, + #[stat(desc = "Number of times a task was enqueued to CPU DSQ")] + pub enq_cpu: u64, #[stat(desc = "Number of times a task was enqueued to LLC DSQ")] pub enq_llc: u64, #[stat(desc = "Number of times a task was enqueued to interactive DSQ")] @@ -53,12 +55,13 @@ impl Metrics { fn format(&self, w: &mut W) -> Result<()> { writeln!( w, - "direct/idle/keep {}/{}/{}\n\tdsq same/migrate {}/{}\n\tenq llc/intr/mig {}/{}/{}", + "direct/idle/keep {}/{}/{}\n\tdsq same/migrate {}/{}\n\tenq cpu/llc/intr/mig {}/{}/{}/{}", self.direct, self.idle, self.keep, self.same_dsq, self.dsq_change, + self.enq_cpu, self.enq_llc, self.enq_intr, self.enq_mig, @@ -84,6 +87,7 @@ impl Metrics { dsq_change: self.dsq_change - rhs.dsq_change, same_dsq: self.same_dsq - rhs.same_dsq, keep: self.keep - rhs.keep, + enq_cpu: self.enq_cpu - rhs.enq_cpu, enq_llc: self.enq_llc - rhs.enq_llc, enq_intr: self.enq_intr - rhs.enq_intr, enq_mig: self.enq_mig - rhs.enq_mig,