diff options
| -rw-r--r-- | kernel/sched/ext/ext.c | 28 | ||||
| -rw-r--r-- | kernel/sched/ext/sub.c | 9 | ||||
| -rw-r--r-- | kernel/sched/sched.h | 3 |
3 files changed, 25 insertions, 15 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 154ef4ada614..af46c3fee610 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2268,6 +2268,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, /** * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ + * @sch: scheduler placing @p * @p: task to move * @enq_flags: %SCX_ENQ_* * @src_rq: rq to move the task from, locked on entry, released on return @@ -2275,7 +2276,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, * * Move @p which is currently on @src_rq to @dst_rq's local DSQ. */ -static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, +static void move_remote_task_to_local_dsq(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { lockdep_assert_rq_held(src_rq); @@ -2291,15 +2293,19 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, switch_rq_lock(src_rq, dst_rq); /* - * We want to pass scx-specific enq_flags but activate_task() will - * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.remote_activate_enq_flags instead. + * activate_task() below truncates enq_flags to 32 bits and re-derives + * @p's owner, dropping our scx flags and the placing @sch. We own @rq, + * so stash both across the call. The enqueue reads them back, keeping + * the scx flags and checking caps against the placer, not the owner. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags); + WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || + dst_rq->scx.remote_activate_sch); dst_rq->scx.remote_activate_enq_flags = enq_flags; + dst_rq->scx.remote_activate_sch = sch; activate_task(dst_rq, p, 0); dst_rq->scx.remote_activate_enq_flags = 0; + dst_rq->scx.remote_activate_sch = NULL; } /* @@ -2431,12 +2437,12 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, !WARN_ON_ONCE(src_rq != task_rq(p)); } -static bool consume_remote_task(struct rq *this_rq, +static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { - move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { switch_rq_lock(src_rq, this_rq); @@ -2497,8 +2503,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } } else { /* @@ -2551,7 +2556,7 @@ retry: } if (task_can_run_on_remote_rq(sch, p, rq, false)) { - if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) + if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) return true; goto retry; } @@ -2644,8 +2649,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); } else { - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 270473c67a41..655632b5511f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -237,12 +237,17 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq struct task_struct *p, u64 *enq_flags) { s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); u64 missing; + /* + * On a remote activation the scheduling sched (@asch) differs from + * @p's owner (@sch). Check caps against the scheduling sched. + */ if (*enq_flags & SCX_ENQ_PREEMPT) - needed |= scx_caps_for_preempt(sch, rq); - missing = scx_missing_caps(sch, cpu_of(rq), needed); + needed |= scx_caps_for_preempt(asch, rq); + missing = scx_missing_caps(asch, cpu_of(rq), needed); /* requirements met */ if (likely(!missing)) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c0a4699a6c0a..cdd21c814008 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -800,8 +800,9 @@ struct scx_rq { struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; - /* see move_remote_task_to_local_dsq() */ + /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ u64 remote_activate_enq_flags; + struct scx_sched *remote_activate_sch; u32 nr_running; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ bool in_select_cpu; |