diff options
| author | Mark Brown <broonie@kernel.org> | 2026-07-06 14:53:51 +0100 |
|---|---|---|
| committer | Mark Brown <broonie@kernel.org> | 2026-07-06 14:53:51 +0100 |
| commit | 24717dc42fb70555564d4b0fce6801e65b6a5413 (patch) | |
| tree | a094f4e564b5fd133f94819f0d675ca41f57b786 | |
| parent | 45808caa3e8767a1047523ca0e5396991e163291 (diff) | |
| parent | aa6d9def48ea424a50c21de90ebe609c383cb05d (diff) | |
| download | linux-next-24717dc42fb70555564d4b0fce6801e65b6a5413.tar.gz linux-next-24717dc42fb70555564d4b0fce6801e65b6a5413.zip | |
Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
87 files changed, 2317 insertions, 1452 deletions
diff --git a/Documentation/arch/x86/amd-memory-encryption.rst b/Documentation/arch/x86/amd-memory-encryption.rst index bd840df708ea..92edb26a50d2 100644 --- a/Documentation/arch/x86/amd-memory-encryption.rst +++ b/Documentation/arch/x86/amd-memory-encryption.rst @@ -53,7 +53,7 @@ CPUID function 0x8000001f reports information related to SME:: system physical addresses, not guest physical addresses) -If support for SME is present, MSR 0xc00100010 (MSR_AMD64_SYSCFG) can be used to +If support for SME is present, MSR 0xc0010010 (MSR_AMD64_SYSCFG) can be used to determine if SME is enabled and/or to enable memory encryption:: 0xc0010010: diff --git a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml index 833a01cdd1b1..5c52e57c6647 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml @@ -32,8 +32,9 @@ properties: "#interrupt-cells": description: - SoC interrupt line index. - const: 1 + First cell is the SoC interrupt line index. Optional second cell + specifies the parent interrupt index to route to. + enum: [1, 2] reg: minItems: 1 diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dbc00b6dd69e..222dfab9225f 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -265,6 +265,29 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +static bool amd_uncore_group_valid(struct perf_event *event) +{ + struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counters = 0; + + if (leader->pmu == event->pmu) + counters++; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu == event->pmu && + sibling->state > PERF_EVENT_STATE_OFF) + counters++; + } + + /* + * When pmu->event_init() is called, the event is yet to be linked to + * its leader's sibling list, so it is counted separately + */ + return (counters + 1) <= pmu->num_counters; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore_pmu *pmu; @@ -283,6 +306,14 @@ static int amd_uncore_event_init(struct perf_event *event) return -ENODEV; /* + * Ensure that all events in a group can be scheduled together so that + * a failure can be reported at perf_event_open() time rather than + * silently at pmu->add() time when no free counter is found + */ + if (event->group_leader != event && !amd_uncore_group_valid(event)) + return -EINVAL; + + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, * Interrupts can be directed to a single target core, however, event diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d1af33d96d0a..af0b67ffb43d 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2539,7 +2539,8 @@ static int x86_pmu_event_init(struct perf_event *event) } if (READ_ONCE(x86_pmu.attr_rdpmc) && - !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) + !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) && + !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE)) event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2b35483e2b70..b39c6ce0efb5 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3533,7 +3533,7 @@ static void intel_pmu_update_rdpmc_user_disable(struct perf_event *event) */ if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE || (x86_pmu.attr_rdpmc == X86_USER_RDPMC_CONDITIONAL_ENABLE && - event->ctx->task)) + (event->attach_state & PERF_ATTACH_TASK))) event->hw.config &= ~ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; else event->hw.config |= ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; @@ -3547,8 +3547,6 @@ static void intel_pmu_enable_event(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; - intel_pmu_update_rdpmc_user_disable(event); - if (unlikely(event->attr.precise_ip)) static_call(x86_pmu_pebs_enable)(event); @@ -5147,6 +5145,8 @@ static int intel_pmu_hw_config(struct perf_event *event) leader->hw.flags |= PERF_X86_EVENT_ACR; } + intel_pmu_update_rdpmc_user_disable(event); + if ((event->attr.type == PERF_TYPE_HARDWARE) || (event->attr.type == PERF_TYPE_HW_CACHE)) return 0; @@ -7679,8 +7679,10 @@ static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus) x86_pmu.num_hybrid_pmus = hweight_long(pmus_mask); x86_pmu.hybrid_pmu = kzalloc_objs(struct x86_hybrid_pmu, x86_pmu.num_hybrid_pmus); - if (!x86_pmu.hybrid_pmu) + if (!x86_pmu.hybrid_pmu) { + x86_pmu.num_hybrid_pmus = 0; return -ENOMEM; + } static_branch_enable(&perf_is_hybrid); x86_pmu.filter = intel_pmu_filter; @@ -7863,14 +7865,14 @@ __init int intel_pmu_init(void) struct attribute **td_attr = &empty_attrs; struct attribute **mem_attr = &empty_attrs; struct attribute **tsx_attr = &empty_attrs; + struct x86_hybrid_pmu *pmu; + unsigned int fixed_mask; union cpuid10_edx edx; union cpuid10_eax eax; union cpuid10_ebx ebx; - unsigned int fixed_mask; + int version, i, ret; bool pmem = false; - int version, i; char *name; - struct x86_hybrid_pmu *pmu; /* Architectural Perfmon was introduced starting with Core "Yonah" */ if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { @@ -7940,19 +7942,10 @@ __init int intel_pmu_init(void) x86_pmu.lbr_read = intel_pmu_lbr_read_32; } - if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) - intel_pmu_arch_lbr_init(); - intel_pebs_init(); x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */ - if (version >= 5) { - x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated; - if (x86_pmu.intel_cap.anythread_deprecated) - pr_cont(" AnyThread deprecated, "); - } - /* The perf side of core PMU is ready to support the mediated vPMU. */ x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU; @@ -8546,7 +8539,9 @@ __init int intel_pmu_init(void) * * Initialize the common PerfMon capabilities here. */ - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = grt_latency_data; x86_pmu.get_event_constraints = adl_get_event_constraints; @@ -8604,7 +8599,9 @@ __init int intel_pmu_init(void) case INTEL_METEORLAKE: case INTEL_METEORLAKE_L: case INTEL_ARROWLAKE_U: - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = cmt_latency_data; x86_pmu.get_event_constraints = mtl_get_event_constraints; @@ -8635,7 +8632,9 @@ __init int intel_pmu_init(void) pr_cont("Pantherlake Hybrid events, "); name = "pantherlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8650,7 +8649,9 @@ __init int intel_pmu_init(void) pr_cont("Arrowlake Hybrid events, "); name = "arrowlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8667,7 +8668,9 @@ __init int intel_pmu_init(void) pr_cont("Lunarlake Hybrid events, "); name = "lunarlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8692,7 +8695,9 @@ __init int intel_pmu_init(void) break; case INTEL_ARROWLAKE_H: - intel_pmu_init_hybrid(hybrid_big_small_tiny); + ret = intel_pmu_init_hybrid(hybrid_big_small_tiny); + if (ret) + return ret; x86_pmu.pebs_latency_data = arl_h_latency_data; x86_pmu.get_event_constraints = arl_h_get_event_constraints; @@ -8727,7 +8732,9 @@ __init int intel_pmu_init(void) case INTEL_NOVALAKE_L: pr_cont("Novalake Hybrid events, "); name = "novalake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = nvl_latency_data; x86_pmu.get_event_constraints = mtl_get_event_constraints; @@ -8829,8 +8836,13 @@ __init int intel_pmu_init(void) &x86_pmu.intel_ctrl); /* AnyThread may be deprecated on arch perfmon v5 or later */ - if (x86_pmu.intel_cap.anythread_deprecated) + if (version >= 5 && edx.split.anythread_deprecated) { x86_pmu.format_attrs = intel_arch_formats_attr; + pr_cont("AnyThread deprecated, "); + } + + if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) + intel_pmu_arch_lbr_init(); intel_pmu_check_event_constraints_all(NULL); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e..e86e4ba91e1b 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1444,10 +1444,6 @@ struct event_constraint intel_skl_pebs_event_constraints[] = { }; struct event_constraint intel_icl_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x100000000ULL), /* old INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), /* SLOTS */ - INTEL_PLD_CONSTRAINT(0x1cd, 0xff), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ @@ -1470,9 +1466,6 @@ struct event_constraint intel_icl_pebs_event_constraints[] = { }; struct event_constraint intel_glc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe), INTEL_PLD_CONSTRAINT(0x1cd, 0xfe), INTEL_PSD_CONSTRAINT(0x2cd, 0x1), @@ -1497,9 +1490,6 @@ struct event_constraint intel_glc_pebs_event_constraints[] = { }; struct event_constraint intel_lnc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ @@ -1531,9 +1521,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { }; struct event_constraint intel_pnc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cae2e02fe6cc..f8fadb0b16a4 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1213,7 +1213,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) { u64 from, to; int br_sel = cpuc->br_sel; - int i, j, type, to_plm; + int i, j, type, from_plm, to_plm; bool compress = false; /* if sampling all branches, then nothing to filter */ @@ -1232,7 +1232,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) * Doesn't support OTHER_BRANCH decoding for now. * OTHER_BRANCH branch type still rely on software decoding. */ - if (static_cpu_has(X86_FEATURE_ARCH_LBR) && + if (static_branch_likely(&x86_lbr_type) && type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) { to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; type = arch_lbr_br_type_map[type] | to_plm; @@ -1245,8 +1245,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) type |= X86_BR_NO_TX; } - /* if type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + /* + * If type does not correspond, then discard. + * Specifically reject entries whose from address is in + * kernel space when only X86_BR_USER is requested. + */ + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) { cpuc->lbr_entries[i].from = 0; compress = true; } diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 7857959c6e82..9f8a80c9dcb6 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -757,7 +757,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -953,16 +953,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1148,6 +1150,44 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + if (uncore_pmu_broken(pmu)) + return -ENODEV; + + ret = uncore_box_init(box); + if (ret) + goto err; + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1167,26 +1207,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } @@ -1248,11 +1284,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1272,7 +1313,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } @@ -1486,7 +1526,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1494,12 +1535,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1515,7 +1558,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1526,9 +1569,14 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; - if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) + box = pmu->boxes[die]; + if (box && box->cpu >= 0 && + atomic_dec_return(&box->refcnt) == 0) { + if (uncore_box_active(box) && + atomic_dec_return(&pmu->activeboxes) == 0) + uncore_pmu_unregister(pmu); uncore_box_exit(box); + } } } } @@ -1537,9 +1585,15 @@ static int uncore_event_cpu_offline(unsigned int cpu) { int die, target; + /* Clear the references */ + die = topology_logical_die_id(cpu); + uncore_box_unref(uncore_msr_uncores, die); + uncore_box_unref(uncore_mmio_uncores, die); + /* Check if exiting cpu is used for collecting uncore events */ if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) - goto unref; + return 0; + /* Find a new cpu to collect uncore events */ target = cpumask_any_but(topology_die_cpumask(cpu), cpu); @@ -1552,16 +1606,10 @@ static int uncore_event_cpu_offline(unsigned int cpu) uncore_change_context(uncore_msr_uncores, cpu, target); uncore_change_context(uncore_mmio_uncores, cpu, target); uncore_change_context(uncore_pci_uncores, cpu, target); - -unref: - /* Clear the references */ - die = topology_logical_die_id(cpu); - uncore_box_unref(uncore_msr_uncores, die); - uncore_box_unref(uncore_mmio_uncores, die); return 0; } -static int allocate_boxes(struct intel_uncore_type **types, +static void allocate_boxes(struct intel_uncore_type **types, unsigned int die, unsigned int cpu) { struct intel_uncore_box *box, *tmp; @@ -1575,11 +1623,13 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); goto cleanup; + } box->pmu = pmu; box->dieid = die; list_add(&box->active_list, &allocated); @@ -1590,35 +1640,30 @@ static int allocate_boxes(struct intel_uncore_type **types, list_del_init(&box->active_list); box->pmu->boxes[die] = box; } - return 0; + return; cleanup: list_for_each_entry_safe(box, tmp, &allocated, active_list) { list_del_init(&box->active_list); kfree(box); } - return -ENOMEM; } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, ret; - - ret = allocate_boxes(types, id, cpu); - if (ret) - return ret; + int i; for (; *types; types++) { type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) - uncore_box_init(box); + uncore_box_setup(pmu, box); } } return 0; @@ -1626,93 +1671,35 @@ static int uncore_box_ref(struct intel_uncore_type **types, static int uncore_event_cpu_online(unsigned int cpu) { - int die, target, msr_ret, mmio_ret; + int die, target; die = topology_logical_die_id(cpu); - msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); - mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; + allocate_boxes(uncore_msr_uncores, die, cpu); + allocate_boxes(uncore_mmio_uncores, die, cpu); /* * Check if there is an online cpu in the package * which collects uncore events already. */ target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); - if (target < nr_cpu_ids) - return 0; - - cpumask_set_cpu(cpu, &uncore_cpu_mask); - - if (!msr_ret) + if (target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_mask); uncore_change_context(uncore_msr_uncores, -1, cpu); - if (!mmio_ret) uncore_change_context(uncore_mmio_uncores, -1, cpu); - uncore_change_context(uncore_pci_uncores, -1, cpu); - return 0; -} - -static int __init type_pmu_register(struct intel_uncore_type *type) -{ - int i, ret; - - for (i = 0; i < type->num_boxes; i++) { - ret = uncore_pmu_register(&type->pmus[i]); - if (ret) - return ret; + uncore_change_context(uncore_pci_uncores, -1, cpu); } - return 0; -} - -static int __init uncore_msr_pmus_register(void) -{ - struct intel_uncore_type **types = uncore_msr_uncores; - int ret; - for (; *types; types++) { - ret = type_pmu_register(*types); - if (ret) - return ret; - } - return 0; -} - -static int __init uncore_cpu_init(void) -{ - int ret; - - ret = uncore_types_init(uncore_msr_uncores); - if (ret) - goto err; - - ret = uncore_msr_pmus_register(); - if (ret) - goto err; + uncore_box_ref(uncore_msr_uncores, die, cpu); + uncore_box_ref(uncore_mmio_uncores, die, cpu); return 0; -err: - uncore_types_exit(uncore_msr_uncores); - uncore_msr_uncores = empty_uncore; - return ret; } -static int __init uncore_mmio_init(void) +static int __init uncore_pmu_types_init(struct intel_uncore_type **types) { - struct intel_uncore_type **types = uncore_mmio_uncores; - int ret; - - ret = uncore_types_init(types); + int ret = uncore_types_init(types); if (ret) - goto err; + uncore_types_exit(types); - for (; *types; types++) { - ret = type_pmu_register(*types); - if (ret) - goto err; - } - return 0; -err: - uncore_types_exit(uncore_mmio_uncores); - uncore_mmio_uncores = empty_uncore; return ret; } @@ -2013,12 +2000,16 @@ static int __init intel_uncore_init(void) if (uncore_init->cpu_init) { uncore_init->cpu_init(); - cret = uncore_cpu_init(); + cret = uncore_pmu_types_init(uncore_msr_uncores); + if (cret) + uncore_msr_uncores = empty_uncore; } if (uncore_init->mmio_init) { uncore_init->mmio_init(); - mret = uncore_mmio_init(); + mret = uncore_pmu_types_init(uncore_mmio_uncores); + if (mret) + uncore_mmio_uncores = empty_uncore; } if (cret && pret && mret) { diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c2e5ccb1d72c..c8dfa2d21bfd 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -129,7 +129,7 @@ struct intel_uncore_type { #define events_group attr_groups[2] struct intel_uncore_ops { - void (*init_box)(struct intel_uncore_box *); + int (*init_box)(struct intel_uncore_box *); void (*exit_box)(struct intel_uncore_box *); void (*disable_box)(struct intel_uncore_box *); void (*enable_box)(struct intel_uncore_box *); @@ -146,13 +146,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; @@ -185,7 +196,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -557,17 +568,29 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, return box->pmu->type->ops->read_counter(box, event); } -static inline void uncore_box_init(struct intel_uncore_box *box) +static inline bool uncore_box_active(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { - if (box->pmu->type->ops->init_box) - box->pmu->type->ops->init_box(box); + return (!box->pmu->type->ops->init_box || + test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)); +} + +static inline int uncore_box_init(struct intel_uncore_box *box) +{ + int ret = 0; + + if (!test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags) && + box->pmu->type->ops->init_box) { + ret = box->pmu->type->ops->init_box(box); + if (!ret) + __set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags); } + + return ret; } static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index e50776222256..0a22edf4d509 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -489,14 +489,15 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) return unit->addr; } -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { u64 box_ctl = intel_generic_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); + return 0; } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -578,15 +579,16 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box)); } -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { int box_ctl = intel_pci_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + return pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) @@ -648,7 +650,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = { #define UNCORE_GENERIC_MMIO_SIZE 0x4000 -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { static struct intel_uncore_discovery_unit *unit; struct intel_uncore_type *type = box->pmu->type; @@ -658,13 +660,13 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!unit) { pr_warn("Uncore type %d id %d: Cannot find box control address.\n", type->type_id, box->pmu->pmu_idx); - return; + return -ENODEV; } if (!unit->addr) { pr_warn("Uncore type %d box %d: Invalid box control address.\n", type->type_id, unit->id); - return; + return -ENODEV; } addr = unit->addr; @@ -672,10 +674,11 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!box->io_addr) { pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", type->type_id, unit->id, (unsigned long long)addr); - return; + return -ENOMEM; } writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); + return 0; } void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index e1330342b92e..142e1b56cfc2 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -148,11 +148,11 @@ void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, @@ -160,7 +160,7 @@ void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, struct perf_event *event); -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index 8962e7cb21e3..7a6855281102 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -199,9 +199,10 @@ DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7"); DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63"); DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63"); -static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box) +static int nhmex_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); + return 0; } static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index edddd4f9ab5f..055131c508ff 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -295,12 +295,14 @@ static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct pe wrmsrq(event->hw.config_base, 0); } -static void snb_uncore_msr_init_box(struct intel_uncore_box *box) +static int snb_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } + + return 0; } static void snb_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -394,7 +396,7 @@ void snb_uncore_cpu_init(void) snb_uncore_cbox.num_boxes = topology_num_cores_per_package(); } -static void skl_uncore_msr_init_box(struct intel_uncore_box *box) +static int skl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, @@ -404,6 +406,8 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box) /* The 8th CBOX has different MSR space */ if (box->pmu->pmu_idx == 7) __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags); + + return 0; } static void skl_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -547,10 +551,12 @@ static struct intel_uncore_type *tgl_msr_uncores[] = { NULL, }; -static void rkl_uncore_msr_init_box(struct intel_uncore_box *box) +static int rkl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } void tgl_uncore_cpu_init(void) @@ -707,9 +713,10 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = { .format_group = &adl_uncore_format_group, }; -static void mtl_uncore_msr_init_box(struct intel_uncore_box *box) +static int mtl_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); + return 0; } static struct intel_uncore_ops mtl_uncore_msr_ops = { @@ -773,10 +780,12 @@ static struct intel_uncore_type *lnl_msr_uncores[] = { #define LNL_UNC_MSR_GLOBAL_CTL 0x240e -static void lnl_uncore_msr_init_box(struct intel_uncore_box *box) +static int lnl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } static struct intel_uncore_ops lnl_uncore_msr_ops = { @@ -874,7 +883,7 @@ static const struct attribute_group snb_uncore_imc_format_group = { .attrs = snb_uncore_imc_formats_attr, }; -static void snb_uncore_imc_init_box(struct intel_uncore_box *box) +static int snb_uncore_imc_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; struct pci_dev *pdev = box->pci_dev; @@ -893,10 +902,13 @@ static void snb_uncore_imc_init_box(struct intel_uncore_box *box) addr &= ~(PAGE_SIZE - 1); box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL; + return 0; } static void snb_uncore_imc_enable_box(struct intel_uncore_box *box) @@ -928,7 +940,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -1532,7 +1544,7 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void) #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 #define TGL_UNCORE_PCI_IMC_MAP_SIZE 0xe000 -static void +static int uncore_get_box_mmio_addr(struct intel_uncore_box *box, unsigned int base_offset, int bar_offset, int step) @@ -1541,19 +1553,20 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, struct intel_uncore_pmu *pmu = box->pmu; struct intel_uncore_type *type = pmu->type; resource_size_t addr; + int ret = 0; u32 bar; if (!pdev) { pr_warn("perf uncore: Cannot find matched IMC device.\n"); - return; + return -ENODEV; } pci_read_config_dword(pdev, bar_offset, &bar); if (!(bar & BIT(0))) { pr_warn("perf uncore: BAR 0x%x is disabled. Failed to map %s counters.\n", bar_offset, type->name); - pci_dev_put(pdev); - return; + ret = -ENODEV; + goto out; } bar &= ~BIT(0); addr = (resource_size_t)(bar + step * pmu->pmu_idx); @@ -1565,23 +1578,26 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, addr += base_offset; box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { + ret = -ENOMEM; pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); - + } +out: pci_dev_put(pdev); + return ret; } -static void __uncore_imc_init_box(struct intel_uncore_box *box, +static int __uncore_imc_init_box(struct intel_uncore_box *box, unsigned int base_offset) { - uncore_get_box_mmio_addr(box, base_offset, + return uncore_get_box_mmio_addr(box, base_offset, SNB_UNCORE_PCI_IMC_BAR_OFFSET, TGL_UNCORE_MMIO_IMC_MEM_OFFSET); } -static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, 0); + return __uncore_imc_init_box(box, 0); } static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { @@ -1648,13 +1664,15 @@ void tgl_uncore_mmio_init(void) #define ADL_UNCORE_IMC_CTL_INT (ADL_UNCORE_IMC_CTL_RST_CTRL | \ ADL_UNCORE_IMC_CTL_RST_CTRS) -static void adl_uncore_imc_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); + int ret = __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); /* The global control in MC1 can control both MCs. */ - if (box->io_addr && (box->pmu->pmu_idx == 1)) + if (!ret && (box->pmu->pmu_idx == 1)) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL); + + return ret; } static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box) @@ -1731,9 +1749,9 @@ static struct freerunning_counters adl_uncore_imc_freerunning[] = { [ADL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xA0, 0x0, 0x0, 1, 64 }, }; -static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); + return __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); } static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = { @@ -1803,9 +1821,9 @@ static const struct attribute_group lnl_uncore_format_group = { .attrs = lnl_uncore_formats_attr, }; -static void lnl_uncore_hbo_init_box(struct intel_uncore_box *box) +static int lnl_uncore_hbo_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, + return uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, LNL_UNCORE_HBO_OFFSET); } @@ -1829,14 +1847,16 @@ static struct intel_uncore_type lnl_uncore_hbo = { .format_group = &lnl_uncore_format_group, }; -static void lnl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int lnl_uncore_sncu_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, + int ret = uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, 0); - if (box->io_addr) + if (!ret) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + LNL_UNCORE_GLOBAL_CTL); + + return ret; } static struct intel_uncore_ops lnl_uncore_sncu_ops = { @@ -1887,13 +1907,15 @@ static struct intel_uncore_type ptl_uncore_imc = { .mmio_map_size = 0xf00, }; -static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int ptl_uncore_sncu_init_box(struct intel_uncore_box *box) { - intel_generic_uncore_mmio_init_box(box); + int ret = intel_generic_uncore_mmio_init_box(box); /* Clear the global freeze bit */ if (box->io_addr) writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET); + + return ret; } static struct intel_uncore_ops ptl_uncore_sncu_ops = { diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 334dc384b5b9..a97cd029db36 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -627,12 +627,12 @@ static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct pe return count; } -static void snbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int snbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); } static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -680,12 +680,14 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, wrmsrq(hwc->config_base, hwc->config); } -static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int snbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT); + + return 0; } static struct attribute *snbep_uncore_formats_attr[] = { @@ -1507,18 +1509,21 @@ int snbep_uncore_pci_init(void) /* end of Sandy Bridge-EP uncore support */ /* IvyTown uncore support */ -static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT); + + return 0; } -static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; - pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \ @@ -2784,7 +2789,7 @@ static struct intel_uncore_type hswep_uncore_cbox = { /* * Write SBOX Initialization register bit by bit to avoid spurious #GPs */ -static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) +static int hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); @@ -2798,6 +2803,8 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) wrmsrq(msr, flags); } } + + return 0; } static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = { @@ -4162,12 +4169,13 @@ static const struct attribute_group skx_upi_uncore_format_group = { .attrs = skx_upi_uncore_formats_attr, }; -static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_upi_uncore_pci_ops = { @@ -4323,12 +4331,13 @@ static struct intel_uncore_type skx_uncore_upi = { .cleanup_mapping = skx_upi_cleanup_mapping, }; -static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_m2m_uncore_pci_ops = { @@ -4831,13 +4840,13 @@ void snr_uncore_cpu_init(void) uncore_msr_uncores = snr_msr_uncores; } -static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops snr_m2m_uncore_pci_ops = { @@ -5010,17 +5019,22 @@ static int snr_uncore_mmio_map(struct intel_uncore_box *box, return 0; } -static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, +static int __snr_uncore_mmio_init_box(struct intel_uncore_box *box, unsigned int box_ctl, int mem_offset, unsigned int device) { - if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device)) + int ret; + + ret = snr_uncore_mmio_map(box, box_ctl, mem_offset, device); + if (!ret) writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); + + return ret; } -static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +static int snr_uncore_mmio_init_box(struct intel_uncore_box *box) { - __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), + return __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), SNR_IMC_MMIO_MEM0_OFFSET, SNR_MC_DEVICE_ID); } @@ -5637,14 +5651,14 @@ int icx_uncore_pci_init(void) return 0; } -static void icx_uncore_imc_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_init_box(struct intel_uncore_box *box) { unsigned int box_ctl = box->pmu->type->box_ctl + box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN); int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, + return __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, SNR_MC_DEVICE_ID); } @@ -5701,12 +5715,12 @@ static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = { { /* end: all zeroes */ }, }; -static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), mem_offset, SNR_MC_DEVICE_ID); } @@ -6003,10 +6017,10 @@ static struct intel_uncore_type spr_uncore_mdf = { .name = "mdf", }; -static void spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) +static int spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) { __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - intel_generic_uncore_mmio_init_box(box); + return intel_generic_uncore_mmio_init_box(box); } static struct intel_uncore_ops spr_uncore_mmio_offs8_ops = { @@ -6187,12 +6201,11 @@ static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = { #define SPR_MC_DEVICE_ID 0x3251 -static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), - mem_offset, SPR_MC_DEVICE_ID); + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SPR_MC_DEVICE_ID); } static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = { @@ -6881,20 +6894,24 @@ static unsigned int dmr_iio_freerunning_box_offsets[] = { 0x0, 0x8000, 0x18000, 0x20000 }; -static void dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) +static int dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; u64 mmio_base; if (box->pmu->pmu_idx >= type->num_boxes) - return; + return -ENODEV; mmio_base = DMR_IMH1_HIOP_MMIO_BASE; mmio_base += dmr_iio_freerunning_box_offsets[box->pmu->pmu_idx]; box->io_addr = ioremap(mmio_base, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } + + return 0; } static struct intel_uncore_ops dmr_uncore_freerunning_ops = { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index eae24bb35dc1..a8afea8d38f0 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -668,7 +668,7 @@ union perf_capabilities { u64 perf_metrics:1; u64 pebs_output_pt_available:1; u64 pebs_timing_info:1; - u64 anythread_deprecated:1; + u64 __reserved:1; u64 rdpmc_metrics_clear:1; }; u64 capabilities; @@ -1344,7 +1344,7 @@ static inline u64 x86_pmu_get_event_config(struct perf_event *event) static inline bool x86_pmu_has_rdpmc_user_disable(struct pmu *pmu) { return !!(hybrid(pmu, config_mask) & - ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); + ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); } extern struct event_constraint emptyconstraint; diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index a8de503def37..95f1782d1e17 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -60,17 +60,15 @@ void hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set) static u32 hv_apic_read(u32 reg) { - u32 reg_val, hi; + struct msr reg_val; switch (reg) { case APIC_EOI: - rdmsr(HV_X64_MSR_EOI, reg_val, hi); - (void)hi; - return reg_val; + rdmsrq(HV_X64_MSR_EOI, reg_val.q); + return reg_val.l; case APIC_TASKPRI: - rdmsr(HV_X64_MSR_TPR, reg_val, hi); - (void)hi; - return reg_val; + rdmsrq(HV_X64_MSR_TPR, reg_val.q); + return reg_val.l; default: return native_apic_mem_read(reg); diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index a7294656ad90..c9769a7b6e66 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -103,7 +103,7 @@ .byte 0xb8 ASM_NL \ .long __kcfi_typeid_##name ASM_NL \ CFI_POST_PADDING \ - SYM_FUNC_END(__cfi_##name) + SYM_END(__cfi_##name, SYM_T_FUNC) /* UML needs to be able to override memcpy() and friends for KASAN. */ #ifdef CONFIG_UML diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 9cd136d4515c..645e65ac1586 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -62,8 +62,6 @@ static inline int intel_microcode_get_datasize(struct microcode_header_intel *hd return hdr->datasize ? : DEFAULT_UCODE_DATASIZE; } -extern u32 intel_get_platform_id(void); - static inline u32 intel_get_microcode_revision(void) { u32 rev, dummy; diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 87b1d4c0727e..8d8f890c4bc0 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -238,6 +238,7 @@ extern void early_cpu_init(void); extern void identify_secondary_cpu(unsigned int cpu); extern void print_cpu_info(struct cpuinfo_x86 *); void print_cpu_msr(struct cpuinfo_x86 *); +extern u32 intel_get_platform_id(void); /* * Friendlier CR3 helpers. diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index 575f8408a9e7..8f6edcdcfd87 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -102,6 +102,7 @@ static inline void __resctrl_sched_in(struct task_struct *tsk) struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); u32 closid = READ_ONCE(state->default_closid); u32 rmid = READ_ONCE(state->default_rmid); + struct msr val; u32 tmp; /* @@ -123,7 +124,9 @@ static inline void __resctrl_sched_in(struct task_struct *tsk) if (closid != state->cur_closid || rmid != state->cur_rmid) { state->cur_closid = closid; state->cur_rmid = rmid; - wrmsr(MSR_IA32_PQR_ASSOC, rmid, closid); + val.l = rmid; + val.h = closid; + wrmsrq(MSR_IA32_PQR_ASSOC, val.q); } } diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62936a3bde19..400baffcd609 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1775,8 +1775,8 @@ static int cfi_rewrite_callers(s32 *start, s32 *end) #define FINEIBT_WARN(_f, _v) \ WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v) -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { int ret; @@ -2088,8 +2088,8 @@ bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) #else /* !CONFIG_FINEIBT: */ -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { if (IS_ENABLED(CONFIG_CFI) && builtin) pr_info("CFI: Using standard kCFI\n"); @@ -2101,8 +2101,8 @@ static void poison_cfi(void *addr) { } #endif /* !CONFIG_FINEIBT */ -void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi) +void __init_or_module apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi) { return __apply_fineibt(start_retpoline, end_retpoline, start_cfi, end_cfi, diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index aa1e19979aa8..90025451ace2 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1191,11 +1191,11 @@ void disable_local_APIC(void) * restore the disabled state. */ if (enabled_via_apicbase) { - unsigned int l, h; + struct msr val; - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_ENABLE; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_ENABLE; + wrmsrq(MSR_IA32_APICBASE, val.q); } #endif } @@ -1960,7 +1960,8 @@ static bool __init detect_init_APIC(void) static bool __init apic_verify(unsigned long addr) { - u32 features, h, l; + struct msr val; + u32 features; /* * The APIC feature bit should now be enabled @@ -1975,9 +1976,9 @@ static bool __init apic_verify(unsigned long addr) /* The BIOS may have set up the APIC at some other address */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (l & MSR_IA32_APICBASE_ENABLE) - addr = l & MSR_IA32_APICBASE_BASE; + rdmsrq(MSR_IA32_APICBASE, val.q); + if (val.l & MSR_IA32_APICBASE_ENABLE) + addr = val.l & MSR_IA32_APICBASE_BASE; } register_lapic_address(addr); @@ -1987,7 +1988,7 @@ static bool __init apic_verify(unsigned long addr) bool __init apic_force_enable(unsigned long addr) { - u32 h, l; + struct msr val; if (apic_is_disabled) return false; @@ -1998,12 +1999,12 @@ bool __init apic_force_enable(unsigned long addr) * and AMD K7 (Model > 1) or later. */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (!(l & MSR_IA32_APICBASE_ENABLE)) { + rdmsrq(MSR_IA32_APICBASE, val.q); + if (!(val.l & MSR_IA32_APICBASE_ENABLE)) { pr_info("Local APIC disabled by BIOS -- reenabling.\n"); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | addr; - wrmsr(MSR_IA32_APICBASE, l, h); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | addr; + wrmsrq(MSR_IA32_APICBASE, val.q); enabled_via_apicbase = 1; } } @@ -2442,7 +2443,7 @@ static int lapic_suspend(void *data) static void lapic_resume(void *data) { - unsigned int l, h; + struct msr val; unsigned long flags; int maxlvt; @@ -2475,10 +2476,10 @@ static void lapic_resume(void *data) * SMP! We'll need to do this as part of the CPU restore! */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; + wrmsrq(MSR_IA32_APICBASE, val.q); } } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 487ac147e11f..169e373418bb 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -113,7 +113,7 @@ static void init_amd_k5(struct cpuinfo_x86 *c) static void init_amd_k6(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; int mbytes = get_num_physpages() >> (20-PAGE_SHIFT); if (c->x86_model < 6) { @@ -160,13 +160,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 508) mbytes = 508; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0x0000FFFF) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0x0000FFFF) == 0) { unsigned long flags; - l = (1<<0)|((mbytes/4)<<1); + val.l = (1 << 0) | ((mbytes / 4) << 1); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling old style K6 write allocation for %d Mb\n", mbytes); @@ -181,13 +181,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 4092) mbytes = 4092; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0xFFFF0000) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0xFFFF0000) == 0) { unsigned long flags; - l = ((mbytes>>2)<<22)|(1<<16); + val.l = ((mbytes >> 2) << 22) | (1 << 16); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling new style K6 write allocation for %d Mb\n", mbytes); @@ -207,7 +207,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c) static void init_amd_k7(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; /* * Bit 15 of Athlon specific MSR 15, needs to be 0 @@ -228,11 +228,12 @@ static void init_amd_k7(struct cpuinfo_x86 *c) * As per AMD technical note 27212 0.2 */ if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) { - rdmsr(MSR_K7_CLK_CTL, l, h); - if ((l & 0xfff00000) != 0x20000000) { + rdmsrq(MSR_K7_CLK_CTL, val.q); + if ((val.l & 0xfff00000) != 0x20000000) { pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n", - l, ((l & 0x000fffff)|0x20000000)); - wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h); + val.l, ((val.l & 0x000fffff) | 0x20000000)); + val.l = (val.l & 0x000fffff) | 0x20000000; + wrmsrq(MSR_K7_CLK_CTL, val.q); } } @@ -614,12 +615,13 @@ clear_sev: static void early_init_amd(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; if (c->x86 >= 0xf) set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 681d2da49341..513fa1f640f9 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -22,7 +22,7 @@ static void init_c3(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Centaur Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -30,17 +30,17 @@ static void init_c3(struct cpuinfo_x86 *c) /* enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= ACE_FCR; /* enable ACE unit */ - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + /* enable ACE unit */ + wrmsrq(MSR_VIA_FCR, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_VIA_RNG, lo, hi); - lo |= RNG_ENABLE; /* enable RNG unit */ - wrmsr(MSR_VIA_RNG, lo, hi); + rdmsrq(MSR_VIA_RNG, msr); + /* enable RNG unit */ + wrmsrq(MSR_VIA_RNG, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } @@ -52,9 +52,8 @@ static void init_c3(struct cpuinfo_x86 *c) #ifdef CONFIG_X86_32 /* Cyrix III family needs CX8 & PGE explicitly enabled. */ if (c->x86_model >= 6 && c->x86_model <= 13) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= (1<<1 | 1<<7); - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + wrmsrq(MSR_VIA_FCR, msr | (1 << 1 | 1 << 7)); set_cpu_cap(c, X86_FEATURE_CX8); } @@ -115,8 +114,9 @@ static void init_centaur(struct cpuinfo_x86 *c) char *name; u32 fcr_set = 0; u32 fcr_clr = 0; - u32 lo, hi, newlo; + u32 newlo; u32 aa, bb, cc, dd; + struct msr val; #endif early_init_centaur(c); init_intel_cacheinfo(c); @@ -169,15 +169,16 @@ static void init_centaur(struct cpuinfo_x86 *c) name = "??"; } - rdmsr(MSR_IDT_FCR1, lo, hi); - newlo = (lo|fcr_set) & (~fcr_clr); + rdmsrq(MSR_IDT_FCR1, val.q); + newlo = (val.l | fcr_set) & (~fcr_clr); - if (newlo != lo) { + if (newlo != val.l) { pr_info("Centaur FCR was 0x%X now 0x%X\n", - lo, newlo); - wrmsr(MSR_IDT_FCR1, newlo, hi); + val.l, newlo); + val.l = newlo; + wrmsrq(MSR_IDT_FCR1, val.q); } else { - pr_info("Centaur FCR is 0x%X\n", lo); + pr_info("Centaur FCR is 0x%X\n", val.l); } /* Emulate MTRRs using Centaur's MCR. */ set_cpu_cap(c, X86_FEATURE_CENTAUR_MCR); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..cbef2c6c8478 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -339,16 +339,16 @@ bool cpuid_feature(void) static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) { - unsigned long lo, hi; + struct msr val; if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr) return; /* Disable processor serial number: */ - rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi); - lo |= 0x200000; - wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi); + rdmsrq(MSR_IA32_BBL_CR_CTL, val.q); + val.l |= 0x200000; + wrmsrq(MSR_IA32_BBL_CR_CTL, val.q); pr_notice("CPU serial number disabled.\n"); clear_cpu_cap(c, X86_FEATURE_PN); @@ -2299,8 +2299,10 @@ static inline void idt_syscall_init(void) /* May not be marked __init: used by software suspend */ void syscall_init(void) { + struct msr val = { .h = (__USER32_CS << 16) | __KERNEL_CS }; + /* The default user and kernel segments */ - wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS); + wrmsrq(MSR_STAR, val.q); /* * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 3e8891a9caf2..ec51c2b9a257 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -125,11 +125,12 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) static void early_init_hygon(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index abb3984336eb..4297ceb2cb24 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -199,6 +199,41 @@ void intel_unlock_cpuid_leafs(struct cpuinfo_x86 *c) c->cpuid_level = cpuid_eax(0); } +/* + * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 + * structures are populated. + */ +static u32 intel_cpuid_vfm(void) +{ + u32 eax = cpuid_eax(1); + u32 fam = x86_family(eax); + u32 model = x86_model(eax); + + return IFM(fam, model); +} + +u32 intel_get_platform_id(void) +{ + unsigned int val[2]; + + if (x86_hypervisor_present) + return 0; + + /* + * This can be called early. Use CPUID directly instead of + * relying on cpuinfo_x86 which may not be fully initialized. + * The PII does not have MSR_IA32_PLATFORM_ID. Everything + * before _it_ has no microcode (for Linux at least). + */ + if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) + return 0; + + /* get processor flags from MSR 0x17 */ + native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); + + return (val[1] >> 18) & 7; +} + static void early_init_intel(struct cpuinfo_x86 *c) { u64 misc_enable; @@ -542,12 +577,12 @@ static void init_intel(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); if (boot_cpu_has(X86_FEATURE_DS)) { - unsigned int l1, l2; + u64 l; - rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); - if (!(l1 & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) + rdmsrq(MSR_IA32_MISC_ENABLE, l); + if (!(l & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_BTS); - if (!(l1 & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) + if (!(l & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_PEBS); } diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 36e0df4d1342..f916fb4c5d13 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -280,11 +280,11 @@ static void smca_configure(unsigned int bank, unsigned int cpu) u8 *bank_counts = this_cpu_ptr(smca_bank_counts); const struct smca_hwid *s_hwid; unsigned int i, hwid_mcatype; - u32 high, low; + struct msr val; u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank); /* Set appropriate bits in MCA_CONFIG */ - if (!rdmsr_safe(smca_config, &low, &high)) { + if (!rdmsrq_safe(smca_config, &val.q)) { /* * OS is required to set the MCAX bit to acknowledge that it is * now using the new MSR ranges and new registers under each @@ -294,7 +294,7 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * * MCA_CONFIG[MCAX] is bit 32 (0 in the high portion of the MSR.) */ - high |= BIT(0); + val.h |= BIT(0); /* * SMCA sets the Deferred Error Interrupt type per bank. @@ -307,9 +307,9 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * APIC based interrupt. First, check that no interrupt has been * set. */ - if ((low & BIT(5)) && !((high >> 5) & 0x3) && data->dfr_intr_en) { + if ((val.l & BIT(5)) && !((val.h >> 5) & 0x3) && data->dfr_intr_en) { __set_bit(bank, data->dfr_intr_banks); - high |= BIT(5); + val.h |= BIT(5); } /* @@ -324,33 +324,33 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * The OS should set this to inform the platform that the OS is ready * to handle the MCA Thresholding interrupt. */ - if ((low & BIT(10)) && data->thr_intr_en) { + if ((val.l & BIT(10)) && data->thr_intr_en) { __set_bit(bank, data->thr_intr_banks); - high |= BIT(8); + val.h |= BIT(8); } - this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(low & BIT(8)); + this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(val.l & BIT(8)); - if (low & MCI_CONFIG_PADDRV) + if (val.l & MCI_CONFIG_PADDRV) this_cpu_ptr(smca_banks)[bank].paddrv = 1; - wrmsr(smca_config, low, high); + wrmsrq(smca_config, val.q); } - if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) { + if (rdmsrq_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &val.q)) { pr_warn("Failed to read MCA_IPID for bank %d\n", bank); return; } - hwid_mcatype = HWID_MCATYPE(high & MCI_IPID_HWID, - (high & MCI_IPID_MCATYPE) >> 16); + hwid_mcatype = HWID_MCATYPE(val.h & MCI_IPID_HWID, + (val.h & MCI_IPID_MCATYPE) >> 16); for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) { s_hwid = &smca_hwid_mcatypes[i]; if (hwid_mcatype == s_hwid->hwid_mcatype) { this_cpu_ptr(smca_banks)[bank].hwid = s_hwid; - this_cpu_ptr(smca_banks)[bank].id = low; + this_cpu_ptr(smca_banks)[bank].id = val.l; this_cpu_ptr(smca_banks)[bank].sysfs_id = bank_counts[s_hwid->bank_type]++; break; } @@ -432,50 +432,50 @@ static bool lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi) static void threshold_restart_block(void *_tr) { struct thresh_restart *tr = _tr; - u32 hi, lo; + struct msr val; /* sysfs write might race against an offline operation */ if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off) return; - rdmsr(tr->b->address, lo, hi); + rdmsrq(tr->b->address, val.q); /* * Reset error count and overflow bit. * This is done during init or after handling an interrupt. */ - if (hi & MASK_OVERFLOW_HI || tr->set_lvt_off) { - hi &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); - hi |= THRESHOLD_MAX - tr->b->threshold_limit; + if (val.h & MASK_OVERFLOW_HI || tr->set_lvt_off) { + val.h &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); + val.h |= THRESHOLD_MAX - tr->b->threshold_limit; } else if (tr->old_limit) { /* change limit w/o reset */ - int new_count = (hi & THRESHOLD_MAX) + + int new_count = (val.h & THRESHOLD_MAX) + (tr->old_limit - tr->b->threshold_limit); - hi = (hi & ~MASK_ERR_COUNT_HI) | + val.h = (val.h & ~MASK_ERR_COUNT_HI) | (new_count & THRESHOLD_MAX); } /* clear IntType */ - hi &= ~MASK_INT_TYPE_HI; + val.h &= ~MASK_INT_TYPE_HI; if (!tr->b->interrupt_capable) goto done; if (tr->set_lvt_off) { - if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) { + if (lvt_off_valid(tr->b, tr->lvt_off, val.l, val.h)) { /* set new lvt offset */ - hi &= ~MASK_LVTOFF_HI; - hi |= tr->lvt_off << 20; + val.h &= ~MASK_LVTOFF_HI; + val.h |= tr->lvt_off << 20; } } if (tr->b->interrupt_enable) - hi |= INT_TYPE_APIC; + val.h |= INT_TYPE_APIC; done: - hi |= MASK_COUNT_EN_HI; - wrmsr(tr->b->address, lo, hi); + val.h |= MASK_COUNT_EN_HI; + wrmsrq(tr->b->address, val.q); } static void threshold_restart_bank(unsigned int bank, bool intr_en) @@ -726,7 +726,8 @@ static void smca_enable_interrupt_vectors(void) void mce_amd_feature_init(struct cpuinfo_x86 *c) { unsigned int bank, block, cpu = smp_processor_id(); - u32 low = 0, high = 0, address = 0; + struct msr val = { .q = 0 }; + u32 address = 0; int offset = -1; amd_apply_cpu_quirks(c); @@ -746,21 +747,21 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c) disable_err_thresholding(c, bank); for (block = 0; block < NR_BLOCKS; ++block) { - address = get_block_address(address, low, high, bank, block, cpu); + address = get_block_address(address, val.l, val.h, bank, block, cpu); if (!address) break; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) break; - if (!(high & MASK_VALID_HI)) + if (!(val.h & MASK_VALID_HI)) continue; - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) continue; - offset = prepare_threshold_block(bank, block, address, offset, high); + offset = prepare_threshold_block(bank, block, address, offset, val.h); } } } @@ -1083,24 +1084,24 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb u32 address) { struct threshold_block *b = NULL; - u32 low, high; + struct msr val; int err; if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS)) return 0; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) return 0; - if (!(high & MASK_VALID_HI)) { + if (!(val.h & MASK_VALID_HI)) { if (block) goto recurse; else return 0; } - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) goto recurse; b = kzalloc_obj(struct threshold_block); @@ -1112,7 +1113,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb b->cpu = cpu; b->address = address; b->interrupt_enable = 0; - b->interrupt_capable = lvt_interrupt_supported(bank, high); + b->interrupt_capable = lvt_interrupt_supported(bank, val.h); b->threshold_limit = get_thr_limit(); if (b->interrupt_capable) { @@ -1124,13 +1125,13 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb list_add(&b->miscj, &tb->miscj); - mce_threshold_block_init(b, (high & MASK_LVTOFF_HI) >> 20); + mce_threshold_block_init(b, (val.h & MASK_LVTOFF_HI) >> 20); err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(cpu, bank, b)); if (err) goto out_free; recurse: - address = get_block_address(address, low, high, bank, ++block, cpu); + address = get_block_address(address, val.l, val.h, bank, ++block, cpu); if (!address) return 0; diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 9bba1e2f03af..017aaf57ba47 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1866,7 +1866,7 @@ static void __mcheck_cpu_init_generic(void) rdmsrq(MSR_IA32_MCG_CAP, cap); if (cap & MCG_CTL_P) - wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); + wrmsrq(MSR_IA32_MCG_CTL, ~0ULL); } static void __mcheck_cpu_init_prepare_banks(void) diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c index 2272ad53fc33..eb99f384d747 100644 --- a/arch/x86/kernel/cpu/mce/p5.c +++ b/arch/x86/kernel/cpu/mce/p5.c @@ -23,16 +23,16 @@ int mce_p5_enabled __read_mostly; /* Machine check handler for Pentium class Intel CPUs: */ noinstr void pentium_machine_check(struct pt_regs *regs) { - u32 loaddr, hi, lotype; + u64 addr, type; instrumentation_begin(); - rdmsr(MSR_IA32_P5_MC_ADDR, loaddr, hi); - rdmsr(MSR_IA32_P5_MC_TYPE, lotype, hi); + rdmsrq(MSR_IA32_P5_MC_ADDR, addr); + rdmsrq(MSR_IA32_P5_MC_TYPE, type); pr_emerg("CPU#%d: Machine Check Exception: 0x%8X (type 0x%8X).\n", - smp_processor_id(), loaddr, lotype); + smp_processor_id(), (u32)addr, (u32)type); - if (lotype & (1<<5)) { + if (type & (1<<5)) { pr_emerg("CPU#%d: Possible thermal failure (CPU on fire ?).\n", smp_processor_id()); } @@ -44,7 +44,7 @@ noinstr void pentium_machine_check(struct pt_regs *regs) /* Set up machine check reporting for processors with Intel style MCE: */ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) { - u32 l, h; + u64 q; /* Default P5 to off as its often misconnected: */ if (!mce_p5_enabled) @@ -55,8 +55,8 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) return; /* Read registers before enabling: */ - rdmsr(MSR_IA32_P5_MC_ADDR, l, h); - rdmsr(MSR_IA32_P5_MC_TYPE, l, h); + rdmsrq(MSR_IA32_P5_MC_ADDR, q); + rdmsrq(MSR_IA32_P5_MC_TYPE, q); pr_info("Intel old style machine check architecture supported.\n"); /* Enable MCE: */ diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c index 6c99f2941909..7040243533d9 100644 --- a/arch/x86/kernel/cpu/mce/winchip.c +++ b/arch/x86/kernel/cpu/mce/winchip.c @@ -28,12 +28,12 @@ noinstr void winchip_machine_check(struct pt_regs *regs) /* Set up machine check reporting on the Winchip C6 series */ void winchip_mcheck_init(struct cpuinfo_x86 *c) { - u32 lo, hi; + struct msr val; - rdmsr(MSR_IDT_FCR1, lo, hi); - lo |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ - lo &= ~(1<<4); /* Enable MCE */ - wrmsr(MSR_IDT_FCR1, lo, hi); + rdmsrq(MSR_IDT_FCR1, val.q); + val.l |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ + val.l &= ~(1<<4); /* Enable MCE */ + wrmsrq(MSR_IDT_FCR1, val.q); cr4_set_bits(X86_CR4_MCE); diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index f4a444e6114d..1142183c950c 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -121,42 +121,6 @@ static inline unsigned int exttable_size(struct extended_sigtable *et) return et->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE; } - -/* - * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 - * structures are populated. - */ -static u32 intel_cpuid_vfm(void) -{ - u32 eax = cpuid_eax(1); - u32 fam = x86_family(eax); - u32 model = x86_model(eax); - - return IFM(fam, model); -} - -u32 intel_get_platform_id(void) -{ - unsigned int val[2]; - - if (x86_hypervisor_present) - return 0; - - /* - * This can be called early. Use CPUID directly instead of - * relying on cpuinfo_x86 which may not be fully initialized. - * The PII does not have MSR_IA32_PLATFORM_ID. Everything - * before _it_ has no microcode (for Linux at least). - */ - if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) - return 0; - - /* get processor flags from MSR 0x17 */ - native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); - - return (val[1] >> 18) & 7; -} - void intel_collect_cpu_info(struct cpu_signature *sig) { sig->sig = cpuid_eax(1); diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 9c01d2562b7a..f452e8ce4cef 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -725,13 +725,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) static void clear_closid_rmid(int cpu) { struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); + struct msr val = { + .l = RESCTRL_RESERVED_RMID, + .h = RESCTRL_RESERVED_CLOSID + }; state->default_closid = RESCTRL_RESERVED_CLOSID; state->default_rmid = RESCTRL_RESERVED_RMID; state->cur_closid = RESCTRL_RESERVED_CLOSID; state->cur_rmid = RESCTRL_RESERVED_RMID; - wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID, - RESCTRL_RESERVED_CLOSID); + wrmsrq(MSR_IA32_PQR_ASSOC, val.q); } static int resctrl_arch_online_cpu(unsigned int cpu) diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 569894d6e5c8..d096ba7d1b88 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -136,7 +136,7 @@ static int logical_rmid_to_physical_rmid(int cpu, int lrmid) static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) { - u64 msr_val; + struct msr msr_val = { .l = eventid, .h = prmid }; /* * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured @@ -146,15 +146,15 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62) * are error bits. */ - wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } @@ -283,7 +283,10 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, static int __cntr_id_read(u32 cntr_id, u64 *val) { - u64 msr_val; + struct msr msr_val = { + .l = ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, + .h = cntr_id + }; /* * QM_EVTSEL Register definition: @@ -306,15 +309,15 @@ static int __cntr_id_read(u32 cntr_id, u64 *val) * ID is set in the QM_EVTSEL.RMID field. The RMID_VAL_UNAVAIL bit * is set if the counter data is unavailable. */ - wrmsr(MSR_IA32_QM_EVTSEL, ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, cntr_id); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index de580eca3363..d7caab0409b6 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -241,16 +241,16 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) int resctrl_arch_measure_cycles_lat_fn(void *_plr) { struct pseudo_lock_region *plr = _plr; - u32 saved_low, saved_high; unsigned long i; u64 start, end; void *mem_r; + u64 saved; local_irq_disable(); /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); mem_r = READ_ONCE(plr->kmem); /* @@ -267,7 +267,7 @@ int resctrl_arch_measure_cycles_lat_fn(void *_plr) end = rdtsc_ordered(); trace_pseudo_lock_mem_latency((u32)(end - start)); } - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); @@ -312,11 +312,11 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; struct perf_event *miss_event, *hit_event; int hit_pmcnum, miss_pmcnum; - u32 saved_low, saved_high; unsigned int line_size; unsigned int size; unsigned long i; void *mem_r; + u64 saved; u64 tmp; miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, @@ -346,7 +346,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); /* Initialize rest of local variables */ @@ -405,7 +405,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, */ rmb(); /* Re-enable hardware prefetchers */ - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); out_hit: perf_event_release_kernel(hit_event); diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 1fdcd69c625c..c670fbb6ee50 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -24,7 +24,8 @@ static void early_init_transmeta(struct cpuinfo_x86 *c) static void init_transmeta(struct cpuinfo_x86 *c) { - unsigned int cap_mask, uk, max, dummy; + u64 msr; + unsigned int max, dummy; unsigned int cms_rev1, cms_rev2; unsigned int cpu_rev, cpu_freq = 0, cpu_flags, new_cpu_rev; char cpu_info[65]; @@ -86,10 +87,10 @@ static void init_transmeta(struct cpuinfo_x86 *c) } /* Unhide possibly hidden capability flags */ - rdmsr(0x80860004, cap_mask, uk); - wrmsr(0x80860004, ~0, uk); + rdmsrq(0x80860004, msr); + wrmsrq(0x80860004, msr | ~0U); c->x86_capability[CPUID_1_EDX] = cpuid_edx(0x00000001); - wrmsr(0x80860004, cap_mask, uk); + wrmsrq(0x80860004, msr); /* All Transmeta CPUs have a constant TSC */ set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c index 761aef5590ac..fe504fd43c77 100644 --- a/arch/x86/kernel/cpu/zhaoxin.c +++ b/arch/x86/kernel/cpu/zhaoxin.c @@ -21,7 +21,7 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -29,19 +29,17 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) /* Enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable ACE unit */ - lo |= ACE_FCR; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* Enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable RNG unit */ - lo |= RNG_ENABLE; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 584fb9913be4..2ed26134c0d1 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -294,15 +294,15 @@ void fpu_free_guest_fpstate(struct fpu_guest *gfpu) } EXPORT_SYMBOL_FOR_KVM(fpu_free_guest_fpstate); -/* - * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable - * @guest_fpu: Pointer to the guest FPU container - * @xfeatures: Features requested by guest CPUID - * - * Enable all dynamic xfeatures according to guest perm and requested CPUID. - * - * Return: 0 on success, error code otherwise - */ +/** + * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable + * @guest_fpu: Pointer to the guest FPU container + * @xfeatures: Features requested by guest CPUID + * + * Enable all dynamic xfeatures according to guest perm and requested CPUID. + * + * Return: 0 on success, error code otherwise + */ int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) { lockdep_assert_preemption_enabled(); diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index ce10ae4b298b..723347e2cf7f 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1221,11 +1221,11 @@ static void __init check_system_tsc_reliable(void) if (is_geode_lx()) { /* RTSC counts during suspend */ #define RTSC_SUSP 0x100 - unsigned long res_low, res_high; + u64 res; - rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high); + rdmsrq_safe(MSR_GEODE_BUSCONT_CONF0, &res); /* Geode_LX - the OLPC CPU has a very reliable TSC */ - if (res_low & RTSC_SUSP) + if (res & RTSC_SUSP) tsc_clocksource_reliable = 1; } #endif diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 48e6cc1cb017..d74743c8d2a4 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -165,7 +165,8 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = { */ unsigned long cpu_khz_from_msr(void) { - u32 lo, hi, ratio, freq, tscref; + u32 ratio, freq, tscref; + struct msr val; const struct freq_desc *freq_desc; const struct x86_cpu_id *id; const struct muldiv *md; @@ -178,16 +179,16 @@ unsigned long cpu_khz_from_msr(void) freq_desc = (struct freq_desc *)id->driver_data; if (freq_desc->use_msr_plat) { - rdmsr(MSR_PLATFORM_INFO, lo, hi); - ratio = (lo >> 8) & 0xff; + rdmsrq(MSR_PLATFORM_INFO, val.q); + ratio = (val.l >> 8) & 0xff; } else { - rdmsr(MSR_IA32_PERF_STATUS, lo, hi); - ratio = (hi >> 8) & 0x1f; + rdmsrq(MSR_IA32_PERF_STATUS, val.q); + ratio = (val.h >> 8) & 0x1f; } /* Get FSB FREQ ID */ - rdmsr(MSR_FSB_FREQ, lo, hi); - index = lo & freq_desc->mask; + rdmsrq(MSR_FSB_FREQ, val.q); + index = val.l & freq_desc->mask; md = &freq_desc->muldiv[index]; /* diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d3..65a2de82ecd2 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call <uprobe-trampoline> + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ unlock: mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index 0c6627773c80..7b6cfc2c0970 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -15,7 +15,7 @@ static void __rdmsr_on_cpu(void *info) else reg = &rv->reg; - rdmsr(rv->msr_no, reg->l, reg->h); + rdmsrq(rv->msr_no, reg->q); } static void __wrmsr_on_cpu(void *info) @@ -28,7 +28,7 @@ static void __wrmsr_on_cpu(void *info) else reg = &rv->reg; - wrmsr(rv->msr_no, reg->l, reg->h); + wrmsrq(rv->msr_no, reg->q); } int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) @@ -121,7 +121,7 @@ static void __rdmsr_safe_on_cpu(void *info) { struct msr_info_completion *rv = info; - rv->msr.err = rdmsr_safe(rv->msr.msr_no, &rv->msr.reg.l, &rv->msr.reg.h); + rv->msr.err = rdmsrq_safe(rv->msr.msr_no, &rv->msr.reg.q); complete(&rv->done); } @@ -129,7 +129,7 @@ static void __wrmsr_safe_on_cpu(void *info) { struct msr_info *rv = info; - rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h); + rv->err = wrmsrq_safe(rv->msr_no, rv->reg.q); } int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index bd5d101c5c37..5a15bffe6574 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -111,7 +111,7 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, use_gbpage = info->direct_gbpages; /* Don't use gbpage if it maps more than the requested region. */ - /* at the begining: */ + /* at the beginning: */ use_gbpage &= ((addr & ~PUD_MASK) == 0); /* ... or at the end: */ use_gbpage &= ((next & ~PUD_MASK) == 0); diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index acdb8dcaeb52..9749820f68f7 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -189,7 +189,7 @@ static const char *__init pci_mmcfg_intel_945(void) static const char *__init pci_mmcfg_amd_fam10h(void) { - u32 low, high, address; + u32 address; u64 base, msr; int i; unsigned segnbits = 0, busnbits, end_bus; @@ -198,13 +198,9 @@ static const char *__init pci_mmcfg_amd_fam10h(void) return NULL; address = MSR_FAM10H_MMIO_CONF_BASE; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &msr)) return NULL; - msr = high; - msr <<= 32; - msr |= low; - /* ECAM is not enabled */ if (!(msr & FAM10H_MMIO_CONF_ENABLE)) return NULL; diff --git a/arch/x86/platform/intel-quark/imr.c b/arch/x86/platform/intel-quark/imr.c index ee25b032c0b3..9238473842c7 100644 --- a/arch/x86/platform/intel-quark/imr.c +++ b/arch/x86/platform/intel-quark/imr.c @@ -291,8 +291,8 @@ static inline int imr_address_overlap(phys_addr_t addr, struct imr_regs *imr) * * @base: physical base address of region aligned to 1KiB. * @size: physical size of region in bytes must be aligned to 1KiB. - * @read_mask: read access mask. - * @write_mask: write access mask. + * @rmask: read access mask. + * @wmask: write access mask. * @return: zero on success or negative value indicating error. */ int imr_add_range(phys_addr_t base, size_t size, diff --git a/arch/x86/platform/olpc/olpc-xo1-sci.c b/arch/x86/platform/olpc/olpc-xo1-sci.c index 30751b42d54e..97eb4738d602 100644 --- a/arch/x86/platform/olpc/olpc-xo1-sci.c +++ b/arch/x86/platform/olpc/olpc-xo1-sci.c @@ -311,12 +311,13 @@ static int xo1_sci_resume(struct platform_device *pdev) static int setup_sci_interrupt(struct platform_device *pdev) { - u32 lo, hi; + u64 msr; + u32 lo; u32 sts; int r; - rdmsr(0x51400020, lo, hi); - sci_irq = (lo >> 20) & 15; + rdmsrq(0x51400020, msr); + sci_irq = (msr >> 20) & 15; if (sci_irq) { dev_info(&pdev->dev, "SCI is mapped to IRQ %d\n", sci_irq); @@ -324,8 +325,8 @@ static int setup_sci_interrupt(struct platform_device *pdev) /* Zero means masked */ dev_info(&pdev->dev, "SCI unmapped. Mapping to IRQ 3\n"); sci_irq = 3; - lo |= 0x00300000; - wrmsrq(0x51400020, lo); + msr |= 0x00300000; + wrmsrq(0x51400020, msr); } /* Select level triggered in PIC */ diff --git a/arch/x86/ras/Kconfig b/arch/x86/ras/Kconfig deleted file mode 100644 index 7488c715427e..000000000000 --- a/arch/x86/ras/Kconfig +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -config RAS_CEC - bool "Correctable Errors Collector" - depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS - help - This is a small cache which collects correctable memory errors per 4K - page PFN and counts their repeated occurrence. Once the counter for a - PFN overflows, we try to soft-offline that page as we take it to mean - that it has reached a relatively high error count and would probably - be best if we don't use it anymore. - - Bear in mind that this is absolutely useless if your platform doesn't - have ECC DIMMs and doesn't have DRAM ECC checking enabled in the BIOS. - -config RAS_CEC_DEBUG - bool "CEC debugging machinery" - default n - depends on RAS_CEC - help - Add extra files to (debugfs)/ras/cec to test the correctable error - collector feature. "pfn" is a writable file that allows user to - simulate an error in a particular page frame. "array" is a read-only - file that dumps out the current state of all pages logged so far. diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index e3bd6436669b..94ab80197c2e 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -261,11 +261,11 @@ static void ie31200_clear_error_info(struct mem_ctl_info *mci) * the ECC error log registers in all memory controllers. */ if (cfg->msr_clear_eccerrlog_offset) { - if (wrmsr_safe(cfg->msr_clear_eccerrlog_offset, - cfg->reg_eccerrlog_ce_mask | - cfg->reg_eccerrlog_ce_ovfl_mask | - cfg->reg_eccerrlog_ue_mask | - cfg->reg_eccerrlog_ue_ovfl_mask, 0) < 0) + if (wrmsrq_safe(cfg->msr_clear_eccerrlog_offset, + cfg->reg_eccerrlog_ce_mask | + cfg->reg_eccerrlog_ce_ovfl_mask | + cfg->reg_eccerrlog_ue_mask | + cfg->reg_eccerrlog_ue_ovfl_mask) < 0) ie31200_printk(KERN_ERR, "Failed to wrmsr.\n"); return; diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index bd252cb3c38e..7aee57a25b68 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -806,7 +806,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) struct mce *m = (struct mce *)data; struct mce_hw_err *err = to_mce_hw_err(m); unsigned int fam = x86_family(m->cpuid); - u32 mca_config_lo = 0, dummy; + u64 mca_config = 0; int ecc; if (m->kflags & MCE_HANDLED_CEC) @@ -826,9 +826,9 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) ((m->status & MCI_STATUS_PCC) ? "PCC" : "-")); if (boot_cpu_has(X86_FEATURE_SMCA)) { - rdmsr_safe(MSR_AMD64_SMCA_MCx_CONFIG(m->bank), &mca_config_lo, &dummy); + rdmsrq_safe(MSR_AMD64_SMCA_MCx_CONFIG(m->bank), &mca_config); - if (mca_config_lo & MCI_CONFIG_MCAX) + if (mca_config & MCI_CONFIG_MCAX) pr_cont("|%s", ((m->status & MCI_STATUS_TCC) ? "TCC" : "-")); pr_cont("|%s", ((m->status & MCI_STATUS_SYNDV) ? "SyndV" : "-")); @@ -863,7 +863,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) if (m->status & MCI_STATUS_SYNDV) { pr_cont(", Syndrome: 0x%016llx\n", m->synd); - if (mca_config_lo & MCI_CONFIG_FRUTEXT) { + if (mca_config & MCI_CONFIG_FRUTEXT) { char frutext[17]; frutext[16] = '\0'; diff --git a/drivers/hwmon/hwmon-vid.c b/drivers/hwmon/hwmon-vid.c index 2df4956296ed..dee42c163d92 100644 --- a/drivers/hwmon/hwmon-vid.c +++ b/drivers/hwmon/hwmon-vid.c @@ -237,16 +237,17 @@ static struct vrm_model vrm_models[] = { */ static u8 get_via_model_d_vrm(void) { - unsigned int vid, brand, __maybe_unused dummy; + u64 msr; + unsigned int vid, brand; static const char *brands[4] = { "C7-M", "C7", "Eden", "C7-D" }; - rdmsr(0x198, dummy, vid); - vid &= 0xff; + rdmsrq(0x198, msr); + vid = (msr >> 32) & 0xff; - rdmsr(0x1154, brand, dummy); - brand = ((brand >> 4) ^ (brand >> 2)) & 0x03; + rdmsrq(0x1154, msr); + brand = ((msr >> 4) ^ (msr >> 2)) & 0x03; if (vid > 0x3f) { pr_info("Using %d-bit VID table for VIA %s CPU\n", diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 2ae3be7fa633..c8becb458da2 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -25,6 +25,15 @@ #define REG(cpu, x) (realtek_ictl_base[cpu] + x) +struct realtek_ictl_output { + struct fwnode_handle *fwnode; + struct irq_domain *domain; + unsigned int parent_irq; + unsigned int parent_hwirq; + unsigned int index; + u32 mask; +}; + static DEFINE_RAW_SPINLOCK(irq_lock); static void __iomem *realtek_ictl_base[NR_CPUS]; @@ -105,53 +114,141 @@ static struct irq_chip realtek_ictl_irq = { .irq_set_affinity = realtek_ictl_irq_affinity, }; -static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) +static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw_irq) { + struct realtek_ictl_output *output = d->host_data; unsigned int cpu; irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq); guard(raw_spinlock_irqsave)(&irq_lock); + output->mask |= BIT(hw_irq); for_each_present_cpu(cpu) - write_irr(cpu, hw, 1); + write_irr(cpu, hw_irq, output->parent_hwirq - 1); return 0; } +static int intc_select(struct irq_domain *d, struct irq_fwspec *fwspec, + enum irq_domain_bus_token bus_token) +{ + struct realtek_ictl_output *output = d->host_data; + unsigned int index = 0; + + if (fwspec->fwnode != output->fwnode) + return false; + + if (fwspec->param_count == 2) + index = fwspec->param[1]; + + return index == output->index; +} + static const struct irq_domain_ops irq_domain_ops = { - .map = intc_map, - .xlate = irq_domain_xlate_onecell, + .map = intc_map, + .select = intc_select, + .xlate = irq_domain_xlate_onecell, }; static void realtek_irq_dispatch(struct irq_desc *desc) { + struct realtek_ictl_output *output = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned int cpu = smp_processor_id(); - struct irq_domain *domain; unsigned long pending; - unsigned int soc_int; + unsigned int hw_irq; chained_irq_enter(chip, desc); - pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR)); + pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR)) & output->mask; if (unlikely(!pending)) { spurious_interrupt(); goto out; } - domain = irq_desc_get_handler_data(desc); - for_each_set_bit(soc_int, &pending, RTL_ICTL_NUM_INPUTS) - generic_handle_domain_irq(domain, soc_int); + for_each_set_bit(hw_irq, &pending, RTL_ICTL_NUM_INPUTS) + generic_handle_domain_irq(output->domain, hw_irq); out: chained_irq_exit(chip, desc); } -static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +static int __init realtek_setup_parents(struct device_node *node) { + int p, cnt, err, parent_irq, num_parents = of_irq_count(node); + struct realtek_ictl_output *output; + struct irq_data *parent_data; struct of_phandle_args oirq; struct irq_domain *domain; - int cpu, parent_irq; + + cnt = max(1, num_parents); + output = kcalloc(cnt, sizeof(*output), GFP_KERNEL); + if (!output) + return -ENOMEM; + + for (p = 0; p < cnt; p++) { + if (WARN_ON(!num_parents)) { + /* + * If DT contains no parent interrupts, assume MIPS IRQ 2 (HW0) is + * connected to the first output. This is the case for all known hardware. + */ + oirq.np = of_find_compatible_node(NULL, NULL, + "mti,cpu-interrupt-controller"); + if (!oirq.np) { + err = -EINVAL; + goto err_out; + } + + oirq.args_count = 1; + oirq.args[0] = 2; + parent_irq = irq_create_of_mapping(&oirq); + of_node_put(oirq.np); + } else { + parent_irq = of_irq_get(node, p); + } + + if (parent_irq <= 0) { + err = parent_irq ? parent_irq : -ENODEV; + goto err_out; + } + + parent_data = irq_get_irq_data(parent_irq); + if (!parent_data) { + err = -EINVAL; + goto err_out; + } + + domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, + &irq_domain_ops, &output[p]); + if (!domain) { + err = -ENOMEM; + goto err_out; + } + + output[p].domain = domain; + output[p].fwnode = of_fwnode_handle(node); + output[p].index = p; + output[p].parent_irq = parent_irq; + output[p].parent_hwirq = irqd_to_hwirq(parent_data); + irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, &output[p]); + } + + return 0; + +err_out: + while (p--) { + irq_set_chained_handler_and_data(output[p].parent_irq, NULL, NULL); + irq_domain_remove(output[p].domain); + } + + kfree(output); + + return err; +} + +static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +{ + unsigned int cpu; for_each_present_cpu(cpu) { realtek_ictl_base[cpu] = of_iomap(node, cpu); @@ -165,36 +262,7 @@ static int __init realtek_rtl_of_init(struct device_node *node, struct device_no } } - if (WARN_ON(!of_irq_count(node))) { - /* - * If DT contains no parent interrupts, assume MIPS CPU IRQ 2 - * (HW0) is connected to the first output. This is the case for - * all known hardware anyway. "interrupt-map" is deprecated, so - * don't bother trying to parse that. - */ - oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller"); - oirq.args_count = 1; - oirq.args[0] = 2; - - parent_irq = irq_create_of_mapping(&oirq); - - of_node_put(oirq.np); - } else { - parent_irq = of_irq_get(node, 0); - } - - if (parent_irq < 0) - return parent_irq; - else if (!parent_irq) - return -ENODEV; - - domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); - if (!domain) - return -ENOMEM; - - irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain); - - return 0; + return realtek_setup_parents(node); } IRQCHIP_DECLARE(realtek_rtl_intc, "realtek,rtl-intc", realtek_rtl_of_init); diff --git a/drivers/ras/Kconfig b/drivers/ras/Kconfig index fc4f4bb94a4c..070e17639faa 100644 --- a/drivers/ras/Kconfig +++ b/drivers/ras/Kconfig @@ -31,7 +31,29 @@ menuconfig RAS if RAS -source "arch/x86/ras/Kconfig" +config RAS_CEC + bool "Correctable Errors Collector" + depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS + help + This is a small cache which collects correctable memory errors per 4K + page PFN and counts their repeated occurrence. Once the counter for a + PFN overflows, we try to soft-offline that page as we take it to mean + that it has reached a relatively high error count and would probably + be best if we don't use it anymore. + + Bear in mind that this is absolutely useless if your platform doesn't + have ECC DIMMs and doesn't have DRAM ECC checking enabled in the BIOS. + +config RAS_CEC_DEBUG + bool "CEC debugging machinery" + default n + depends on RAS_CEC + help + Add extra files to (debugfs)/ras/cec to test the correctable error + collector feature. "pfn" is a writable file that allows user to + simulate an error in a particular page frame. "array" is a read-only + file that dumps out the current state of all pages logged so far. + source "drivers/ras/amd/atl/Kconfig" config RAS_FMPM diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index 5c6d9799f4e7..a6b03cfba0e2 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_VSYSCALL_H #define __ASM_GENERIC_VSYSCALL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __arch_get_vdso_u_time_data static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) @@ -30,6 +30,6 @@ static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vd } #endif /* __arch_sync_vdso_time_data */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __ASM_GENERIC_VSYSCALL_H */ diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index a80d38e752d2..a67ba1b32d16 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,6 +80,7 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; diff --git a/include/linux/sched.h b/include/linux/sched.h index bc40c2052caf..bcd101db428e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -550,7 +550,6 @@ struct sched_statistics { s64 exec_max; u64 slice_max; - u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; @@ -563,8 +562,6 @@ struct sched_statistics { u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; - u64 nr_wakeups_passive; - u64 nr_wakeups_idle; #ifdef CONFIG_SCHED_CORE u64 core_forceidle_sum; @@ -575,6 +572,7 @@ struct sched_statistics { struct sched_entity { /* For load-balancing: */ struct load_weight load; + struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec..18be159bbc34 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -186,9 +186,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +235,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 5977723fb3b5..09897f76ae07 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -2,7 +2,7 @@ #ifndef __VDSO_DATAPAGE_H #define __VDSO_DATAPAGE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -176,7 +176,7 @@ enum vdso_pages { VDSO_NR_PAGES }; -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ #ifdef CONFIG_VDSO_GETRANDOM #define __vdso_u_rng_data PROVIDE(vdso_u_rng_data = vdso_u_data + 2 * PAGE_SIZE); @@ -197,6 +197,6 @@ enum vdso_pages { __vdso_u_arch_data \ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_DATAPAGE_H */ diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index a3bf4f1c0d37..65151b681c4f 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -2,7 +2,7 @@ #ifndef __VDSO_HELPERS_H #define __VDSO_HELPERS_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/barrier.h> #include <vdso/datapage.h> @@ -111,6 +111,6 @@ static __always_inline void vdso_write_end(struct vdso_time_data *vd) vdso_write_seq_end(&vc[CS_RAW]); } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_HELPERS_H */ diff --git a/include/vdso/processor.h b/include/vdso/processor.h index fbe8265ea3c4..cc781912a696 100644 --- a/include/vdso/processor.h +++ b/include/vdso/processor.h @@ -5,10 +5,10 @@ #ifndef __VDSO_PROCESSOR_H #define __VDSO_PROCESSOR_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/vdso/processor.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __VDSO_PROCESSOR_H */ diff --git a/include/vdso/vsyscall.h b/include/vdso/vsyscall.h index b0fdc9c6bf43..c5c2a2c07857 100644 --- a/include/vdso/vsyscall.h +++ b/include/vdso/vsyscall.h @@ -2,13 +2,13 @@ #ifndef __VDSO_VSYSCALL_H #define __VDSO_VSYSCALL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/vdso/vsyscall.h> unsigned long vdso_update_begin(void); void vdso_update_end(unsigned long flags); -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_VSYSCALL_H */ diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index c1de9a17cbc1..48f5ceeeb8a7 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4124,6 +4124,28 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) return changed; } +/* + * Returns the number of CPUs available for this cgroup. + * + * This only really works for cgroup-v2 where all the controllers are mounted + * in the same hierarchy. If not cgroup-v2 or no cpuset controller is + * configured it reverts to num_online_cpus(). + */ +int cpuset_num_cpus(struct cgroup *cgrp) +{ + int nr = num_online_cpus(); + struct cpuset *cs; + + if (is_in_v2_mode()) { + guard(rcu)(); + cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); + if (cs) + nr = cpumask_weight(cs->effective_cpus); + } + + return nr; +} + void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); diff --git a/kernel/events/core.c b/kernel/events/core.c index 5d496acd4e1b..5f6390369475 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7791,10 +7791,20 @@ unsigned long perf_misc_flags(struct perf_event *event, unsigned long perf_instruction_pointer(struct perf_event *event, struct pt_regs *regs) { - if (should_sample_guest(event)) - return perf_guest_get_ip(); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (should_sample_guest(event)) { + return event->attr.exclude_kernel && + !(perf_guest_state() & PERF_GUEST_USER) ? + 0 : perf_guest_get_ip(); + } - return perf_arch_instruction_pointer(regs); + return event->attr.exclude_kernel && !user_mode(regs) ? + 0 : perf_arch_instruction_pointer(regs); } static void @@ -7828,10 +7838,22 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, } static void perf_sample_regs_intr(struct perf_regs *regs_intr, - struct pt_regs *regs) + struct pt_regs *regs, + bool exclude_kernel) { - regs_intr->regs = regs; - regs_intr->abi = perf_reg_abi(current); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (exclude_kernel && !user_mode(regs)) { + regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE; + regs_intr->regs = NULL; + } else { + regs_intr->regs = regs; + regs_intr->abi = perf_reg_abi(current); + } } @@ -8722,7 +8744,8 @@ void perf_prepare_sample(struct perf_sample_data *data, /* regs dump ABI info */ int size = sizeof(u64); - perf_sample_regs_intr(&data->regs_intr, regs); + perf_sample_regs_intr(&data->regs_intr, regs, + event->attr.exclude_kernel); if (data->regs_intr.regs) { u64 mask = event->attr.sample_regs_intr; @@ -13909,7 +13932,9 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel) { + if (!attr.exclude_kernel || + ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && + !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); if (err) return err; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..b5c516168f84 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1806,14 +1806,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1817,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; diff --git a/kernel/fork.c b/kernel/fork.c index 94e021eabf1d..175c73bbe2bf 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index f15c9f1223bb..57c819da30c2 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1658,7 +1658,7 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, for (i = 0; i < nr_irqs; i++) { ret = irq_domain_trim_hierarchy(virq + i); if (ret) - goto out_free_irq_data; + goto out_free_irqs; } for (i = 0; i < nr_irqs; i++) @@ -1666,6 +1666,8 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, return virq; +out_free_irqs: + irq_domain_free_irqs_hierarchy(domain, virq, nr_irqs); out_free_irq_data: irq_domain_free_irq_data(virq, nr_irqs); out_free_desc: @@ -1963,7 +1965,7 @@ EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent); static void __irq_domain_deactivate_irq(struct irq_data *irq_data) { - if (irq_data && irq_data->domain) { + if (irq_data->domain) { struct irq_domain *domain = irq_data->domain; if (domain->ops->deactivate) @@ -1977,7 +1979,7 @@ static int __irq_domain_activate_irq(struct irq_data *irqd, bool reserve) { int ret = 0; - if (irqd && irqd->domain) { + if (irqd->domain) { struct irq_domain *domain = irqd->domain; if (irqd->parent_data) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index f73fda08417a..2fbff2618a1e 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -2704,11 +2704,7 @@ static int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state return -ENODEV; if (chip->irq_get_irqchip_state) break; -#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY - data = data->parent_data; -#else - data = NULL; -#endif + data = irqd_get_parent_data(data); } while (data); if (data) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..2e7cde033a31 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5657,11 +5657,8 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat); */ static inline void prefetch_curr_exec_start(struct task_struct *p) { -#ifdef CONFIG_FAIR_GROUP_SCHED - struct sched_entity *curr = p->se.cfs_rq->curr; -#else struct sched_entity *curr = task_rq(p)->cfs.curr; -#endif + prefetch(curr); prefetch(&curr->exec_start); } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0f858b98c9aa..a3003b0f4522 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2733,15 +2733,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf) */ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags) { + struct task_struct *donor = rq->donor; /* * Can only get preempted by stop-class, and those should be * few and short lived, doesn't really make sense to push * anything away for that. */ - if (p->sched_class != &dl_sched_class) + if (p->sched_class != &dl_sched_class || + donor->sched_class != &dl_sched_class) return; - if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { + if (dl_entity_preempt(&p->dl, &donor->dl)) { resched_curr(rq); return; } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 40584b27ea0c..72236db67983 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -633,6 +633,82 @@ static void debugfs_fair_server_init(void) } } +#ifdef CONFIG_FAIR_GROUP_SCHED +static int cgroup_mode = 2; + +/* See __sched_cgroup_mode_update(). */ +static const char *cgroup_mode_str[] = { + "up", + "smp", + "concur", + "max", + "tasks", +}; + +static int sched_cgroup_mode(const char *str) +{ + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (!strcmp(str, cgroup_mode_str[i])) + return i; + } + return -EINVAL; +} + +static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char buf[16]; + int mode; + + if (cnt > 15) + cnt = 15; + + if (copy_from_user(buf, ubuf, cnt)) + return -EFAULT; + + buf[cnt] = 0; + mode = sched_cgroup_mode(strstrip(buf)); + if (mode < 0) + return mode; + + __sched_cgroup_mode_update(mode); + WRITE_ONCE(cgroup_mode, mode); + + *ppos += cnt; + return cnt; +} + +static int sched_cgroup_show(struct seq_file *m, void *v) +{ + int mode = READ_ONCE(cgroup_mode); + + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (mode == i) + seq_puts(m, "("); + seq_puts(m, cgroup_mode_str[i]); + if (mode == i) + seq_puts(m, ")"); + + seq_puts(m, " "); + } + seq_puts(m, "\n"); + return 0; +} + +static int sched_cgroup_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_cgroup_show, NULL); +} + +static const struct file_operations sched_cgroup_fops = { + .open = sched_cgroup_open, + .write = sched_cgroup_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; @@ -686,6 +762,10 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); +#ifdef CONFIG_FAIR_GROUP_SCHED + debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); +#endif + debugfs_fair_server_init(); #ifdef CONFIG_SCHED_CLASS_EXT debugfs_ext_server_init(); @@ -895,10 +975,11 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), + p->se.h_load.weight, SPLIT_NS(p->se.vruntime), - entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', + entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', SPLIT_NS(p->se.deadline), p->se.custom_slice ? 'S' : ' ', SPLIT_NS(p->se.slice), @@ -927,7 +1008,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID vruntime eligible " + SEQ_printf(m, " S task PID weight vruntime eligible " "deadline slice sum-exec switches " "prio wait-time sum-sleep sum-block" #ifdef CONFIG_NUMA_BALANCING @@ -1035,6 +1116,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->tg_load_avg_contrib); SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", atomic_long_read(&cfs_rq->tg->load_avg)); + SEQ_printf(m, " .%-30s: %lu\n", "h_load", + cfs_rq->h_load); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "throttled", @@ -1359,7 +1442,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(wait_count); PN_SCHEDSTAT(iowait_sum); P_SCHEDSTAT(iowait_count); - P_SCHEDSTAT(nr_migrations_cold); P_SCHEDSTAT(nr_failed_migrations_affine); P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); @@ -1371,8 +1453,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_wakeups_remote); P_SCHEDSTAT(nr_wakeups_affine); P_SCHEDSTAT(nr_wakeups_affine_attempts); - P_SCHEDSTAT(nr_wakeups_passive); - P_SCHEDSTAT(nr_wakeups_idle); avg_atom = p->se.sum_exec_runtime; if (nr_switches) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d78467ec6ee1..df8c9c2c7918 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -38,6 +38,7 @@ #include <linux/sched/isolation.h> #include <linux/sched/nohz.h> #include <linux/sched/prio.h> +#include <linux/static_call.h> #include <linux/cpuidle.h> #include <linux/interrupt.h> @@ -72,7 +73,10 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; /* - * Minimal preemption granularity for CPU-bound tasks: + * Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH: + * + * Under EEVDF this is the request size used to compute the virtual + * deadline; see update_deadline(). * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ @@ -296,8 +300,8 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = __calc_delta(delta, NICE_0_LOAD, &se->load); + if (se->h_load.weight != NICE_0_LOAD) + delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); return delta; } @@ -427,38 +431,6 @@ static inline struct sched_entity *parent_entity(const struct sched_entity *se) return se->parent; } -static void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ - int se_depth, pse_depth; - - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ - - /* First walk up until both entities are at same depth */ - se_depth = (*se)->depth; - pse_depth = (*pse)->depth; - - while (se_depth > pse_depth) { - se_depth--; - *se = parent_entity(*se); - } - - while (pse_depth > se_depth) { - pse_depth--; - *pse = parent_entity(*pse); - } - - while (!is_same_group(*se, *pse)) { - *se = parent_entity(*se); - *pse = parent_entity(*pse); - } -} - static int tg_is_idle(struct task_group *tg) { return tg->idle > 0; @@ -502,11 +474,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return NULL; } -static inline void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ -} - static inline int tg_is_idle(struct task_group *tg) { return 0; @@ -685,7 +652,7 @@ static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned static inline void __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 w_vruntime, key = entity_key(cfs_rq, se); w_vruntime = key * weight; @@ -702,7 +669,7 @@ sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se) s64 key, tmp; again: - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); key = entity_key(cfs_rq, se); if (check_mul_overflow(key, weight, &key)) @@ -748,7 +715,7 @@ sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) static void sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 key = entity_key(cfs_rq, se); cfs_rq->sum_w_vruntime -= key * weight; @@ -790,7 +757,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) s64 runtime = cfs_rq->sum_w_vruntime; if (curr) { - unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); runtime += entity_key(cfs_rq, curr) * w; weight += w; @@ -813,6 +780,67 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) return cfs_rq->zero_vruntime; } +/* + * \Sum (v_i - v0)*w_i + * V = ------------------- + v0 + * \Sum w_i + * + * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus: + * + * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j) + * + * \Sum_i!=j (v_i - v0)*w_i + * v_j = ------------------------ + v0 + * W - w_j + * + * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i' + * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr + * is not included in the sum. + */ +static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) +{ + struct sched_entity *curr = cfs_rq->curr; + long weight = cfs_rq->sum_weight; + s64 delta = 0; + + if (curr && !curr->on_rq) + curr = NULL; + + /* + * This is called from set_next_task_fair(.first=true) / + * set_protect_slice() so curr had better be set and on_rq. + */ + WARN_ON_ONCE(!curr); + + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + /* + * Do not add @curr to obtain the effective '- w_j' terms. + */ + + /* sign flips effective floor / ceiling */ + if (runtime < 0) + runtime -= (weight - 1); + + delta = div64_long(runtime, weight); + } + + return cfs_rq->zero_vruntime + delta + 1; +} + static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); /* @@ -861,8 +889,6 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) u64 avruntime = avg_vruntime(cfs_rq); s64 vlag = entity_lag(cfs_rq, se, avruntime); - WARN_ON_ONCE(!se->on_rq); - if (se->sched_delayed) { /* previous vlag < 0 otherwise se would not be delayed */ vlag = max(vlag, se->vlag); @@ -898,7 +924,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { - unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); avg += entity_key(cfs_rq, curr) * weight; load += weight; @@ -1039,6 +1065,9 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; @@ -1048,6 +1077,9 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); sum_w_vruntime_sub(cfs_rq, se); @@ -1090,8 +1122,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity slice = cfs_rq_min_slice(cfs_rq); slice = min(slice, se->slice); - if (slice != se->slice) - vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + + /* If there are shorter slices than se's one */ + if (slice != se->slice) { + if (sched_feat(PREEMPT_SHORT)) + vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); + else + vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + } se->vprot = vprot; } @@ -1099,8 +1137,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { u64 slice = cfs_rq_min_slice(cfs_rq); + u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); - se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); + se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); } static inline bool protect_slice(struct sched_entity *se) @@ -1144,7 +1183,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) * We can safely skip eligibility check if there is only one entity * in this cfs_rq, saving some cycles. */ - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return curr && curr->on_rq ? curr : se; /* @@ -1394,8 +1433,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) return delta_exec; } -static void set_next_buddy(struct sched_entity *se); - #ifdef CONFIG_SCHED_CACHE /* @@ -1990,7 +2027,7 @@ static void update_curr(struct cfs_rq *cfs_rq) * not necessarily be the actual task running * (rq->curr.se). This is easy to confuse! */ - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); s64 delta_exec; bool resched; @@ -2002,26 +2039,29 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; + account_cfs_rq_runtime(cfs_rq, delta_exec); + + if (!entity_is_task(curr)) + return; + + cfs_rq = &rq->cfs; + curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - if (entity_is_task(curr)) { - /* - * If the fair_server is active, we need to account for the - * fair_server time whether or not the task is running on - * behalf of fair_server or not: - * - If the task is running on behalf of fair_server, we need - * to limit its time based on the assigned runtime. - * - Fair task that runs outside of fair_server should account - * against fair_server such that it can account for this time - * and possibly avoid running this period. - */ - dl_server_update(&rq->fair_server, delta_exec); - } - - account_cfs_rq_runtime(cfs_rq, delta_exec); + /* + * If the fair_server is active, we need to account for the + * fair_server time whether or not the task is running on + * behalf of fair_server or not: + * - If the task is running on behalf of fair_server, we need + * to limit its time based on the assigned runtime. + * - Fair task that runs outside of fair_server should account + * against fair_server such that it can account for this time + * and possibly avoid running this period. + */ + dl_server_update(&rq->fair_server, delta_exec); - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return; if (resched || !protect_slice(curr)) { @@ -2032,7 +2072,10 @@ static void update_curr(struct cfs_rq *cfs_rq) static void update_curr_fair(struct rq *rq) { - update_curr(cfs_rq_of(&rq->donor->se)); + struct sched_entity *se = &rq->donor->se; + + for_each_sched_entity(se) + update_curr(cfs_rq_of(se)); } static inline void @@ -2108,7 +2151,7 @@ update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_start_fair(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) @@ -2126,7 +2169,7 @@ update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Mark the end of the wait period if dequeueing a * waiting task: */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_end_fair(cfs_rq, se); if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { @@ -4467,6 +4510,7 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4482,6 +4526,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4563,7 +4608,7 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) static void rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) { - unsigned long old_weight = se->load.weight; + long old_weight = se->h_load.weight; /* * VRUNTIME @@ -4663,16 +4708,17 @@ rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) se->vprot = div64_long(se->vprot * old_weight, weight); } -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) +static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight, bool on_rq) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; u64 avruntime = 0; - if (se->on_rq) { - /* commit outstanding execution time */ - update_curr(cfs_rq); + if (se->h_load.weight == weight) + return; + + if (on_rq) { avruntime = avg_vruntime(cfs_rq); se->vlag = entity_lag(cfs_rq, se, avruntime); se->deadline -= avruntime; @@ -4682,46 +4728,90 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, rel_vprot = true; } - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); - update_load_sub(&cfs_rq->load, se->load.weight); } - dequeue_load_avg(cfs_rq, se); rescale_entity(se, weight, rel_vprot); - update_load_set(&se->load, weight); + update_load_set(&se->h_load, weight); - do { - u32 divider = get_pelt_divider(&se->avg); - se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); - } while (0); - - enqueue_load_avg(cfs_rq, se); - if (se->on_rq) { + if (on_rq) { if (rel_vprot) se->vprot += avruntime; se->deadline += avruntime; se->rel_deadline = 0; se->vruntime = avruntime - se->vlag; - update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; + } +} + +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + if (se->load.weight == weight) + return; + + if (se->on_rq) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); + update_load_sub(&cfs_rq->load, se->load.weight); } + dequeue_load_avg(cfs_rq, se); + + update_load_set(&se->load, weight); + + do { + u32 divider = get_pelt_divider(&se->avg); + se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); + } while (0); + + enqueue_load_avg(cfs_rq, se); + + if (se->on_rq) + update_load_add(&cfs_rq->load, se->load.weight); +} + +/* + * weight = NICE_0_LOAD; + * for_each_entity_se(se) + * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + */ +static __always_inline +unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + weight *= se->load.weight; + if (parent_entity(se)) + weight /= cfs_rq->load.weight; + else + weight /= NICE_0_LOAD; + + return max(weight, MIN_SHARES); } static void reweight_task_fair(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - struct load_weight *load = &se->load; + unsigned long weight = NICE_0_LOAD; + + if (se->on_rq) + update_curr_fair(rq); + + reweight_entity(cfs_rq_of(se), se, lw->weight); + se->load.inv_weight = lw->inv_weight; + + if (!se->on_rq) + return; - reweight_entity(cfs_rq, se, lw->weight); - load->inv_weight = lw->inv_weight; + for_each_sched_entity(se) + weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + + reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); @@ -4800,12 +4890,10 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); * * hence icky! */ -static long calc_group_shares(struct cfs_rq *cfs_rq) +static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) { - long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - - tg_shares = READ_ONCE(tg->shares); + long tg_weight, load, shares; load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -4831,7 +4919,115 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, tg_shares); + return clamp_t(long, shares, MIN_SHARES, shares_max); +} + +static int tg_cpus(struct task_group *tg) +{ + int nr = num_online_cpus(); + + if (cpusets_enabled()) { + struct cgroup *cgrp = tg->css.cgroup; + if (cgrp) + nr = cpuset_num_cpus(cgrp); + } + + return nr; +} + +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + +/* + * Func: fraction(nr_tasks * tg->shares) + * + * Scale tg->shares by the number of tasks. + */ +static long calc_tasks_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_tasks(tg); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: min(fraction(nr_cpus * tg->shares), nice -20) + * + * Scale tg->shares by the maximal number of CPUs; but clip the max shares at + * nice -20, otherwise a single spinner on a 512 CPU machine would result in + * 512*NICE_0_LOAD, which is also crazy. + */ +static long calc_max_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_cpus(tg); + long tg_shares = READ_ONCE(tg->shares); + long max_shares = scale_load(sched_prio_to_weight[0]); + return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); +} + +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: fraction(tg->shares) + * + * This infamously results in tiny shares when you have many CPUs. + */ +static long calc_smp_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); +} + +/* + * Ignore this pesky SMP stuff, use (4). + */ +static long calc_up_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + return READ_ONCE(tg->shares); +} + +DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); + +void __sched_cgroup_mode_update(int mode) +{ + long (*func)(struct cfs_rq *); + switch (mode) { + case 0: + func = &calc_up_shares; + break; + case 1: + func = &calc_smp_shares; + break; + case 2: + default: + func = &calc_concur_shares; + break; + case 3: + func = &calc_max_shares; + break; + case 4: + func = &calc_tasks_shares; + break; + } + static_call_update(calc_group_shares, func); } /* @@ -4850,9 +5046,8 @@ static void update_cfs_group(struct sched_entity *se) if (!gcfs_rq || !gcfs_rq->load.weight) return; - shares = calc_group_shares(gcfs_rq); - if (unlikely(se->load.weight != shares)) - reweight_entity(cfs_rq_of(se), se, shares); + shares = static_call(calc_group_shares)(gcfs_rq); + reweight_entity(cfs_rq_of(se), se, shares); } #else /* !CONFIG_FAIR_GROUP_SCHED: */ @@ -4970,11 +5165,11 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) * differential update where we store the last value we propagated. This in * turn allows skipping updates if the differential is 'small'. * - * Updating tg's load_avg is necessary before update_cfs_share(). + * Updating tg's load_avg is necessary before update_cfs_group(). */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -4995,17 +5190,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5015,9 +5214,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } @@ -5174,7 +5376,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -5188,7 +5391,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -5430,7 +5633,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {} * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) * avg. The immediate corollary is that all (fair) tasks must be attached. * - * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. + * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. * * Return: true if the load decayed or we removed load. * @@ -5968,6 +6171,7 @@ static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { u64 vslice, vruntime = avg_vruntime(cfs_rq); + unsigned int nr_queued = cfs_rq->h_nr_queued; bool update_zero = false; s64 lag = 0; @@ -5975,6 +6179,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) se->slice = sysctl_sched_base_slice; vslice = calc_delta_fair(se->slice, se); + if (flags & ENQUEUE_QUEUED) + nr_queued -= 1; + /* * Due to how V is constructed as the weighted average of entities, * adding tasks with positive lag, or removing tasks with negative lag @@ -5983,7 +6190,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ - if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { + if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { struct sched_entity *curr = cfs_rq->curr; long load, weight; @@ -6043,9 +6250,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ load = cfs_rq->sum_weight; if (curr && curr->on_rq) - load += avg_vruntime_weight(cfs_rq, curr->load.weight); + load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; @@ -6104,22 +6311,8 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq); static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); static void -requeue_delayed_entity(struct sched_entity *se); - -static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool curr = cfs_rq->curr == se; - - /* - * If we're the current task, we must renormalise before calling - * update_curr(). - */ - if (curr) - place_entity(cfs_rq, se, flags); - - update_curr(cfs_rq); - /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. @@ -6138,13 +6331,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ update_cfs_group(se); - /* - * XXX now that the entity has been re-weighted, and it's lag adjusted, - * we can place the entity. - */ - if (!curr) - place_entity(cfs_rq, se, flags); - account_entity_enqueue(cfs_rq, se); /* Entity has migrated, no longer consider this task hot */ @@ -6153,8 +6339,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); - if (!curr) - __enqueue_entity(cfs_rq, se); se->on_rq = 1; if (cfs_rq->nr_queued == 1) { @@ -6172,21 +6356,19 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } } -static void __clear_buddies_next(struct sched_entity *se) +static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next != se) - break; - - cfs_rq->next = NULL; - } + if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) + return; + if (se_is_idle(se)) + return; + cfs_rq->next = se; } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->next == se) - __clear_buddies_next(se); + cfs_rq->next = NULL; } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -6197,7 +6379,7 @@ static void set_delayed(struct sched_entity *se) /* * Delayed se of cfs_rq have no tasks queued on them. - * Do not adjust h_nr_runnable since dequeue_entities() + * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. */ if (!entity_is_task(se)) @@ -6230,45 +6412,16 @@ static void clear_delayed(struct sched_entity *se) } } -static bool +static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool sleep = flags & DEQUEUE_SLEEP; - int action = 0; - - update_curr(cfs_rq); - clear_buddies(cfs_rq, se); - - if (flags & DEQUEUE_DELAYED) { - WARN_ON_ONCE(!se->sched_delayed); - } else { - bool delay = sleep; - /* - * DELAY_DEQUEUE relies on spurious wakeups, special task - * states must not suffer spurious wakeups, excempt them. - */ - if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) - delay = false; - - WARN_ON_ONCE(delay && se->sched_delayed); - - if (sched_feat(DELAY_DEQUEUE) && delay && - !entity_eligible(cfs_rq, se)) { - if (entity_is_task(se)) - action |= UPDATE_UTIL_EST; - update_load_avg(cfs_rq, se, action); - update_entity_lag(cfs_rq, se); - set_delayed(se); - return false; - } - } + int action = UPDATE_TG; - action = UPDATE_TG; if (entity_is_task(se)) { if (task_on_rq_migrating(task_of(se))) action |= DO_DETACH; - if (sleep && !(flags & DEQUEUE_DELAYED)) + if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) action |= UPDATE_UTIL_EST; } @@ -6286,14 +6439,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_stats_dequeue_fair(cfs_rq, se, flags); - update_entity_lag(cfs_rq, se); - if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { - se->deadline -= se->vruntime; - se->rel_deadline = 1; - } - - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); se->on_rq = 0; account_entity_dequeue(cfs_rq, se); @@ -6302,9 +6447,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_cfs_group(se); - if (flags & DEQUEUE_DELAYED) - clear_delayed(se); - if (cfs_rq->nr_queued == 0) { update_idle_cfs_rq_clock_pelt(cfs_rq); #ifdef CONFIG_CFS_BANDWIDTH @@ -6317,15 +6459,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } #endif } - - return true; } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - clear_buddies(cfs_rq, se); - /* 'current' is not kept within the tree. */ if (se->on_rq) { /* @@ -6334,16 +6472,12 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) * runqueue. */ update_stats_wait_end_fair(cfs_rq, se); - __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - - if (first) - set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); - WARN_ON_ONCE(cfs_rq->curr); - cfs_rq->curr = se; + WARN_ON_ONCE(cfs_rq->h_curr); + cfs_rq->h_curr = se; /* * Track our maximum slice length, if the CPU's load is at @@ -6363,23 +6497,17 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) se->prev_sum_exec_runtime = se->sum_exec_runtime; } -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); -/* - * Pick the next process, keeping these things in mind, in this order: - * 1) keep things fair between processes/task groups - * 2) pick the "next" process, since someone really wants that to run - * 3) pick the "last" process, for cache locality - * 4) do not run the "skip" process, if something else is available - */ static struct sched_entity * -pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) +pick_next_entity(struct rq *rq, bool protect) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* * Must not reference @se again, see __block_task(). */ @@ -6399,13 +6527,11 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); - /* Put 'current' back into the tree. */ - __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ update_load_avg(cfs_rq, prev, 0); } - WARN_ON_ONCE(cfs_rq->curr != prev); - cfs_rq->curr = NULL; + WARN_ON_ONCE(cfs_rq->h_curr != prev); + cfs_rq->h_curr = NULL; } static void @@ -6960,7 +7086,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ - if (rq->curr == rq->idle && rq->cfs.nr_queued) + if (rq->curr == rq->idle && rq->cfs.h_nr_queued) resched_curr(rq); } @@ -7295,7 +7421,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) return; /* an active group must be handled by the update_curr() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) + if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) return; /* ensure the group is not already throttled */ @@ -7667,7 +7793,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) resched_curr(rq); return; } - delta = (se->load.weight * vdelta) / NICE_0_LOAD; + delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; /* * Correct for instantaneous load of other classes. @@ -7767,10 +7893,8 @@ static int choose_idle_cpu(int cpu, struct task_struct *p) } static void -requeue_delayed_entity(struct sched_entity *se) +requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - /* * se->sched_delayed should imply: se->on_rq == 1. * Because a delayed entity is one that is still on @@ -7780,19 +7904,58 @@ requeue_delayed_entity(struct sched_entity *se) WARN_ON_ONCE(!se->on_rq); if (update_entity_lag(cfs_rq, se)) { - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); place_entity(cfs_rq, se, 0); if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } update_load_avg(cfs_rq, se, 0); clear_delayed(se); } +static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) +{ + unsigned long weight = NICE_0_LOAD; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + int h_nr_idle = task_has_idle_policy(p); + int h_nr_runnable = 1; + + if (task_new && se->sched_delayed) + h_nr_runnable = 0; + + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + update_curr(cfs_rq); + + if (!se->on_rq) { + enqueue_entity(cfs_rq, se, flags); + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); + } + + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; + + weight = __calc_prop_weight(cfs_rq, se, weight); + + flags = ENQUEUE_WAKEUP; + } + + return weight; +} + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -7801,13 +7964,12 @@ requeue_delayed_entity(struct sched_entity *se) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - int h_nr_idle = task_has_idle_policy(p); - int h_nr_runnable = 1; - int task_new = !(flags & ENQUEUE_WAKEUP); int rq_h_nr_queued = rq->cfs.h_nr_queued; - u64 slice = 0; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight; + bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -7819,10 +7981,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) * estimated utilization, before we update schedutil. */ if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) - util_est_enqueue(&rq->cfs, p); + util_est_enqueue(cfs_rq, p); if (flags & ENQUEUE_DELAYED) { - requeue_delayed_entity(se); + requeue_delayed_entity(cfs_rq, se); return; } @@ -7834,57 +7996,22 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - if (task_new && se->sched_delayed) - h_nr_runnable = 0; - - for_each_sched_entity(se) { - if (se->on_rq) { - if (se->sched_delayed) - requeue_delayed_entity(se); - break; - } - cfs_rq = cfs_rq_of(se); - - /* - * Basically set the slice of group entries to the min_slice of - * their respective cfs_rq. This ensures the group can service - * its entities in the desired time-frame. - */ - if (slice) { - se->slice = slice; - se->custom_slice = 1; - } - enqueue_entity(cfs_rq, se, flags); - slice = cfs_rq_min_slice(cfs_rq); - - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; - - flags = ENQUEUE_WAKEUP; - } - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + /* + * XXX comment on the curr thing + */ + curr = (cfs_rq->curr == se); + if (curr) + place_entity(cfs_rq, se, flags); - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + if (se->on_rq && se->sched_delayed) + requeue_delayed_entity(cfs_rq, se); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + weight = enqueue_hierarchy(p, flags); - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; + if (!curr) { + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) @@ -7915,105 +8042,109 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) hrtick_update(rq); } -/* - * Basically dequeue_task_fair(), except it can deal with dequeue_entity() - * failing half-way through and resume the dequeue later. - * - * Returns: - * -1 - dequeue delayed - * 0 - dequeue throttled - * 1 - dequeue complete - */ -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) +static void dequeue_hierarchy(struct task_struct *p, int flags) { - bool was_sched_idle = sched_idle_rq(rq); + struct sched_entity *se = &p->se; bool task_sleep = flags & DEQUEUE_SLEEP; bool task_delayed = flags & DEQUEUE_DELAYED; bool task_throttled = flags & DEQUEUE_THROTTLE; - struct task_struct *p = NULL; - int h_nr_idle = 0; - int h_nr_queued = 0; int h_nr_runnable = 0; - struct cfs_rq *cfs_rq; - u64 slice = 0; + int h_nr_idle = task_has_idle_policy(p); + bool dequeue = true; - if (entity_is_task(se)) { - p = task_of(se); - h_nr_queued = 1; - h_nr_idle = task_has_idle_policy(p); - if (task_sleep || task_delayed || !se->sched_delayed) - h_nr_runnable = 1; - } + if (task_sleep || task_delayed || !se->sched_delayed) + h_nr_runnable = 1; for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!dequeue_entity(cfs_rq, se, flags)) { - if (p && &p->se == se) - return -1; + update_curr(cfs_rq); - slice = cfs_rq_min_slice(cfs_rq); - break; + if (dequeue) { + dequeue_entity(cfs_rq, se, flags); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) + dequeue = false; + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); } cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; + cfs_rq->h_nr_queued--; cfs_rq->h_nr_idle -= h_nr_idle; if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + h_nr_idle = 1; if (throttled_hierarchy(cfs_rq) && task_throttled) record_throttle_clock(cfs_rq); - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) { - slice = cfs_rq_min_slice(cfs_rq); - - /* Avoid re-evaluating load for this entity: */ - se = parent_entity(se); - /* - * Bias pick_next to pick a task from this cfs_rq, as - * p is sleeping when it is within its sched_slice. - */ - if (task_sleep && se) - set_next_buddy(se); - break; - } flags |= DEQUEUE_SLEEP; flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); } +} - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); +/* + * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. + * + * Returns: + * true - dequeued + * false - delayed + */ +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) +{ + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + bool was_sched_idle = sched_idle_rq(rq); + bool task_sleep = flags & DEQUEUE_SLEEP; + bool task_delayed = flags & DEQUEUE_DELAYED; - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + clear_buddies(cfs_rq, se); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + update_curr(cfs_rq_of(se)); + update_entity_lag(cfs_rq, se); - cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; - cfs_rq->h_nr_idle -= h_nr_idle; + if (flags & DEQUEUE_DELAYED) { + WARN_ON_ONCE(!se->sched_delayed); + } else { + bool delay = task_sleep; + /* + * DELAY_DEQUEUE relies on spurious wakeups, special task + * states must not suffer spurious wakeups, excempt them. + */ + if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) + delay = false; - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + WARN_ON_ONCE(delay && se->sched_delayed); - if (throttled_hierarchy(cfs_rq) && task_throttled) - record_throttle_clock(cfs_rq); + if (sched_feat(DELAY_DEQUEUE) && delay && + !entity_eligible(cfs_rq, se)) { + update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); + set_delayed(se); + return false; + } } - sub_nr_running(rq, h_nr_queued); + dequeue_hierarchy(p, flags); + + if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { + se->deadline -= se->vruntime; + se->rel_deadline = 1; + } + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + + sub_nr_running(rq, 1); /* balance early to pull high priority tasks */ if (unlikely(!was_sched_idle && sched_idle_rq(rq))) rq->next_balance = jiffies; - if (p && task_delayed) { + if (task_delayed) { + clear_delayed(se); + WARN_ON_ONCE(!task_sleep); WARN_ON_ONCE(p->on_rq != 1); @@ -8025,7 +8156,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) __block_task(rq, p); } - return 1; + return true; } /* @@ -8043,11 +8174,11 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - if (dequeue_entities(rq, &p->se, flags) < 0) + if (!__dequeue_task(rq, p, flags)) return false; /* - * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). + * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). */ return true; } @@ -9642,19 +9773,6 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) static void task_dead_fair(struct task_struct *p) { struct sched_entity *se = &p->se; - - if (se->sched_delayed) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - task_rq_unlock(rq, p, &rf); - } - remove_entity_load_avg(se); } @@ -9688,27 +9806,14 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context set_task_max_allowed_capacity(p); } -static void set_next_buddy(struct sched_entity *se) -{ - for_each_sched_entity(se) { - if (WARN_ON_ONCE(!se->on_rq)) - return; - if (se_is_idle(se)) - return; - cfs_rq_of(se)->next = se; - } -} - enum preempt_wakeup_action { PREEMPT_WAKEUP_NONE, /* No preemption. */ PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ - PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ + PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ }; -static inline bool -set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, - struct sched_entity *pse, struct sched_entity *se) +static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) { /* * Keep existing buddy if the deadline is sooner than pse. @@ -9719,7 +9824,16 @@ set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, if (cfs_rq->next && entity_before(cfs_rq->next, pse)) return false; - set_next_buddy(pse); + set_next_buddy(cfs_rq, pse); + return true; +} + +static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) +{ + if (cfs_rq->next && cfs_rq->next->slice < pse->slice) + return false; + + set_next_buddy(cfs_rq, pse); return true; } @@ -9772,13 +9886,14 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; struct sched_entity *nse, *se = &donor->se, *pse = &p->se; - struct cfs_rq *cfs_rq = task_cfs_rq(donor); + struct cfs_rq *cfs_rq = &rq->cfs; int cse_is_idle, pse_is_idle; /* * XXX Getting preempted by higher class, try and find idle CPU? */ - if (p->sched_class != &fair_sched_class) + if (p->sched_class != &fair_sched_class || + donor->sched_class != &fair_sched_class) return; if (unlikely(se == pse)) @@ -9803,42 +9918,43 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f * prevents us from potentially nominating it as a false LAST_BUDDY * below. */ - if (test_tsk_need_resched(rq->curr)) + if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr)) return; if (!sched_feat(WAKEUP_PREEMPTION)) return; - find_matching_se(&se, &pse); WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); pse_is_idle = se_is_idle(pse); + nse = se; /* * Preempt an idle entity in favor of a non-idle entity (and don't preempt * in the inverse case). */ - if (cse_is_idle && !pse_is_idle) { - /* - * When non-idle entity preempt an idle entity, - * don't give idle entity slice protection. - */ - preempt_action = PREEMPT_WAKEUP_SHORT; + if (cse_is_idle && !pse_is_idle) goto preempt; - } + + update_curr_fair(rq); if (cse_is_idle != pse_is_idle) - return; + goto update; /* * BATCH and IDLE tasks do not preempt others. */ if (unlikely(!normal_policy(p->policy))) - return; + goto update; + + /* + * Do not preempt for tasks that are sched_delayed as it would violate + * EEVDF to forcibly queue an ineligible task. + */ + if (pse->sched_delayed) + goto update; - cfs_rq = cfs_rq_of(se); - update_curr(cfs_rq); /* * If @p has a shorter slice than current and @p is eligible, override * current's slice protection in order to allow preemption. @@ -9850,17 +9966,13 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f /* * Ignore wakee preemption on WF_FORK as it is less likely that - * there is shared data as exec often follow fork. Do not - * preempt for tasks that are sched_delayed as it would violate - * EEVDF to forcibly queue an ineligible task. + * there is shared data as exec often follow fork. */ - if ((wake_flags & WF_FORK) || pse->sched_delayed) - return; + if (wake_flags & WF_FORK) + goto update; /* Prefer picking wakee soon if appropriate. */ - if (sched_feat(NEXT_BUDDY) && - set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { - + if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) { /* * Decide whether to obey WF_SYNC hint for a new buddy. Old * buddies are ignored as they may not be relevant to the @@ -9882,29 +9994,33 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f } pick: - nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); - /* If @p has become the most eligible task, force preemption */ - if (nse == pse) - goto preempt; + if (cfs_rq->h_nr_queued) { + nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); + if (unlikely(!nse)) + goto pick; + + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; + } /* - * Because p is enqueued, nse being null can only mean that we - * dequeued a delayed task. If there are still entities queued in - * cfs, check if the next one will be p. + * If @p is eligible but not the next task to run then cancel protection + * to prevent large scheduling latency */ - if (!nse && cfs_rq->nr_queued) - goto pick; - + if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse)) + goto preempt; +update: if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); return; preempt: - if (preempt_action == PREEMPT_WAKEUP_SHORT) { - cancel_protect_slice(se); - clear_buddies(cfs_rq, se); - } + cancel_protect_slice(se); + + if (preempt_action == PREEMPT_WAKEUP_SHORT) + set_short_buddy(cfs_rq, pse); resched_curr_lazy(rq); } @@ -9912,33 +10028,24 @@ preempt: struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; - struct cfs_rq *cfs_rq; struct task_struct *p; - bool throttled; int new_tasks; again: - cfs_rq = &rq->cfs; - if (!cfs_rq->nr_queued) + if (!cfs_rq->h_nr_queued) goto idle; - throttled = false; - - do { - /* Might not have done put_prev_entity() */ - if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + /* Might not have done put_prev_entity() */ + if (cfs_rq->curr && cfs_rq->curr->on_rq) + update_curr(cfs_rq); - se = pick_next_entity(rq, cfs_rq, true); - if (!se) - goto again; - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + se = pick_next_entity(rq, true); + if (!se) + goto again; p = task_of(se); - if (unlikely(throttled)) - task_throttle_setup_work(p); return p; idle: @@ -9975,7 +10082,7 @@ void fair_server_init(struct rq *rq) static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *nse = NULL; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -9985,7 +10092,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t while (se) { cfs_rq = cfs_rq_of(se); - if (!nse || cfs_rq->curr) + if (!nse || cfs_rq->h_curr) put_prev_entity(cfs_rq, se); #ifdef CONFIG_FAIR_GROUP_SCHED if (nse) { @@ -10004,6 +10111,14 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t #endif se = parent_entity(se); } + + /* Put 'current' back into the tree. */ + cfs_rq = &rq->cfs; + se = &prev->se; + WARN_ON_ONCE(cfs_rq->curr != se); + cfs_rq->curr = NULL; + if (se->on_rq) + __enqueue_entity(cfs_rq, se); } /* @@ -10012,8 +10127,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t static void yield_task_fair(struct rq *rq) { struct task_struct *curr = rq->donor; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq = &rq->cfs; /* * Are we the only task in the tree? @@ -10054,11 +10169,11 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) struct sched_entity *se = &p->se; /* !se->on_rq also covers throttled task */ - if (!se->on_rq) + if (!se->on_rq || se->sched_delayed) return false; /* Tell the scheduler that we'd really like se to run next. */ - set_next_buddy(se); + set_next_buddy(&task_rq(p)->cfs, se); yield_task_fair(rq); @@ -10397,15 +10512,10 @@ static inline long migrate_degrades_locality(struct task_struct *p, */ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) { - struct cfs_rq *dst_cfs_rq; + struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#ifdef CONFIG_FAIR_GROUP_SCHED - dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); -#else - dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#endif - if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && - !entity_eligible(task_cfs_rq(p), &p->se)) + if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && + !entity_eligible(&task_rq(p)->cfs, &p->se)) return 1; return 0; @@ -11188,7 +11298,7 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { load = cfs_rq->h_load; load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq_load_avg(cfs_rq) + 1); cfs_rq = group_cfs_rq(se); cfs_rq->h_load = load; cfs_rq->last_h_load_update = now; @@ -13437,75 +13547,78 @@ more_balance: } } - if (!ld_moved) { - schedstat_inc(sd->lb_failed[idle]); - /* - * Increment the failure counter only on periodic balance. - * We do not want newidle balance, which can be very - * frequent, pollute the failure counter causing - * excessive cache_hot migrations and active balances. - * - * Similarly for migration_misfit which is not related to - * load/util migration, don't pollute nr_balance_failed. - * - * The same for cache aware scheduling's allowance for - * load imbalance. If regular load balance does not - * migrate task due to LLC locality, it is a expected - * behavior and don't pollute nr_balance_failed. - * See can_migrate_task(). - */ - if (idle != CPU_NEWLY_IDLE && - env.migration_type != migrate_misfit && - !(env.flags & LBF_LLC_PINNED)) - sd->nr_balance_failed++; + if (ld_moved) { + sd->nr_balance_failed = 0; + goto out_unbalanced; + } - if (need_active_balance(&env)) { - unsigned long flags; + schedstat_inc(sd->lb_failed[idle]); + /* + * Increment the failure counter only on periodic balance. + * We do not want newidle balance, which can be very + * frequent, pollute the failure counter causing + * excessive cache_hot migrations and active balances. + * + * Similarly for migration_misfit which is not related to + * load/util migration, don't pollute nr_balance_failed. + * + * The same for cache aware scheduling's allowance for + * load imbalance. If regular load balance does not + * migrate task due to LLC locality, it is a expected + * behavior and don't pollute nr_balance_failed. + * See can_migrate_task(). + */ + if (idle != CPU_NEWLY_IDLE && + env.migration_type != migrate_misfit && + !(env.flags & LBF_LLC_PINNED)) + sd->nr_balance_failed++; - raw_spin_rq_lock_irqsave(busiest, flags); + if (!need_active_balance(&env)) + goto out_unbalanced; - /* - * Don't kick the active_load_balance_cpu_stop, - * if the curr task on busiest CPU can't be - * moved to this_cpu: - */ - if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { - raw_spin_rq_unlock_irqrestore(busiest, flags); - goto out_one_pinned; - } + scoped_guard (raw_spin_rq_lock_irqsave, busiest) { + /* + * Don't kick the active_load_balance_cpu_stop, + * if the curr task on busiest CPU can't be + * moved to this_cpu: + */ + if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) + goto out_one_pinned; - /* Record that we found at least one task that could run on this_cpu */ - env.flags &= ~LBF_ALL_PINNED; + /* Record that we found at least one task that could run on this_cpu */ + env.flags &= ~LBF_ALL_PINNED; - /* - * ->active_balance synchronizes accesses to - * ->active_balance_work. Once set, it's cleared - * only after active load balance is finished. - */ - if (!busiest->active_balance) { - busiest->active_balance = 1; - busiest->push_cpu = this_cpu; - active_balance = 1; - } + /* + * ->active_balance synchronizes accesses to + * ->active_balance_work. Once set, it's cleared + * only after active load balance is finished. + */ + if (busiest->active_balance) + goto out_unbalanced; - preempt_disable(); - raw_spin_rq_unlock_irqrestore(busiest, flags); - if (active_balance) { - stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, - &busiest->active_balance_work); - } - preempt_enable(); - } - } else { - sd->nr_balance_failed = 0; - } + /* + * @busiest dropped its rq_lock in the middle of + * scheduling out its ->curr task (->on_rq := 0), no + * need to forcefully punt it away with active balance. + */ + if (!busiest->curr->on_rq) + goto out_unbalanced; - if (likely(!active_balance) || need_active_balance(&env)) { - /* We were unbalanced, so reset the balancing interval */ - sd->balance_interval = sd->min_interval; + busiest->active_balance = 1; + busiest->push_cpu = this_cpu; + active_balance = 1; + preempt_disable(); } + if (active_balance) { + stop_one_cpu_nowait(cpu_of(busiest), + active_load_balance_cpu_stop, busiest, + &busiest->active_balance_work); + } + preempt_enable(); +out_unbalanced: + /* We were unbalanced, so reset the balancing interval */ + sd->balance_interval = sd->min_interval; goto out; out_balanced: @@ -14571,7 +14684,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * if the sibling is forced idle, then trigger schedule to * give forced idle task a chance. * - * sched_slice() considers only this active rq and it gets the + * __entity_slice_used() considers only this active rq and it gets the * whole slice. But during force idle, we have siblings acting * like a single runqueue and hence we need to consider runnable * tasks on this CPU and the forced idle CPU. Ideally, we should @@ -14580,7 +14693,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && + if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } @@ -14789,30 +14902,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, WARN_ON_ONCE(task_rq(b)->core != rq->core); -#ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Find an se in the hierarchy for tasks a and b, such that the se's - * are immediate siblings. - */ - while (sea->cfs_rq->tg != seb->cfs_rq->tg) { - int sea_depth = sea->depth; - int seb_depth = seb->depth; - - if (sea_depth >= seb_depth) - sea = parent_entity(sea); - if (sea_depth <= seb_depth) - seb = parent_entity(seb); - } - - se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); - se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); - - cfs_rqa = sea->cfs_rq; - cfs_rqb = seb->cfs_rq; -#else /* !CONFIG_FAIR_GROUP_SCHED: */ cfs_rqa = &task_rq(a)->cfs; cfs_rqb = &task_rq(b)->cfs; -#endif /* !CONFIG_FAIR_GROUP_SCHED */ /* * Find delta after normalizing se's vruntime with its cfs_rq's @@ -14851,11 +14942,20 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct sched_entity *se = &curr->se; - struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); + if (se->on_rq) { + unsigned long weight = NICE_0_LOAD; + struct cfs_rq *cfs_rq; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); + + weight = __calc_prop_weight(cfs_rq, se, weight); + } + + se = &curr->se; + reweight_eevdf(cfs_rq, se, weight, se->on_rq); } if (queued) @@ -14895,7 +14995,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) if (p->prio == oldprio) return; - if (rq->cfs.nr_queued == 1) + if (rq->cfs.h_nr_queued == 1) return; /* @@ -15024,33 +15124,44 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -/* - * Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; bool throttled = false; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight = NICE_0_LOAD; + bool on_rq = se->on_rq; + + clear_buddies(cfs_rq, se); + + if (on_rq) + __dequeue_entity(cfs_rq, se); for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + cfs_rq = cfs_rq_of(se); - if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && - first && cfs_rq->curr) - break; + if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || + !first || !cfs_rq->h_curr) + set_next_entity(cfs_rq, se); - set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ throttled |= account_cfs_rq_runtime(cfs_rq, 0); + + if (on_rq) + weight = __calc_prop_weight(cfs_rq, se, weight); } if (throttled) task_throttle_setup_work(p); se = &p->se; + cfs_rq->curr = se; + + if (on_rq) { + reweight_eevdf(cfs_rq, se, weight, se->on_rq); + if (first) + set_protect_slice(cfs_rq, se); + } if (task_on_rq_queued(p)) { /* @@ -15163,17 +15274,8 @@ void unregister_fair_sched_group(struct task_group *tg) struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); - if (se) { - if (se->sched_delayed) { - guard(rq_lock_irqsave)(rq); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - list_del_leaf_cfs_rq(cfs_rq); - } + if (se) remove_entity_load_avg(se); - } /* * Only empty task groups can be destroyed; so we can speculatively diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 897790889ba3..779eb58a4261 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -206,7 +206,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa, /* * running is a subset of runnable (weight) so running can't be set if * runnable is clear. But there are some corner cases where the current - * se has been already dequeued but cfs_rq->curr still points to it. + * se has been already dequeued but cfs_rq->h_curr still points to it. * This means that weight will be 0 but not running for a sched_entity * but also for a cfs_rq if the latter becomes idle. As an example, * this happens during sched_balance_newidle() which calls @@ -307,7 +307,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se), - cfs_rq->curr == se)) { + cfs_rq->h_curr == se)) { ___update_load_avg(&se->avg, se_weight(se)); cfs_se_util_change(&se->avg); @@ -323,7 +323,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) if (___update_load_sum(now, &cfs_rq->avg, scale_load_down(cfs_rq->load.weight), cfs_rq->h_nr_runnable, - cfs_rq->curr != NULL)) { + cfs_rq->h_curr != NULL)) { ___update_load_avg(&cfs_rq->avg, 1); trace_pelt_cfs_tp(cfs_rq); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index d9c9d9480a45..848955f8893d 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1023,7 +1023,7 @@ void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct task_st irq = irq_time_read(cpu); delta = (s64)(irq - rq->psi_irq_time); - if (delta < 0) + if (delta <= 0) return; rq->psi_irq_time = irq; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e474c31d8fe6..e6e5f8a2caaf 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) /* * XXX If we're preempted by DL, queue a push? */ - if (p->sched_class != &rt_sched_class) + if (p->sched_class != &rt_sched_class || + donor->sched_class != &rt_sched_class) return; if (p->prio < donor->prio) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba26..26ae13c86b69 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,8 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -528,21 +530,8 @@ struct task_group { }; -#ifdef CONFIG_GROUP_SCHED_WEIGHT #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD -/* - * A weight of 0 or 1 can cause arithmetics problems. - * A weight of a cfs_rq is the sum of weights of which entities - * are queued on this cfs_rq, so a weight of a entity should not be - * too large, so as the shares value of a task group. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ -#define MIN_SHARES (1UL << 1) -#define MAX_SHARES (1UL << 18) -#endif - typedef int (*tg_visitor)(struct task_group *, void *); extern int walk_tg_tree_from(struct task_group *from, @@ -571,6 +560,7 @@ extern void free_fair_sched_group(struct task_group *tg); extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); +extern void __sched_cgroup_mode_update(int mode); #else /* !CONFIG_FAIR_GROUP_SCHED: */ static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -628,6 +618,17 @@ static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false #endif /* !CONFIG_CGROUP_SCHED */ +/* + * A weight of 0 or 1 can cause arithmetics problems. + * A weight of a cfs_rq is the sum of weights of which entities + * are queued on this cfs_rq, so a weight of a entity should not be + * too large, so as the shares value of a task group. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ +#define MIN_SHARES (1UL << 1) +#define MAX_SHARES (1UL << 18) + extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); @@ -706,6 +707,7 @@ struct cfs_rq { /* * CFS load tracking */ + struct sched_entity *h_curr; struct sched_avg avg; #ifndef CONFIG_64BIT u64 last_update_time_copy; @@ -721,6 +723,7 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum; @@ -2018,7 +2021,8 @@ DEFINE_LOCK_GUARD_1(rq_lock, struct rq, rq_unlock(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock, _T) DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, @@ -2026,7 +2030,8 @@ DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, rq_unlock_irq(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irq, _T) DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, @@ -2034,9 +2039,20 @@ DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, rq_unlock_irqrestore(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, _T) +DEFINE_LOCK_GUARD_1(raw_spin_rq_lock_irqsave, struct rq, + raw_spin_rq_lock_irqsave(_T->lock, _T->flags), + raw_spin_rq_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + +DECLARE_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); +#define class_raw_spin_rq_lock_irqsave_constructor(_T) \ + WITH_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, _T) + #define this_rq_lock_irq(...) __acquire_ret(_this_rq_lock_irq(__VA_ARGS__), __rq_lockp(__ret)) static inline struct rq *_this_rq_lock_irq(struct rq_flags *rf) __acquires_ret { @@ -2571,6 +2587,7 @@ extern const u32 sched_prio_to_wmult[40]; #define ENQUEUE_MIGRATED 0x00040000 #define ENQUEUE_INITIAL 0x00080000 #define ENQUEUE_RQ_SELECTED 0x00100000 +#define ENQUEUE_QUEUED 0x00200000 #define RETRY_TASK ((void *)-1UL) diff --git a/kernel/smp.c b/kernel/smp.c index a0bb56bd8dda..8a847a34f132 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -182,16 +182,22 @@ static atomic_t csd_bug_count = ATOMIC_INIT(0); static void __csd_lock_record(call_single_data_t *csd) { if (!csd) { - smp_mb(); /* NULL cur_csd after unlock. */ - __this_cpu_write(cur_csd, NULL); + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): orders any preceding CSD + * callback/unlock before a remote reader observes NULL. + */ + smp_store_release(this_cpu_ptr(&cur_csd), NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); - smp_wmb(); /* func and info before csd. */ - __this_cpu_write(cur_csd, csd); - smp_mb(); /* Update cur_csd before function call. */ - /* Or before unlock, as the case may be. */ + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): publishes cur_csd_func and + * cur_csd_info before the non-NULL pointer becomes visible. + */ + smp_store_release(this_cpu_ptr(&cur_csd), csd); } static __always_inline void csd_lock_record(call_single_data_t *csd) @@ -272,7 +278,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in cpux = 0; else cpux = cpu; - cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ + /* + * Pairs with smp_store_release() of cur_csd in __csd_lock_record(): + * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info + * are the matching publication; a NULL value is ordered after any + * preceding CSD callback/unlock on the remote CPU. + */ + cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* How long since this CSD lock was stuck. */ ts_delta = ts2 - ts0; pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 313dcea127fe..2a278ae550a1 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2060,13 +2060,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc base->running = NULL; } -static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base) -{ - struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); - - return next ? hrtimer_from_timerqueue_node(next) : NULL; -} - static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, unsigned long flags, unsigned int active_mask) { diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0b..e5641fa60163 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd) static int probe_uprobe_syscall(int token_fd) { /* - * If kernel supports uprobe() syscall, it will return -ENXIO when called + * If kernel supports uprobe() syscall, it will return -EPROTO when called * from the outside of a kernel-generated uprobe trampoline. */ - return syscall(__NR_uprobe) < 0 && errno == ENXIO; + return syscall(__NR_uprobe) < 0 && errno == EPROTO; } #else static int probe_uprobe_syscall(int token_fd) diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index db9432adb967..2e56e3ab5b6c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) /* * Detect kernel support for uprobe() syscall, it's presence means we can - * take advantage of faster nop5 uprobe handling. + * take advantage of faster nop10 uprobe handling. * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") */ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); @@ -605,14 +605,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, #if defined(__x86_64__) static bool has_nop_combo(int fd, long off) { - unsigned char nop_combo[6] = { - 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; - unsigned char buf[6]; + unsigned char buf[11]; - if (pread(fd, buf, 6, off) != 6) + if (pread(fd, buf, 11, off) != 11) return false; - return memcmp(buf, nop_combo, 6) == 0; + return memcmp(buf, nop_combo, 11) == 0; } #else static bool has_nop_combo(int fd, long off) @@ -825,8 +825,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, memset(target, 0, sizeof(*target)); /* - * We have uprobe syscall and usdt with nop,nop5 instructions combo, - * so we can place the uprobe directly on nop5 (+1) and get this probe + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe * optimized. */ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 3d9d2cd7764b..c4a3a6b3eb83 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -539,12 +539,12 @@ extern const struct bench bench_trig_uretprobe_multi_push; extern const struct bench bench_trig_uprobe_multi_ret; extern const struct bench bench_trig_uretprobe_multi_ret; #ifdef __x86_64__ -extern const struct bench bench_trig_uprobe_nop5; -extern const struct bench bench_trig_uretprobe_nop5; -extern const struct bench bench_trig_uprobe_multi_nop5; -extern const struct bench bench_trig_uretprobe_multi_nop5; +extern const struct bench bench_trig_uprobe_nop10; +extern const struct bench bench_trig_uretprobe_nop10; +extern const struct bench bench_trig_uprobe_multi_nop10; +extern const struct bench bench_trig_uretprobe_multi_nop10; extern const struct bench bench_trig_usdt_nop; -extern const struct bench bench_trig_usdt_nop5; +extern const struct bench bench_trig_usdt_nop10; #endif extern const struct bench bench_rb_libbpf; @@ -622,12 +622,12 @@ static const struct bench *benchs[] = { &bench_trig_uprobe_multi_ret, &bench_trig_uretprobe_multi_ret, #ifdef __x86_64__ - &bench_trig_uprobe_nop5, - &bench_trig_uretprobe_nop5, - &bench_trig_uprobe_multi_nop5, - &bench_trig_uretprobe_multi_nop5, + &bench_trig_uprobe_nop10, + &bench_trig_uretprobe_nop10, + &bench_trig_uprobe_multi_nop10, + &bench_trig_uretprobe_multi_nop10, &bench_trig_usdt_nop, - &bench_trig_usdt_nop5, + &bench_trig_usdt_nop10, #endif /* ringbuf/perfbuf benchmarks */ &bench_rb_libbpf, diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c index 2f22ec61667b..61513efc167a 100644 --- a/tools/testing/selftests/bpf/benchs/bench_trigger.c +++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c @@ -396,15 +396,15 @@ static void *uprobe_producer_ret(void *input) } #ifdef __x86_64__ -__nocf_check __weak void uprobe_target_nop5(void) +__nocf_check __weak void uprobe_target_nop10(void) { - asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); + asm volatile (".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); } -static void *uprobe_producer_nop5(void *input) +static void *uprobe_producer_nop10(void *input) { while (true) - uprobe_target_nop5(); + uprobe_target_nop10(); return NULL; } @@ -418,7 +418,7 @@ static void *uprobe_producer_usdt_nop(void *input) return NULL; } -static void *uprobe_producer_usdt_nop5(void *input) +static void *uprobe_producer_usdt_nop10(void *input) { while (true) usdt_2(); @@ -542,24 +542,24 @@ static void uretprobe_multi_ret_setup(void) } #ifdef __x86_64__ -static void uprobe_nop5_setup(void) +static void uprobe_nop10_setup(void) { - usetup(false, false /* !use_multi */, &uprobe_target_nop5); + usetup(false, false /* !use_multi */, &uprobe_target_nop10); } -static void uretprobe_nop5_setup(void) +static void uretprobe_nop10_setup(void) { - usetup(true, false /* !use_multi */, &uprobe_target_nop5); + usetup(true, false /* !use_multi */, &uprobe_target_nop10); } -static void uprobe_multi_nop5_setup(void) +static void uprobe_multi_nop10_setup(void) { - usetup(false, true /* use_multi */, &uprobe_target_nop5); + usetup(false, true /* use_multi */, &uprobe_target_nop10); } -static void uretprobe_multi_nop5_setup(void) +static void uretprobe_multi_nop10_setup(void) { - usetup(true, true /* use_multi */, &uprobe_target_nop5); + usetup(true, true /* use_multi */, &uprobe_target_nop10); } static void usdt_setup(const char *name) @@ -598,7 +598,7 @@ static void usdt_nop_setup(void) usdt_setup("usdt_1"); } -static void usdt_nop5_setup(void) +static void usdt_nop10_setup(void) { usdt_setup("usdt_2"); } @@ -665,10 +665,10 @@ BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop"); BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push"); BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret"); #ifdef __x86_64__ -BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5"); -BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5"); -BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5"); -BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5"); +BENCH_TRIG_USERMODE(uprobe_nop10, nop10, "uprobe-nop10"); +BENCH_TRIG_USERMODE(uretprobe_nop10, nop10, "uretprobe-nop10"); +BENCH_TRIG_USERMODE(uprobe_multi_nop10, nop10, "uprobe-multi-nop10"); +BENCH_TRIG_USERMODE(uretprobe_multi_nop10, nop10, "uretprobe-multi-nop10"); BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop"); -BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5"); +BENCH_TRIG_USERMODE(usdt_nop10, usdt_nop10, "usdt-nop10"); #endif diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh index 9ec59423b949..e490b337e960 100755 --- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh +++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh @@ -2,7 +2,7 @@ set -eufo pipefail -for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5 +for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop10} usdt-nop usdt-nop10 do summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-) printf "%-15s: %s\n" $i "$summary" diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52..e193206fc5d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -4,6 +4,8 @@ #ifdef __x86_64__ +#define _GNU_SOURCE +#include <sched.h> #include <unistd.h> #include <asm/ptrace.h> #include <linux/compiler.h> @@ -13,11 +15,12 @@ #include <sys/syscall.h> #include <sys/prctl.h> #include <asm/prctl.h> +#include <stdnoreturn.h> #include "uprobe_syscall.skel.h" #include "uprobe_syscall_executed.skel.h" #include "bpf/libbpf_internal.h" -#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 +#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 #include "usdt.h" #pragma GCC diagnostic ignored "-Wattributes" @@ -26,7 +29,7 @@ __attribute__((aligned(16))) __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) { asm volatile ( - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */ + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ "movq $0xdeadbeef, %rax\n" "ret\n" ); @@ -345,9 +348,9 @@ cleanup: __attribute__((aligned(16))) __nocf_check __weak __naked void uprobe_test(void) { - asm volatile (" \n" - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n" - "ret \n" + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" ); } @@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void) USDT(optimized_uprobe, usdt); } +/* + * Assembly-level red zone clobbering test. Stores known values in the + * red zone (below RSP), executes a nop10 (uprobe site), and checks that + * the values survived. Returns 0 if intact, 1 if clobbered. + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + * + * Align the code at 64 bytes, to make sure nop10 is not on page boundary. + */ +__attribute__((aligned(64))) +__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) +{ + asm volatile ( + "movabs $0x1111111111111111, %%rax\n" + "movq %%rax, -8(%%rsp)\n" + "movabs $0x2222222222222222, %%rax\n" + "movq %%rax, -16(%%rsp)\n" + "movabs $0x3333333333333333, %%rax\n" + "movq %%rax, -24(%%rsp)\n" + + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ + + "movabs $0x1111111111111111, %%rax\n" + "cmpq %%rax, -8(%%rsp)\n" + "jne 1f\n" + "movabs $0x2222222222222222, %%rax\n" + "cmpq %%rax, -16(%%rsp)\n" + "jne 1f\n" + "movabs $0x3333333333333333, %%rax\n" + "cmpq %%rax, -24(%%rsp)\n" + "jne 1f\n" + + "xorl %%eax, %%eax\n" + "retq\n" + "1:\n" + "movl $1, %%eax\n" + "retq\n" + ::: "rax", "memory" + ); +} + static int find_uprobes_trampoline(void *tramp_addr) { void *start, *end; @@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr) return ret; } -static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; -static void *find_nop5(void *fn) +static void *find_nop10(void *fn) { int i; - for (i = 0; i < 10; i++) { - if (!memcmp(nop5, fn + i, 5)) + for (i = 0; i < 128; i++) { + if (!memcmp(nop10, fn + i, 10)) return fn + i; } return NULL; @@ -403,6 +451,16 @@ static void *find_nop5(void *fn) typedef void (__attribute__((nocf_check)) *trigger_t)(void); +static void check_attach_notrigger(struct uprobe_syscall_executed *skel, + void *addr, int executed) +{ + unsigned char *op = addr; + + /* Make sure bpf program was not executed. */ + ASSERT_EQ(skel->bss->executed, executed, "executed"); + ASSERT_EQ(*op, 0xcc, "int3"); +} + static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, void *addr, int executed) { @@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge ASSERT_EQ(skel->bss->executed, executed, "executed"); /* .. and check the trampoline is as expected. */ - call = (struct __arch_relative_insn *) addr; + ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); + call = (struct __arch_relative_insn *)(addr + 5); tramp = (void *) (call + 1) + call->raddr; ASSERT_EQ(call->op, 0xe8, "call"); ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); @@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge return tramp; } -static void check_detach(void *addr, void *tramp) +static bool check_detach(void *addr, void *tramp) { + static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; + bool ok = true; + /* [uprobes_trampoline] stays after detach */ - ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); - ASSERT_OK(memcmp(addr, nop5, 5), "nop5"); + ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); + ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); + return ok; } -static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link, - trigger_t trigger, void *addr, int executed) +static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, + trigger_t trigger, void *addr, int executed) { void *tramp; tramp = check_attach(skel, trigger, addr, executed); bpf_link__destroy(link); check_detach(addr, tramp); + return tramp; } static void test_uprobe_legacy(void) @@ -453,6 +517,7 @@ static void test_uprobe_legacy(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -470,7 +535,30 @@ static void test_uprobe_legacy(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe */ skel->bss->executed = 0; @@ -492,6 +580,7 @@ static void test_uprobe_multi(void) LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -512,7 +601,30 @@ static void test_uprobe_multi(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe.multi */ skel->bss->executed = 0; @@ -536,6 +648,7 @@ static void test_uprobe_session(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -555,7 +668,30 @@ static void test_uprobe_session(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 4); + tramp = check(skel, link, uprobe_test, uprobe_test, 4); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 4); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 8); cleanup: uprobe_syscall_executed__destroy(skel); @@ -565,11 +701,11 @@ static void test_uprobe_usdt(void) { struct uprobe_syscall_executed *skel; struct bpf_link *link; - void *addr; + void *addr, *tramp; errno = 0; - addr = find_nop5(usdt_test); - if (!ASSERT_OK_PTR(addr, "find_nop5")) + addr = find_nop10(usdt_test); + if (!ASSERT_OK_PTR(addr, "find_nop10")) return; skel = uprobe_syscall_executed__open_and_load(); @@ -584,7 +720,32 @@ static void test_uprobe_usdt(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) goto cleanup; - check(skel, link, usdt_test, addr, 2); + tramp = check(skel, link, usdt_test, addr, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check_attach_notrigger(skel, addr, 2); + + bpf_link__destroy(link); + if (!check_detach(addr, tramp)) + goto cleanup; + + usdt_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check(skel, link, usdt_test, addr, 4); cleanup: uprobe_syscall_executed__destroy(skel); @@ -757,12 +918,124 @@ cleanup: #define __NR_uprobe 336 #endif +static void test_uprobe_red_zone(void) +{ + struct uprobe_syscall_executed *skel; + struct bpf_link *link; + void *nop10_addr; + size_t offset; + int i; + + nop10_addr = find_nop10(uprobe_red_zone_test); + if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + offset = get_uprobe_offset(nop10_addr); + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "attach_uprobe")) + goto cleanup; + + for (i = 0; i < 10; i++) + ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); + + bpf_link__destroy(link); + +cleanup: + uprobe_syscall_executed__destroy(skel); +} + static void test_uprobe_error(void) { long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); +} + +__attribute__((aligned(16))) +__nocf_check __weak __naked void uprobe_fork_test(void) +{ + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" + ); +} + +static noreturn int child_func(void *arg) +{ + struct uprobe_syscall_executed *skel = arg; + + /* Make sure the child's probe is still there and optimized.. */ + if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) + _exit(1); + + skel->bss->pid = getpid(); + + /* .. and it executes properly. */ + uprobe_fork_test(); + + if (skel->bss->executed != 3) + _exit(2); + + _exit(0); +} + +static void test_uprobe_fork_optimized(bool clone_vm) +{ + struct uprobe_syscall_executed *skel = NULL; + unsigned long offset; + int pid, status, err; + char stack[65535]; + + offset = get_uprobe_offset(&uprobe_fork_test); + if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + -1, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) + goto cleanup; + + skel->bss->pid = getpid(); + + /* Trigger optimization of uprobe in uprobe_fork_test. */ + uprobe_fork_test(); + uprobe_fork_test(); + + /* Make sure it got optimied. */ + if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) + goto cleanup; + + if (clone_vm) { + pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); + if (!ASSERT_GT(pid, 0, "clone")) + goto cleanup; + } else { + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto cleanup; + if (pid == 0) + child_func(skel); + } + + /* Wait for the child and verify it exited properly with 0. */ + err = waitpid(pid, &status, 0); + if (ASSERT_EQ(err, pid, "waitpid")) { + ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); + ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); + } + +cleanup: + uprobe_syscall_executed__destroy(skel); } static void __test_uprobe_syscall(void) @@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void) test_uprobe_usdt(); if (test__start_subtest("uprobe_race")) test_uprobe_race(); + if (test__start_subtest("uprobe_red_zone")) + test_uprobe_red_zone(); + if (test__start_subtest("uprobe_optimized_fork")) + test_uprobe_fork_optimized(false); + if (test__start_subtest("uprobe_optimized_clone_vm")) + test_uprobe_fork_optimized(true); if (test__start_subtest("uprobe_error")) test_uprobe_error(); if (test__start_subtest("uprobe_regs_equal")) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 69759b27794d..8004c9568ffa 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,9 +250,10 @@ cleanup: #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; -static unsigned char nop1_nop5_combo[6] = { 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; static void *find_instr(void *fn, unsigned char *instr, size_t cnt) { @@ -271,17 +272,17 @@ static void subtest_optimized_attach(void) __u8 *addr_1, *addr_2; /* usdt_1 USDT probe has single nop instruction */ - addr_1 = find_instr(usdt_1, nop1_nop5_combo, 6); - if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop5_combo")) + addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11); + if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo")) return; addr_1 = find_instr(usdt_1, nop1, 1); if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1")) return; - /* usdt_2 USDT probe has nop,nop5 instructions combo */ - addr_2 = find_instr(usdt_2, nop1_nop5_combo, 6); - if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop5_combo")) + /* usdt_2 USDT probe has nop,nop10 instructions combo */ + addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11); + if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo")) return; skel = test_usdt__open_and_load(); @@ -309,12 +310,12 @@ static void subtest_optimized_attach(void) bpf_link__destroy(skel->links.usdt_executed); - /* we expect the nop5 ip */ + /* we expect the nop10 ip */ skel->bss->expected_ip = (unsigned long) addr_2 + 1; /* * Attach program on top of usdt_2 which is probe defined on top - * of nop1,nop5 combo, so the probe gets optimized on top of nop5. + * of nop1,nop10 combo, so the probe gets optimized on top of nop10. */ skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, 0 /*self*/, "/proc/self/exe", @@ -328,13 +329,64 @@ static void subtest_optimized_attach(void) /* nop stays on addr_2 address */ ASSERT_EQ(*addr_2, 0x90, "nop"); - /* call is on addr_2 + 1 address */ - ASSERT_EQ(*(addr_2 + 1), 0xe8, "call"); + /* + * lea -0x80(%rsp), %rsp + * call ... + */ + static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 }; + + ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call"); ASSERT_EQ(skel->bss->executed, 4, "executed"); cleanup: test_usdt__destroy(skel); } + +/* + * Test that USDT arguments survive nop10 optimization in a function where + * the compiler places operands in the red zone. + * + * Signal handlers are prone to having the compiler place USDT argument + * operands in the red zone (below rsp). + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + */ +static void subtest_optimized_red_zone(void) +{ + struct test_usdt *skel; + int i; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->expected_arg[0] = 0xDEADBEEF; + skel->bss->expected_arg[1] = 0xCAFEBABE; + skel->bss->expected_arg[2] = 0xFEEDFACE; + skel->bss->expected_pid = getpid(); + + skel->links.usdt_check_arg = bpf_program__attach_usdt( + skel->progs.usdt_check_arg, 0, "/proc/self/exe", + "optimized_attach", "usdt_red_zone", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone")) + goto cleanup; + + for (i = 0; i < 10; i++) + usdt_red_zone_trigger(); + + ASSERT_EQ(skel->bss->arg_total, 10, "arg_total"); + ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad"); + ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1"); + ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2"); + ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3"); + +cleanup: + test_usdt__destroy(skel); +} + #endif unsigned short test_usdt_100_semaphore SEC(".probes"); @@ -608,6 +660,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_red_zone")) + subtest_optimized_red_zone(); #endif if (test__start_subtest("multispec")) subtest_multispec_usdt(); diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c index f00cb52874e0..0ee78fb050a1 100644 --- a/tools/testing/selftests/bpf/progs/test_usdt.c +++ b/tools/testing/selftests/bpf/progs/test_usdt.c @@ -149,5 +149,30 @@ int usdt_executed(struct pt_regs *ctx) executed++; return 0; } + +int arg_total; +int arg_bad; +long arg_last[3]; +long expected_arg[3]; +int expected_pid; + +SEC("usdt") +int BPF_USDT(usdt_check_arg, long arg1, long arg2, long arg3) +{ + if (expected_pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; + + __sync_fetch_and_add(&arg_total, 1); + arg_last[0] = arg1; + arg_last[1] = arg2; + arg_last[2] = arg3; + + if (arg1 != expected_arg[0] || + arg2 != expected_arg[1] || + arg3 != expected_arg[2]) + __sync_fetch_and_add(&arg_bad, 1); + + return 0; +} #endif char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h index c71e21df38b3..75687f50f4e2 100644 --- a/tools/testing/selftests/bpf/usdt.h +++ b/tools/testing/selftests/bpf/usdt.h @@ -313,7 +313,7 @@ struct usdt_sema { volatile unsigned short active; }; #if defined(__ia64__) || defined(__s390__) || defined(__s390x__) #define USDT_NOP nop 0 #elif defined(__x86_64__) -#define USDT_NOP .byte 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x0 /* nop, nop5 */ +#define USDT_NOP .byte 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 /* nop, nop10 */ #else #define USDT_NOP nop #endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 789883aaca4c..5e38f8605b02 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -3,7 +3,7 @@ #if defined(__x86_64__) /* - * Include usdt.h with default nop,nop5 instructions combo. + * Include usdt.h with default nop,nop10 instructions combo. */ #include "usdt.h" @@ -13,4 +13,17 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; +static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; +static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; + +void __attribute__((noinline)) usdt_red_zone_trigger(void) +{ + unsigned long a1 = usdt_red_zone_arg1; + unsigned long a2 = usdt_red_zone_arg2; + unsigned long a3 = usdt_red_zone_arg3; + + USDT(optimized_attach, usdt_red_zone, a1, a2, a3); +} + #endif diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h index d3133587d996..5a29d6bec51f 100644 --- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -8,13 +8,11 @@ #ifndef _RSEQ_X86_THREAD_POINTER #define _RSEQ_X86_THREAD_POINTER -#include <features.h> - #ifdef __cplusplus extern "C" { #endif -#if __GNUC_PREREQ (11, 1) +#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1) static inline void *rseq_thread_pointer(void) { return __builtin_thread_pointer(); |