diff options
| -rw-r--r-- | Documentation/core-api/entry.rst | 45 | ||||
| -rw-r--r-- | arch/loongarch/kernel/syscall.c | 14 | ||||
| -rw-r--r-- | arch/powerpc/kernel/syscall.c | 3 | ||||
| -rw-r--r-- | arch/riscv/kernel/traps.c | 11 | ||||
| -rw-r--r-- | arch/s390/kernel/syscall.c | 7 | ||||
| -rw-r--r-- | arch/x86/entry/syscall_32.c | 25 | ||||
| -rw-r--r-- | arch/x86/entry/syscall_64.c | 12 | ||||
| -rw-r--r-- | include/linux/entry-common.h | 32 |
8 files changed, 88 insertions, 61 deletions
diff --git a/Documentation/core-api/entry.rst b/Documentation/core-api/entry.rst index 44e9608038ea..79fdaed954d9 100644 --- a/Documentation/core-api/entry.rst +++ b/Documentation/core-api/entry.rst @@ -58,26 +58,51 @@ state transitions must run with interrupts disabled. Syscalls -------- -Syscall-entry code starts in assembly code and calls out into low-level C code -after establishing low-level architecture-specific state and stack frames. This -low-level C code must not be instrumented. A typical syscall handling function -invoked from low-level assembly code looks like this: +Syscall-entry code starts in assembly code and calls out into low-level C +code after establishing low-level architecture-specific state and stack +frames. This low-level C code must not be instrumented. The recommended +syscall handling function invoked from low-level assembly code looks like +this: .. code-block:: c - noinstr void syscall(struct pt_regs *regs, int nr) + noinstr void syscall(struct pt_regs *regs, long nr) { arch_syscall_enter(regs); - nr = syscall_enter_from_user_mode_randomize_stack(regs, nr); + result_reg(regs) = -ENOSYS; + if (syscall_enter_from_user_mode_randomize_stack(regs, &nr)) { + instrumentation_begin(); + if (valid(nr) + result_reg(regs) = invoke_syscall(regs, nr); + instrumentation_end(); + } + syscall_exit_to_user_mode(regs); + } - instrumentation_begin(); - if (!invoke_syscall(regs, nr) && nr != -1) - result_reg(regs) = __sys_ni_syscall(regs); - instrumentation_end(); +This is the most resilent variant as it has always a guaranteed valid +return code. The alternative variant is: +.. code-block:: c + + noinstr void syscall(struct pt_regs *regs, long nr) + { + arch_syscall_enter(regs); + if (syscall_enter_from_user_mode_randomize_stack(regs, &nr)) { + instrumentation_begin(); + if (valid(nr) + result_reg(regs) = invoke_syscall(regs, nr); + else + result_reg(regs) = -ENOSYS; + instrumentation_end(); + } syscall_exit_to_user_mode(regs); } +That works for most situations except when a probe/BPF attached to the +syscall tracepoint sets an invalid syscall number e.g. -1 and also modifies +the result register. So this variant will obviously overwrite the modified +result with -ENOSYS. + syscall_enter_from_user_mode_randomize_stack() first invokes enter_from_user_mode_randomize_stack() which establishes state in the following order: diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c index 142a9c20292c..62264e422467 100644 --- a/arch/loongarch/kernel/syscall.c +++ b/arch/loongarch/kernel/syscall.c @@ -57,8 +57,8 @@ typedef long (*sys_call_fn)(unsigned long, unsigned long, void noinstr __no_stack_protector do_syscall(struct pt_regs *regs) { - unsigned long nr; sys_call_fn syscall_fn; + unsigned long nr; nr = regs->regs[11]; /* Set for syscall restarting */ @@ -69,12 +69,12 @@ void noinstr __no_stack_protector do_syscall(struct pt_regs *regs) regs->orig_a0 = regs->regs[4]; regs->regs[4] = -ENOSYS; - nr = syscall_enter_from_user_mode_randomize_stack(regs, nr); - - if (nr < NR_syscalls) { - syscall_fn = sys_call_table[array_index_nospec(nr, NR_syscalls)]; - regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6], - regs->regs[7], regs->regs[8], regs->regs[9]); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) { + if (nr < NR_syscalls) { + syscall_fn = sys_call_table[array_index_nospec(nr, NR_syscalls)]; + regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6], + regs->regs[7], regs->regs[8], regs->regs[9]); + } } syscall_exit_to_user_mode(regs); diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c index b4d0159e29f8..1440dcabe052 100644 --- a/arch/powerpc/kernel/syscall.c +++ b/arch/powerpc/kernel/syscall.c @@ -18,7 +18,8 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0) long ret; syscall_fn f; - r0 = syscall_enter_from_user_mode_randomize_stack(regs, r0); + if (unlikely(!syscall_enter_from_user_mode_randomize_stack(regs, &r0))) + return syscall_get_error(current, regs); if (unlikely(r0 >= NR_syscalls)) { if (unlikely(trap_is_unsupported_scv(regs))) { diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index bb180116ef0a..2f57fd48ec28 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -332,13 +332,12 @@ void do_trap_ecall_u(struct pt_regs *regs) riscv_v_vstate_discard(regs); - syscall = syscall_enter_from_user_mode_randomize_stack(regs, syscall); - - if (syscall >= 0 && syscall < NR_syscalls) { - syscall = array_index_nospec(syscall, NR_syscalls); - syscall_handler(regs, syscall); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) { + if (syscall >= 0 && syscall < NR_syscalls) { + syscall = array_index_nospec(syscall, NR_syscalls); + syscall_handler(regs, syscall); + } } - syscall_exit_to_user_mode(regs); } else { irqentry_state_t state = irqentry_nmi_enter(regs); diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 73abda2a9080..4ac80e2478d7 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -96,6 +96,7 @@ SYSCALL_DEFINE0(ni_syscall) void noinstr __do_syscall(struct pt_regs *regs, int per_trap) { unsigned long nr; + bool permit; enter_from_user_mode_randomize_stack(regs); @@ -121,7 +122,9 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) regs->psw.addr = current->restart_block.arch_data; current->restart_block.arch_data = 1; } - nr = syscall_enter_from_user_mode_work(regs, nr); + + permit = syscall_enter_from_user_mode_work(regs, &nr); + /* * In the s390 ptrace ABI, both the syscall number and the return value * use gpr2. However, userspace puts the syscall number either in the @@ -129,7 +132,7 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) * work, the ptrace code sets PIF_SYSCALL_RET_SET, which is checked here * and if set, the syscall will be skipped. */ - if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) + if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET) || !permit)) goto out; regs->gprs[2] = -ENOSYS; if (likely(nr < NR_syscalls)) { diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c index 92369d84c0c8..91123a90710c 100644 --- a/arch/x86/entry/syscall_32.c +++ b/arch/x86/entry/syscall_32.c @@ -161,8 +161,9 @@ __visible noinstr void do_int80_emulation(struct pt_regs *regs) nr = syscall_32_enter(regs); local_irq_enable(); - nr = syscall_enter_from_user_mode_work(regs, nr); - do_syscall_32_irqs_on(regs, nr); + + if (likely(syscall_enter_from_user_mode_work(regs, &nr))) + do_syscall_32_irqs_on(regs, nr); instrumentation_end(); syscall_exit_to_user_mode(regs); @@ -223,8 +224,8 @@ DEFINE_FREDENTRY_RAW(int80_emulation) nr = syscall_32_enter(regs); local_irq_enable(); - nr = syscall_enter_from_user_mode_work(regs, nr); - do_syscall_32_irqs_on(regs, nr); + if (likely(syscall_enter_from_user_mode_work(regs, &nr))) + do_syscall_32_irqs_on(regs, nr); instrumentation_end(); syscall_exit_to_user_mode(regs); @@ -243,13 +244,13 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) * orig_ax, the int return value truncates it. This matches * the semantics of syscall_get_nr(). */ - nr = syscall_enter_from_user_mode_randomize_stack(regs, nr); - - instrumentation_begin(); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) { + instrumentation_begin(); - do_syscall_32_irqs_on(regs, nr); + do_syscall_32_irqs_on(regs, nr); - instrumentation_end(); + instrumentation_end(); + } syscall_exit_to_user_mode(regs); } #endif /* !CONFIG_IA32_EMULATION */ @@ -286,10 +287,8 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) return false; } - nr = syscall_enter_from_user_mode_work(regs, nr); - - /* Now this is just like a normal syscall. */ - do_syscall_32_irqs_on(regs, nr); + if (likely(syscall_enter_from_user_mode_work(regs, &nr))) + do_syscall_32_irqs_on(regs, nr); instrumentation_end(); syscall_exit_to_user_mode(regs); diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c index 9a2762e32ae8..966d3d1b7586 100644 --- a/arch/x86/entry/syscall_64.c +++ b/arch/x86/entry/syscall_64.c @@ -78,14 +78,14 @@ static __always_inline void do_syscall_x32(struct pt_regs *regs, unsigned long n /* Returns true to return using SYSRET, or false to use IRET */ __visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr) { - nr = syscall_enter_from_user_mode_randomize_stack(regs, nr); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) { + instrumentation_begin(); - instrumentation_begin(); + if (!do_syscall_x64(regs, nr)) + do_syscall_x32(regs, nr); - if (!do_syscall_x64(regs, nr)) - do_syscall_x32(regs, nr); - - instrumentation_end(); + instrumentation_end(); + } syscall_exit_to_user_mode(regs); /* diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index df221d919e76..6574b7183c01 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -114,16 +114,15 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l * @regs: Pointer to currents pt_regs * @syscall: The syscall number * - * Invoked from architecture specific syscall entry code with interrupts - * enabled after invoking enter_from_user_mode(), enabling interrupts and - * extra architecture specific work. + * Invoked from architecture specific syscall entry code with interrupts enabled + * after invoking enter_from_user_mode(), enabling interrupts and extra + * architecture specific work with the syscall return value preset to -ENOSYS. * - * Returns: The original or a modified syscall number + * Returns: True if the syscall should be invoked, False otherwise. * - * If the returned syscall number is -1 then the syscall should be - * skipped. In this case the caller may invoke syscall_set_error() or - * syscall_set_return_value() first. If neither of those are called and -1 - * is returned, then the syscall will fail with ENOSYS. + * If the return value is false, the caller must skip the syscall and leave the + * syscall return value unmodified as it might have been set by one of the entry + * work functions. * * It handles the following work items: * @@ -131,19 +130,20 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l * ptrace_report_syscall_permit_entry(), __seccomp_permit_syscall(), trace_sys_enter() * 2) Invocation of audit_syscall_entry() */ -static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall) +static __always_inline bool syscall_enter_from_user_mode_work(struct pt_regs *regs, long *syscall) { unsigned long work = READ_ONCE(current_thread_info()->syscall_work); - if (work & SYSCALL_WORK_ENTER) { - if (!syscall_trace_enter(regs, work, syscall)) - return -1L; + if (!(work & SYSCALL_WORK_ENTER)) + return true; - /* Reread the syscall number as it might have been modified */ - syscall = syscall_get_nr(current, regs); - } + if (unlikely(!syscall_trace_enter(regs, work, *syscall))) + return false; - return syscall; + /* Reread the syscall number as it might have been modified */ + *syscall = syscall_get_nr(current, regs); + + return true; } /** |