diff options
| author | Danilo Krummrich <dakr@kernel.org> | 2026-07-13 23:50:39 +0200 |
|---|---|---|
| committer | Danilo Krummrich <dakr@kernel.org> | 2026-07-13 23:50:39 +0200 |
| commit | b07fc8d60bd30caaba4d293929459780166da194 (patch) | |
| tree | 4f47c377f5b9c8078f02ef883322719f4109b167 | |
| parent | a56a92be1fa9c6e747fc754c90734ed90cd36670 (diff) | |
| parent | 11a4784f902ebf3e674dbaf07dbec9a37aabb5e4 (diff) | |
| download | linux-next-b07fc8d60bd30caaba4d293929459780166da194.tar.gz linux-next-b07fc8d60bd30caaba4d293929459780166da194.zip | |
Merge patch series "rust: I/O type generalization and projection"
Gary Guo <gary@garyguo.net> says:
This series presents a major rework of I/O types, as a summary:
- Make I/O regions typed. The existing untyped region still exists
with a dynamically sized `Region` type.
- Create I/O view types to represent subregion of a full I/O region mapped.
A projection macro is added to allow safely create such subviews.
- Split I/O traits, make I/O views play a central role, avoid
duplicate monomorphization and less `unsafe` code.
- Add a `SysMem` backend, and make `Coherent` implement `Io`.
- Add copying methods (memcpy_{from,to}io and friends).
This series generalize `Mmio` type from just an untyped region to typed
representations (so `MmioRaw<T>` is `__iomem *T`). This allows us to remove
the `IoKnownSize` trait; the information is sourced from just the pointer
from the `KnownSize` trait instead.
Building on top of that, `Mmio` and `ConfigSpace` have been converted to
typed views of I/O regions rather than just a big chunk of untyped I/O
memory. These changes made it possible to implement `Io` trait for
`Coherent<T>`.
Shared system memory, `SysMem` is also added to the series, given it
similarity in implementation compared to `Coherent`. In fact, the series
use `SysMem` to implement `Coherent`'s I/O methods.
Built on these generalization, this series add `io_project!()`.
`io_project!()` performs a safe way to project a bigger view to a small
subviews, and some Nova code has been converted in this series to
demonstrate cleanups possible with this addition.
New `io_read!()`, `io_write!()` has been added that supersedes
`dma_read!()`, `dma_write!()` macro. Although, they work for primitives
only (to be exact, types that the backend is `IoCapable` of).
One feature that was lost from the old `dma_read!()` and `dma_write!()`
series was the ability to read/write a large structs. However, the
semantics was unclear to begin with, as there was no guarantee about their
atomicity even for structs that were small enough to fit in u32.
Suggested-by: Danilo Krummrich <dakr@kernel.org>
Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078
Link: https://patch.msgid.link/20260706-io_projection-v6-0-72cd5d055d54@garyguo.net
Signed-off-by: Danilo Krummrich <dakr@kernel.org>
| -rw-r--r-- | drivers/gpu/nova-core/gsp.rs | 53 | ||||
| -rw-r--r-- | drivers/gpu/nova-core/gsp/cmdq.rs | 66 | ||||
| -rw-r--r-- | drivers/gpu/nova-core/gsp/fw.rs | 82 | ||||
| -rw-r--r-- | drivers/pwm/pwm_th1520.rs | 7 | ||||
| -rw-r--r-- | rust/helpers/io.c | 13 | ||||
| -rw-r--r-- | rust/kernel/devres.rs | 24 | ||||
| -rw-r--r-- | rust/kernel/dma.rs | 281 | ||||
| -rw-r--r-- | rust/kernel/io.rs | 1502 | ||||
| -rw-r--r-- | rust/kernel/io/mem.rs | 29 | ||||
| -rw-r--r-- | rust/kernel/io/poll.rs | 6 | ||||
| -rw-r--r-- | rust/kernel/io/register.rs | 44 | ||||
| -rw-r--r-- | rust/kernel/lib.rs | 3 | ||||
| -rw-r--r-- | rust/kernel/pci.rs | 1 | ||||
| -rw-r--r-- | rust/kernel/pci/io.rs | 166 | ||||
| -rw-r--r-- | rust/kernel/ptr.rs | 12 | ||||
| -rw-r--r-- | samples/rust/rust_dma.rs | 12 |
16 files changed, 1626 insertions, 675 deletions
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 69175ca3315c..cfa7553cd820 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -9,14 +9,16 @@ use kernel::{ dma::{ Coherent, CoherentBox, + CoherentView, DmaAddress, // }, + io::{ + io_project, + io_write, + Io, // + }, pci, - prelude::*, - transmute::{ - AsBytes, - FromBytes, // - }, // + prelude::*, // }; pub(crate) mod cmdq; @@ -48,21 +50,21 @@ const LOG_BUFFER_SIZE: usize = RM_LOG_BUFFER_NUM_PAGES * GSP_PAGE_SIZE; /// Array of page table entries, as understood by the GSP bootloader. #[repr(C)] +#[derive(FromBytes, IntoBytes)] struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); -/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one. -unsafe impl<const NUM_ENTRIES: usize> FromBytes for PteArray<NUM_ENTRIES> {} - -/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one. -unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {} - impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { - /// Returns the page table entry for `index`, for a mapping starting at `start`. - // TODO: Replace with `IoView` projection once available. - fn entry(start: DmaAddress, index: usize) -> Result<u64> { - start - .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) - .ok_or(EOVERFLOW) + /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. + fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> { + for i in 0..NUM_PAGES { + io_write!(view, .0[build: i], + start + .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) + .ok_or(EOVERFLOW)? + ); + } + + Ok(()) } } @@ -89,17 +91,12 @@ impl LogBuffer { let start_addr = obj.0.dma_handle(); - // SAFETY: `obj` has just been created and we are its sole user. - let pte_region = unsafe { - &mut obj.0.as_mut()[size_of::<u64>()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()] - }; - - // Write values one by one to avoid an on-stack instance of `PteArray`. - for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() { - let pte_value = PteArray::<0>::entry(start_addr, i)?; - - chunk.copy_from_slice(&pte_value.to_ne_bytes()); - } + let pte_view = io_project!( + obj.0, + [build: size_of::<u64>()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()] + ) + .try_cast::<PteArray<RM_LOG_BUFFER_NUM_PAGES>>()?; + PteArray::init(pte_view, start_addr)?; Ok(obj) } diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 070de0731e95..c34b48961496 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,16 +2,23 @@ mod continuation; -use core::mem; +use core::{ + mem, + sync::atomic::{ + fence, + Ordering, // + }, +}; use kernel::{ device, dma::{ Coherent, + CoherentBox, DmaAddress, // }, - dma_write, io::{ + io_project, poll::read_poll_timeout, Io, // }, @@ -171,20 +178,18 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); #[repr(C)] // There is no struct defined for this in the open-gpu-kernel-source headers. // Instead it is defined by code in `GspMsgQueuesInit()`. -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct Msgq { +struct Msgq { /// Header for sending messages, including the write pointer. - pub(super) tx: MsgqTxHeader, + tx: MsgqTxHeader, /// Header for receiving messages, including the read pointer. - pub(super) rx: MsgqRxHeader, + rx: MsgqRxHeader, /// The message queue proper. msgq: MsgqData, } /// Structure shared between the driver and the GSP and containing the command and message queues. #[repr(C)] -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct GspMem { +struct GspMem { /// Self-mapping page table entries. ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the @@ -192,13 +197,13 @@ pub(super) struct GspMem { /// index into the GSP queue. /// /// This member is read-only for the GSP. - pub(super) cpuq: Msgq, + cpuq: Msgq, /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the /// write and read pointers that the GSP updates. This means that the read pointer here is an /// index into the CPU queue. /// /// This member is read-only for the driver. - pub(super) gspq: Msgq, + gspq: Msgq, } impl GspMem { @@ -232,20 +237,12 @@ impl DmaGspMem { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let gsp_mem = Coherent::<GspMem>::zeroed(dev, GFP_KERNEL)?; - - let start = gsp_mem.dma_handle(); - // Write values one by one to avoid an on-stack instance of `PteArray`. - for i in 0..GspMem::PTE_ARRAY_SIZE { - dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?); - } + let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?; + gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); + gsp_mem.cpuq.rx = MsgqRxHeader::new(); - dma_write!( - gsp_mem, - .cpuq.tx, - MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) - ); - dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new()); + let gsp_mem: Coherent<_> = gsp_mem.into(); + PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?; Ok(Self(gsp_mem)) } @@ -406,7 +403,7 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_write_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the GSP will read the next command from. @@ -415,7 +412,7 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the CPU can read the next message from. @@ -424,12 +421,18 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_read_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) + let rx = io_project!(self.0, .cpuq.rx); + let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + + // Ensure read pointer is properly ordered. + fence(Ordering::SeqCst); + + MsgqRxHeader::set_read_ptr(rx, rptr) } // Returns the index of the memory page the CPU can write the next command to. @@ -438,12 +441,17 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_write_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) + let tx = io_project!(self.0, .cpuq.tx); + let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + MsgqTxHeader::set_write_ptr(tx, wptr); + + // Ensure all command data is visible before triggering the GSP read. + fence(Ordering::SeqCst); } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 4db0cfa4dc4d..b0e7de328eaf 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -10,7 +10,14 @@ use r570_144 as bindings; use core::ops::Range; use kernel::{ - dma::Coherent, + dma::{ + Coherent, + CoherentView, // + }, + io::{ + io_read, + io_write, // + }, prelude::*, ptr::{ Alignable, @@ -44,59 +51,6 @@ use crate::{ }, }; -// TODO: Replace with `IoView` projections once available. -pub(super) mod gsp_mem { - use core::sync::atomic::{ - fence, - Ordering, // - }; - - use kernel::{ - dma::Coherent, - dma_read, - dma_write, // - }; - - use crate::gsp::cmdq::{ - GspMem, - MSGQ_NUM_PAGES, // - }; - - pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) { - let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - - dma_write!(qs, .cpuq.rx.0.readPtr, rptr); - } - - pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) { - let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - dma_write!(qs, .cpuq.tx.0.writePtr, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); - } -} - /// Maximum size of a single GSP message queue element in bytes. pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); @@ -720,6 +674,16 @@ impl MsgqTxHeader { entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), }) } + + /// Returns the value of the write pointer for this queue. + pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.writePtr) + } + + /// Sets the value of the write pointer for this queue. + pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.writePtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -735,6 +699,16 @@ impl MsgqRxHeader { pub(crate) fn new() -> Self { Self(Default::default()) } + + /// Returns the value of the read pointer for this queue. + pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.readPtr) + } + + /// Sets the value of the read pointer for this queue. + pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.readPtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index 7102c60fd14b..9372cd3e6a6d 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -20,7 +20,6 @@ //! this method is not used in this driver. //! -use core::ops::Deref; use kernel::{ clk::Clk, device::{Bound, Core, Device}, @@ -212,8 +211,7 @@ impl pwm::PwmOps for Th1520PwmDriverData { ) -> Result<Self::WfHw> { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let ctrl = iomap.try_read32(th1520_pwm_ctrl(hwpwm))?; let period_cycles = iomap.try_read32(th1520_pwm_per(hwpwm))?; @@ -247,8 +245,7 @@ impl pwm::PwmOps for Th1520PwmDriverData { ) -> Result { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let duty_cycles = iomap.try_read32(th1520_pwm_fp(hwpwm))?; let was_enabled = duty_cycles != 0; diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 397810864a24..7ed9a4f77f1b 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) iounmap(addr); } +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} + __rust_helper u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index b7c075a39ba4..20f94030f977 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -70,17 +70,19 @@ struct Inner<T> { /// devres::Devres, /// io::{ /// Io, -/// IoKnownSize, +/// IoBase, /// Mmio, /// MmioRaw, -/// PhysAddr, // +/// MmioBackend, +/// PhysAddr, +/// Region, // /// }, /// prelude::*, /// }; /// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); +/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>); /// /// impl<const SIZE: usize> IoMem<SIZE> { /// /// # Safety @@ -95,7 +97,7 @@ struct Inner<T> { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// @@ -106,12 +108,13 @@ struct Inner<T> { /// } /// } /// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> { +/// type Backend = MmioBackend; +/// type Target = Region<SIZE>; /// -/// fn deref(&self) -> &Self::Target { +/// fn as_view(self) -> Mmio<'a, Region<SIZE>> { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { Mmio::from_raw(self.0) } /// } /// } /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> { @@ -299,10 +302,7 @@ impl<T: Send + 'static> Devres<T> { /// use kernel::{ /// device::Core, /// devres::Devres, - /// io::{ - /// Io, - /// IoKnownSize, // - /// }, + /// io::Io, /// pci, // /// }; /// diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 200def84fb69..e275f2562a5b 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -14,14 +14,22 @@ use crate::{ }, error::to_result, fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, prelude::*, ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, FromBytes, // - }, // - uaccess::UserSliceWriter, + }, + uaccess::UserSliceWriter, // }; use core::{ ops::{ @@ -654,52 +662,6 @@ impl<T: KnownSize + ?Sized> Coherent<T> { // SAFETY: per safety requirement. unsafe { &mut *self.as_mut_ptr() } } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } impl<T: AsBytes + FromBytes> Coherent<T> { @@ -1133,84 +1095,153 @@ unsafe impl Send for CoherentHandle {} // plain `Copy` values. unsafe impl Sync for CoherentHandle {} -/// Reads a field of an item from an allocated region of structs. +/// View type for `Coherent`. /// -/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { field: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// let whole = kernel::dma_read!(alloc, [try: 2]); -/// let field = kernel::dma_read!(alloc, [panic: 1].field); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_read { - ($dma:expr, $($proj:tt)*) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - $crate::dma::Coherent::as_ptr(dma), $($proj)* - ); - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_read(dma, ptr) } - }}; +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_handle: DmaAddress, } -/// Writes to a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`Coherent`], `proj` is a -/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the -/// projected location. -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { member: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// kernel::dma_write!(alloc, [try: 2].member, 0xf); -/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf }); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_write { - (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* - ); - let val = $val; - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } - }}; - (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*]) - }; - ($dma:expr, $($rest:tt)*) => { - $crate::dma_write!(@parse [$dma] [] [$($rest)*]) - }; +impl<T: ?Sized> Copy for CoherentView<'_, T> {} +impl<T: ?Sized> Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns a DMA handle which may be given to the device as the DMA address base of the region. + #[inline] + pub fn dma_handle(self) -> DmaAddress { + self.dma_handle + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_handle = view.dma_handle + offset as DmaAddress; + CoherentView { + dma_handle, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl<T> IoCapable<T> for CoherentIoBackend +where + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_handle: self.dma_handle, + } + } } diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index fcc7678fd9e3..95f46bb75f9e 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,9 +4,18 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + use crate::{ bindings, - prelude::*, // + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // }; pub mod mem; @@ -31,128 +40,226 @@ pub type PhysAddr = bindings::phys_addr_t; /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type ResourceSize = bindings::resource_size_t; +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region<const SIZE: usize = 0> { + inner: [u8], +} + +impl<const SIZE: usize> Region<SIZE> { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE> + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl<const SIZE: usize> KnownSize for Region<SIZE> { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + /// Raw representation of an MMIO region. /// +/// `MmioRaw<T>` is equivalent to `T __iomem *` in C. +/// /// By itself, the existence of an instance of this structure does not provide any guarantees that /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` /// structure any guarantees are given. -pub struct MmioRaw<const SIZE: usize = 0> { - addr: usize, - maxsize: usize, +pub struct MmioRaw<T: ?Sized> { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, } -impl<const SIZE: usize> MmioRaw<SIZE> { - /// Returns a new `MmioRaw` instance on success, an error otherwise. - pub fn new(addr: usize, maxsize: usize) -> Result<Self> { - if maxsize < SIZE { - return Err(EINVAL); +impl<T: ?Sized> Copy for MmioRaw<T> {} +impl<T: ?Sized> Clone for MmioRaw<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Send for MmioRaw<T> {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Sync for MmioRaw<T> {} + +impl<T> MmioRaw<T> { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), } + } +} - Ok(Self { addr, maxsize }) +impl<const SIZE: usize> MmioRaw<Region<SIZE>> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result<Self> { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) } +} +impl<T: ?Sized + KnownSize> MmioRaw<T> { /// Returns the base address of the MMIO region. #[inline] pub fn addr(&self) -> usize { - self.addr + self.ptr.addr() } - /// Returns the maximum size of the MMIO region. + /// Returns the size of the MMIO region. #[inline] - pub fn maxsize(&self) -> usize { - self.maxsize + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) } } -/// IO-mapped memory region. -/// -/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the -/// mapping, performing an additional region request etc. -/// -/// # Invariant -/// -/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size -/// `maxsize`. -/// -/// # Examples -/// -/// ```no_run -/// use kernel::{ -/// bindings, -/// ffi::c_void, -/// io::{ -/// Io, -/// IoKnownSize, -/// Mmio, -/// MmioRaw, -/// PhysAddr, -/// }, -/// }; -/// use core::ops::Deref; -/// -/// // See also `pci::Bar` for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); -/// -/// impl<const SIZE: usize> IoMem<SIZE> { -/// /// # Safety -/// /// -/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs -/// /// virtual address space. -/// unsafe fn new(paddr: usize) -> Result<Self>{ -/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is -/// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; -/// if addr.is_null() { -/// return Err(ENOMEM); -/// } -/// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) -/// } -/// } -/// -/// impl<const SIZE: usize> Drop for IoMem<SIZE> { -/// fn drop(&mut self) { -/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; -/// } -/// } -/// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; -/// -/// fn deref(&self) -> &Self::Target { -/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } -/// } -/// } -/// -///# fn no_run() -> Result<(), Error> { -/// // SAFETY: Invalid usage for example purposes. -/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; -/// iomem.write32(0x42, 0x0); -/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); -/// assert!(iomem.try_write32(0x42, 0x4).is_err()); -/// # Ok(()) -/// # } -/// ``` -#[repr(transparent)] -pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>); - -/// Checks whether an access of type `U` at the given `offset` +/// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. #[inline] -const fn offset_valid<U>(offset: usize, size: usize) -> bool { - let type_size = core::mem::size_of::<U>(); - if let Some(end) = offset.checked_add(type_size) { - end <= size && offset % type_size == 0 +const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::<U>()) { + end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0) } else { false } } +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> <IO::Backend as IoBackend>::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<<IO::Backend as IoBackend>::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + /// Trait indicating that an I/O backend supports operations of a certain type and providing an /// implementation for these operations. /// @@ -161,20 +268,75 @@ const fn offset_valid<U>(offset: usize, size: usize) -> bool { /// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`, /// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit /// system might implement all four. -pub trait IoCapable<T> { - /// Performs an I/O read of type `T` at `address` and returns the result. +pub trait IoCapable<T>: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; + + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); +} + +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. /// /// # Safety /// - /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`. - unsafe fn io_read(&self, address: usize) -> T; + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); - /// Performs an I/O write of `value` at `address`. + /// Copy contents from `buffer` to `view`. /// /// # Safety /// - /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`. - unsafe fn io_write(&self, value: T, address: usize); + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; + + let mut buf = MaybeUninit::<T>::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::<T>()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } } /// Describes a given I/O location: its offset, width, and type to convert the raw value from and @@ -186,15 +348,16 @@ pub trait IoCapable<T> { /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] /// macro). /// -/// An `IoLoc<T>` carries three pieces of information: +/// An `IoLoc<Base, T>` carries the following pieces of information: /// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), /// - The width of the access (determined by [`IoLoc::IoType`]), /// - The type `T` in which the raw data is returned or provided. /// /// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type /// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). -pub trait IoLoc<T> { +pub trait IoLoc<Base: ?Sized, T> { /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). type IoType: Into<T> + From<T>; @@ -202,12 +365,12 @@ pub trait IoLoc<T> { fn offset(self) -> usize; } -/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of -/// [`Io::read`] and [`Io::write`]. +/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( - impl IoLoc<$ty> for usize { + impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize { type IoType = $ty; #[inline(always)] @@ -225,181 +388,414 @@ impl_usize_ioloc!(u8, u16, u32, u64); /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// This is an abstract representation to be implemented by arbitrary I/O -/// backends (e.g. MMIO, PCI config space, etc.). +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. +/// +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. /// -/// The [`Io`] trait provides: -/// - Base address and size information +/// This trait provides: /// - Helper methods for offset validation and address calculation /// - Fallible (runtime checked) accessors for different data widths /// -/// Which I/O methods are available depends on which [`IoCapable<T>`] traits -/// are implemented for the type. -/// -/// # Examples -/// -/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically -/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io { - /// Returns the base address of this mapping. - fn addr(&self) -> usize; +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } - /// Returns the maximum size of this mapping. - fn maxsize(&self) -> usize; + /// Returns the length of the slice in number of elements. + #[inline] + fn len<T>(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty<T>(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } - /// Returns the absolute I/O address for a given `offset`, - /// performing runtime bound checks. + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` #[inline] - fn io_addr<U>(&self, offset: usize) -> Result<usize> { - if !offset_valid::<U>(offset, self.maxsize()) { + fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::<U>() > KnownSize::size(ptr) { return Err(EINVAL); } - // Probably no need to check, since the safety requirements of `Self::new` guarantee that - // this can't overflow. - self.addr().checked_add(offset).ok_or(EINVAL) + if ptr.addr() % align_of::<U>() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } } /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, offset: usize) -> Result<u8> + fn try_read8(self, offset: usize) -> Result<u8> where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.try_read(offset) } /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, offset: usize) -> Result<u16> + fn try_read16(self, offset: usize) -> Result<u16> where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.try_read(offset) } /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, offset: usize) -> Result<u32> + fn try_read32(self, offset: usize) -> Result<u32> where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.try_read(offset) } /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, offset: usize) -> Result<u64> + fn try_read64(self, offset: usize) -> Result<u64> where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.try_read(offset) } /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, value: u8, offset: usize) -> Result + fn try_write8(self, value: u8, offset: usize) -> Result where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.try_write(offset, value) } /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, value: u16, offset: usize) -> Result + fn try_write16(self, value: u16, offset: usize) -> Result where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.try_write(offset, value) } /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, value: u32, offset: usize) -> Result + fn try_write32(self, value: u32, offset: usize) -> Result where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.try_write(offset, value) } /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, value: u64, offset: usize) -> Result + fn try_write64(self, value: u64, offset: usize) -> Result where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.try_write(offset, value) } /// Infallible 8-bit read with compile-time bounds check. #[inline(always)] - fn read8(&self, offset: usize) -> u8 + fn read8(self, offset: usize) -> u8 where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.read(offset) } /// Infallible 16-bit read with compile-time bounds check. #[inline(always)] - fn read16(&self, offset: usize) -> u16 + fn read16(self, offset: usize) -> u16 where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.read(offset) } /// Infallible 32-bit read with compile-time bounds check. #[inline(always)] - fn read32(&self, offset: usize) -> u32 + fn read32(self, offset: usize) -> u32 where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.read(offset) } /// Infallible 64-bit read with compile-time bounds check. #[inline(always)] - fn read64(&self, offset: usize) -> u64 + fn read64(self, offset: usize) -> u64 where - Self: IoKnownSize + IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.read(offset) } /// Infallible 8-bit write with compile-time bounds check. #[inline(always)] - fn write8(&self, value: u8, offset: usize) + fn write8(self, value: u8, offset: usize) where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.write(offset, value) } /// Infallible 16-bit write with compile-time bounds check. #[inline(always)] - fn write16(&self, value: u16, offset: usize) + fn write16(self, value: u16, offset: usize) where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.write(offset, value) } /// Infallible 32-bit write with compile-time bounds check. #[inline(always)] - fn write32(&self, value: u32, offset: usize) + fn write32(self, value: u32, offset: usize) where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.write(offset, value) } /// Infallible 64-bit write with compile-time bounds check. #[inline(always)] - fn write64(&self, value: u64, offset: usize) + fn write64(self, value: u64, offset: usize) where - Self: IoKnownSize + IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.write(offset, value) } @@ -414,9 +810,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio) -> Result { + /// fn do_reads(io: Mmio<'_, Region>) -> Result { /// // 32-bit read from address `0x10`. /// let v: u32 = io.try_read(0x10)?; /// @@ -427,15 +824,13 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn try_read<T, L>(&self, location: L) -> Result<T> + fn try_read<T, L>(self, location: L) -> Result<T> where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr::<L::IoType>(location.offset())?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }.into()) + let view = io_view::<Self, L::IoType>(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) } /// Generic fallible write with runtime bounds check. @@ -448,9 +843,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio) -> Result { + /// fn do_writes(io: Mmio<'_, Region>) -> Result { /// // 32-bit write of value `1` at address `0x10`. /// io.try_write(0x10, 1u32)?; /// @@ -461,17 +857,14 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn try_write<T, L>(&self, location: L, value: T) -> Result + fn try_write<T, L>(self, location: L, value: T) -> Result where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr::<L::IoType>(location.offset())?; + let view = io_view::<Self, L::IoType>(self, location.offset())?; let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } - + Self::Backend::io_write(view, io_value); Ok(()) } @@ -486,6 +879,7 @@ pub trait Io { /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -501,17 +895,17 @@ pub trait Io { /// } /// } /// - /// fn do_write_reg(io: &Mmio) -> Result { + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { /// /// io.try_write_reg(VERSION::new(1, 0)) /// } /// ``` #[inline(always)] - fn try_write_reg<T, L, V>(&self, value: V) -> Result + fn try_write_reg<T, L, V>(self, value: V) -> Result where - L: IoLoc<T>, - V: LocatedRegister<Location = L, Value = T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, { let (location, value) = value.into_io_op(); @@ -531,29 +925,27 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) -> Result { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { /// io.try_update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn try_update<T, L, F>(&self, location: L, f: F) -> Result + fn try_update<T, L, F>(self, location: L, f: F) -> Result where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, F: FnOnce(T) -> T, { - let address = self.io_addr::<L::IoType>(location.offset())?; + let view = io_view::<Self, L::IoType>(self, location.offset())?; - // SAFETY: `address` has been validated by `io_addr`. - let value: T = unsafe { self.io_read(address) }.into(); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); Ok(()) } @@ -568,9 +960,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio<0x1000>) { + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit read from address `0x10`. /// let v: u32 = io.read(0x10); /// @@ -579,15 +972,13 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn read<T, L>(&self, location: L) -> T + fn read<T, L>(self, location: L) -> T where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) }.into() + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + Self::Backend::io_read(view).into() } /// Generic infallible write with compile-time bounds check. @@ -600,9 +991,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio<0x1000>) { + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit write of value `1` at address `0x10`. /// io.write(0x10, 1u32); /// @@ -611,16 +1003,14 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn write<T, L>(&self, location: L, value: T) + fn write<T, L>(self, location: L, value: T) where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } /// Generic infallible write of a fully-located register value. @@ -634,6 +1024,7 @@ pub trait Io { /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -649,16 +1040,16 @@ pub trait Io { /// } /// } /// - /// fn do_write_reg(io: &Mmio<0x1000>) { + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { /// io.write_reg(VERSION::new(1, 0)); /// } /// ``` #[inline(always)] - fn write_reg<T, L, V>(&self, value: V) + fn write_reg<T, L, V>(self, value: V) where - L: IoLoc<T>, - V: LocatedRegister<Location = L, Value = T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, { let (location, value) = value.into_io_op(); @@ -678,143 +1069,208 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { /// io.update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn update<T, L, F>(&self, location: L, f: F) + fn update<T, L, F>(self, location: L, f: F) where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType> + Sized, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, F: FnOnce(T) -> T, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - let value: T = unsafe { self.io_read(address) }.into(); + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } } -/// Trait for types with a known size at compile time. +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + +/// A view of memory-mapped I/O region. /// -/// This trait is implemented by I/O backends that have a compile-time known size, -/// enabling the use of infallible I/O accessors with compile-time bounds checking. +/// # Invariant /// -/// Types implementing this trait can use the infallible methods in [`Io`] trait -/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound. -pub trait IoKnownSize: Io { - /// Minimum usable size of this region. - const MIN_SIZE: usize; +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert<U>(&self, offset: usize) -> usize { - build_assert!(offset_valid::<U>(offset, Self::MIN_SIZE)); +impl<T: ?Sized> Copy for Mmio<'_, T> {} +impl<T: ?Sized> Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {} - self.addr() + offset +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } } } -/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { - ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { - $(#[$attr])* - impl<const SIZE: usize> IoCapable<$ty> for $mmio<SIZE> { - unsafe fn io_read(&self, address: usize) -> $ty { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$read_fn(address as *const c_void) } + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } } - unsafe fn io_write(&self, value: $ty, address: usize) { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$write_fn(value, address as *mut c_void) } + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } } } }; } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(Mmio, u8, readb, writeb); -impl_mmio_io_capable!(Mmio, u16, readw, writew); -impl_mmio_io_capable!(Mmio, u32, readl, writel); +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - Mmio, - #[cfg(CONFIG_64BIT)] - u64, - readq, - writeq -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); -impl<const SIZE: usize> Io for Mmio<SIZE> { - /// Returns the base address of this mapping. +impl IoCopyable for MmioBackend { #[inline] - fn addr(&self) -> usize { - self.0.addr() + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } } - /// Returns the maximum size of this mapping. #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() - } -} - -impl<const SIZE: usize> IoKnownSize for Mmio<SIZE> { - const MIN_SIZE: usize = SIZE; -} - -impl<const SIZE: usize> Mmio<SIZE> { - /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. - /// - /// # Safety - /// - /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size - /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw<SIZE>) -> &Self { - // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. - unsafe { &*core::ptr::from_ref(raw).cast() } + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } } } -/// [`Mmio`] wrapper using relaxed accessors. +/// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of /// the regular ones. /// /// See [`Mmio::relaxed`] for a usage example. -#[repr(transparent)] -pub struct RelaxedMmio<const SIZE: usize = 0>(Mmio<SIZE>); +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); + +impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {} +impl<T: ?Sized> Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; -impl<const SIZE: usize> Io for RelaxedMmio<SIZE> { #[inline] - fn addr(&self) -> usize { - self.0.addr() + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) } #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) } } -impl<const SIZE: usize> IoKnownSize for RelaxedMmio<SIZE> { - const MIN_SIZE: usize = SIZE; +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } } -impl<const SIZE: usize> Mmio<SIZE> { - /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. /// /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses /// and can be used when such ordering is not required. @@ -825,31 +1281,457 @@ impl<const SIZE: usize> Mmio<SIZE> { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// RelaxedMmio, /// }; /// - /// fn do_io(io: &Mmio<0x100>) { + /// fn do_io(io: Mmio<'_, Region<0x100>>) { /// // The access is performed using `readl_relaxed` instead of `readl`. /// let v = io.relaxed().read32(0x10); /// } /// /// ``` - pub fn relaxed(&self) -> &RelaxedMmio<SIZE> { - // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio<SIZE>` and - // `RelaxedMmio<SIZE>` have identical layout. - unsafe { core::mem::transmute(self) } + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) } } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - RelaxedMmio, - #[cfg(CONFIG_64BIT)] - u64, - readq_relaxed, - writeq_relaxed -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl<T: ?Sized> Copy for SysMem<'_, T> {} +impl<T: ?Sized> Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl<T: ?Sized> Copy for IoSysMap<'_, T> {} +impl<T: ?Sized> Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl<T> IoCapable<T> for IoSysMapBackend +where + MmioBackend: IoCapable<T>, + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper<T>(pub T); + +impl<'a, T> ProjectHelper<T> +where + T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view<U: ?Sized + KnownSize>( + self, + ptr: *mut U, + ) -> <T::Backend as IoBackend>::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 931f2fa3bb10..32a919099dcd 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -2,8 +2,6 @@ //! Generic memory-mapped IO. -use core::ops::Deref; - use crate::{ device::{ Bound, @@ -16,7 +14,9 @@ use crate::{ Region, Resource, // }, + IoBase, Mmio, + MmioBackend, MmioRaw, // }, prelude::*, @@ -225,11 +225,13 @@ impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { } } -impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { - &self.iomem + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() } } @@ -244,7 +246,7 @@ impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> { /// start of the I/O memory mapped region. pub struct IoMem<'a, const SIZE: usize = 0> { dev: &'a Device<Bound>, - io: MmioRaw<SIZE>, + io: MmioRaw<super::Region<SIZE>>, } impl<const SIZE: usize> ForLt for IoMem<'static, SIZE> { @@ -292,8 +294,7 @@ impl<'a, const SIZE: usize> IoMem<'a, SIZE> { return Err(ENOMEM); } - let io = MmioRaw::new(addr as usize, size)?; - + let io = MmioRaw::new_region(addr as usize, size)?; Ok(IoMem { dev, io }) } @@ -316,11 +317,13 @@ impl<const SIZE: usize> Drop for IoMem<'_, SIZE> { } } -impl<const SIZE: usize> Deref for IoMem<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 75d1b3e8596c..d75f2fcf46f2 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -48,13 +48,14 @@ use crate::{ /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -135,13 +136,14 @@ where /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 66449377a0b7..6cb07fc92cc3 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -58,7 +58,7 @@ //! }, //! num::Bounded, //! }; -//! # use kernel::io::Mmio; +//! # use kernel::io::{Mmio, Region}; //! # register! { //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; @@ -66,7 +66,7 @@ //! # 3:0 minor_revision; //! # } //! # } -//! # fn test(io: &Mmio<0x1000>) { +//! # fn test(io: Mmio<'_, Region<0x1000>>) { //! # fn obtain_vendor_id() -> u8 { 0xff } //! //! // Read from the register's defined offset (0x100). @@ -113,6 +113,8 @@ use crate::{ io::IoLoc, // }; +use super::Region; + /// Trait implemented by all registers. pub trait Register: Sized { /// Backing primitive type of the register. @@ -129,7 +131,7 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl<T> IoLoc<T> for () +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for () where T: FixedRegister, { @@ -143,7 +145,7 @@ where /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl<T> IoLoc<T> for T +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T where T: FixedRegister, { @@ -168,7 +170,7 @@ impl<T: FixedRegister> FixedRegisterLoc<T> { } } -impl<T> IoLoc<T> for FixedRegisterLoc<T> +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T> where T: FixedRegister, { @@ -239,7 +241,7 @@ where } } -impl<T, B> IoLoc<T> for RelativeRegisterLoc<T, B> +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B> where T: RelativeRegister, B: RegisterBase<T::BaseFamily> + ?Sized, @@ -283,7 +285,7 @@ impl<T: RegisterArray> RegisterArrayLoc<T> { } } -impl<T> IoLoc<T> for RegisterArrayLoc<T> +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T> where T: RegisterArray, { @@ -370,7 +372,7 @@ where } } -impl<T, B> IoLoc<T> for RelativeRegisterArrayLoc<T, B> +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B> where T: RelativeRegisterArray, B: RegisterBase<T::BaseFamily> + ?Sized, @@ -387,18 +389,18 @@ where /// which to write it. /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). -pub trait LocatedRegister { +pub trait LocatedRegister<Base: ?Sized> { /// Register value to write. type Value: Register; /// Full location information at which to write the value. - type Location: IoLoc<Self::Value>; + type Location: IoLoc<Base, Self::Value>; /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write /// operation. fn into_io_op(self) -> (Self::Location, Self::Value); } -impl<T> LocatedRegister for T +impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T where T: FixedRegister, { @@ -444,7 +446,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// register! { /// FIXED_REG(u32) @ 0x100 { @@ -453,7 +455,7 @@ where /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// let val = io.read(FIXED_REG); /// /// // Write from an already-existing value. @@ -557,7 +559,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -582,7 +584,7 @@ where /// } /// } /// -/// # fn test(io: Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// // Read the status of `Cpu0`. /// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>()); /// @@ -599,7 +601,7 @@ where /// } /// } /// -/// # fn test2(io: Mmio<0x1000>) { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { /// // Start the aliased `CPU0`, leaving its other fields untouched. /// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true)); /// # } @@ -636,7 +638,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -649,7 +651,7 @@ where /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) +/// # fn test(io: Mmio<'_, Region<0x1000>>) /// # -> Result<(), Error>{ /// // Read scratch register 0, i.e. I/O address `0x80`. /// let scratch_0 = io.read(SCRATCH::at(0)).value(); @@ -722,7 +724,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -750,7 +752,7 @@ where /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// // Read scratch register 0 of CPU0. /// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0)); /// @@ -792,7 +794,7 @@ where /// } /// } /// -/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status(); /// # Ok(()) /// # } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..68f4d9a3425d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,9 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// // Stable since Rust 1.89.0. #![feature(generic_arg_infer)] // diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index f783b9d9fa26..ee25ae56bb92 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -43,7 +43,6 @@ pub use self::id::{ pub use self::io::{ Bar, ConfigSpace, - ConfigSpaceKind, ConfigSpaceSize, DevresBar, Extended, diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 6ebfbf368cd3..953e16735c6e 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -8,22 +8,21 @@ use crate::{ device, devres::DevresLt, io::{ - Io, + IoBackend, + IoBase, IoCapable, - IoKnownSize, Mmio, - MmioRaw, // + MmioBackend, + MmioRaw, + Region, // }, prelude::*, + ptr::KnownSize, types::{ CovariantForLt, ForLt, // }, // }; -use core::{ - marker::PhantomData, - ops::Deref, // -}; /// Represents the size of a PCI configuration space. /// @@ -50,68 +49,95 @@ impl ConfigSpaceSize { } } -/// Marker type for normal (256-byte) PCI configuration space. -pub struct Normal; +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; -/// Marker type for extended (4096-byte) PCIe configuration space. -pub struct Extended; +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; -/// Trait for PCI configuration space size markers. +/// A view of PCI configuration space of a device. +/// +/// Provides typed read and write accessors for configuration registers +/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. +/// +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). /// -/// This trait is implemented by [`Normal`] and [`Extended`] to provide -/// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind { - /// The size of this configuration space in bytes. - const SIZE: usize; +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { + pub(crate) pdev: &'a Device<device::Bound>, + ptr: *mut T, } -impl ConfigSpaceKind for Normal { - const SIZE: usize = 256; +impl<T: ?Sized> Copy for ConfigSpace<'_, T> {} +impl<T: ?Sized> Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } } -impl ConfigSpaceKind for Extended { - const SIZE: usize = 4096; -} +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for ConfigSpace<'_, T> {} -/// The PCI configuration space of a device. -/// -/// Provides typed read and write accessors for configuration registers -/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. -/// -/// The generic parameter `S` indicates the maximum size of the configuration space. -/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for -/// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { - pub(crate) pdev: &'a Device<device::Bound>, - _marker: PhantomData<S>, +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } } /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { - unsafe fn io_read(&self, address: usize) -> $ty { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + let mut val: $ty = 0; // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) }; - + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; val } - unsafe fn io_write(&self, value: $ty, address: usize) { + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) }; + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; } } }; @@ -122,24 +148,16 @@ impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { - /// Returns the base address of the I/O region. It is always 0 for configuration space. - #[inline] - fn addr(&self) -> usize { - 0 - } +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; - /// Returns the maximum size of the configuration space. #[inline] - fn maxsize(&self) -> usize { - self.pdev.cfg_size().into_raw() + fn as_view(self) -> ConfigSpace<'a, T> { + self } } -impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { - const MIN_SIZE: usize = S::SIZE; -} - /// A PCI BAR to perform I/O-Operations on. /// /// I/O backend assumes that the device is little-endian and will automatically @@ -151,7 +169,7 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { /// memory mapped PCI BAR and its size. pub struct Bar<'a, const SIZE: usize = 0> { pdev: &'a Device<device::Bound>, - io: MmioRaw<SIZE>, + io: MmioRaw<crate::io::Region<SIZE>>, num: i32, } @@ -204,7 +222,7 @@ impl<'a, const SIZE: usize> Bar<'a, SIZE> { return Err(ENOMEM); } - let io = match MmioRaw::new(ioptr, len as usize) { + let io = match MmioRaw::new_region(ioptr, len as usize) { Ok(io) => io, Err(err) => { // SAFETY: @@ -264,12 +282,14 @@ impl<const SIZE: usize> Drop for Bar<'_, SIZE> { } } -impl<const SIZE: usize> Deref for Bar<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } @@ -304,23 +324,25 @@ impl Device<device::Bound> { } } - /// Return an initialized normal (256-byte) config space object. + /// Return a view of the normal (256-byte) config space. pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), } } - /// Return an initialized extended (4096-byte) config space object. + /// Return a view of the extended (4096-byte) config space. pub fn config_space_extended<'a>(&'a self) -> Result<ConfigSpace<'a, Extended>> { if self.cfg_size() != ConfigSpaceSize::Extended { return Err(EINVAL); } + // INVARIANT: null is aligned and we just checked the `cfg_size`. Ok(ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), }) } } diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 3f3e529e9f58..82acb531b17b 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -235,11 +235,20 @@ impl_alignable_uint!(u8, u16, u32, u64, usize); /// /// This is a generalization of [`size_of`] that works for dynamically sized types. pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. fn size(p: *const Self) -> usize; } impl<T> KnownSize for T { + const MIN_SIZE: usize = size_of::<T>(); + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(_: *const Self) -> usize { size_of::<T>() @@ -247,6 +256,9 @@ impl<T> KnownSize for T { } impl<T> KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(p: *const Self) -> usize { p.len() * size_of::<T>() diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 5046b4628d0e..b629acc6d915 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -12,6 +12,11 @@ use kernel::{ Device, DmaMask, // }, + io::{ + io_project, + io_read, + Io, // + }, page, pci, prelude::*, scatterlist::{Owned, SGTable}, @@ -34,6 +39,7 @@ const TEST_VALUES: [(u32, u32); 5] = [ (0xcd, 0xef), ]; +#[derive(FromBytes, IntoBytes)] struct MyStruct { h: u32, b: u32, @@ -77,7 +83,7 @@ impl pci::Driver for DmaSampleDriver { Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { - kernel::dma_write!(ca, [try: i], MyStruct::new(value.0, value.1)); + io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0, value.1)); } let size = 4 * page::PAGE_SIZE; @@ -97,8 +103,8 @@ impl pci::Driver for DmaSampleDriver { impl DmaSampleDriver { fn check_dma(&self) { for (i, value) in TEST_VALUES.into_iter().enumerate() { - let val0 = kernel::dma_read!(self.ca, [panic: i].h); - let val1 = kernel::dma_read!(self.ca, [panic: i].b); + let val0 = io_read!(self.ca, [panic: i].h); + let val1 = io_read!(self.ca, [panic: i].b); assert_eq!(val0, value.0); assert_eq!(val1, value.1); |
