summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDanilo Krummrich <dakr@kernel.org>2026-07-13 23:50:39 +0200
committerDanilo Krummrich <dakr@kernel.org>2026-07-13 23:50:39 +0200
commitb07fc8d60bd30caaba4d293929459780166da194 (patch)
tree4f47c377f5b9c8078f02ef883322719f4109b167
parenta56a92be1fa9c6e747fc754c90734ed90cd36670 (diff)
parent11a4784f902ebf3e674dbaf07dbec9a37aabb5e4 (diff)
downloadlinux-next-b07fc8d60bd30caaba4d293929459780166da194.tar.gz
linux-next-b07fc8d60bd30caaba4d293929459780166da194.zip
Merge patch series "rust: I/O type generalization and projection"
Gary Guo <gary@garyguo.net> says: This series presents a major rework of I/O types, as a summary: - Make I/O regions typed. The existing untyped region still exists with a dynamically sized `Region` type. - Create I/O view types to represent subregion of a full I/O region mapped. A projection macro is added to allow safely create such subviews. - Split I/O traits, make I/O views play a central role, avoid duplicate monomorphization and less `unsafe` code. - Add a `SysMem` backend, and make `Coherent` implement `Io`. - Add copying methods (memcpy_{from,to}io and friends). This series generalize `Mmio` type from just an untyped region to typed representations (so `MmioRaw<T>` is `__iomem *T`). This allows us to remove the `IoKnownSize` trait; the information is sourced from just the pointer from the `KnownSize` trait instead. Building on top of that, `Mmio` and `ConfigSpace` have been converted to typed views of I/O regions rather than just a big chunk of untyped I/O memory. These changes made it possible to implement `Io` trait for `Coherent<T>`. Shared system memory, `SysMem` is also added to the series, given it similarity in implementation compared to `Coherent`. In fact, the series use `SysMem` to implement `Coherent`'s I/O methods. Built on these generalization, this series add `io_project!()`. `io_project!()` performs a safe way to project a bigger view to a small subviews, and some Nova code has been converted in this series to demonstrate cleanups possible with this addition. New `io_read!()`, `io_write!()` has been added that supersedes `dma_read!()`, `dma_write!()` macro. Although, they work for primitives only (to be exact, types that the backend is `IoCapable` of). One feature that was lost from the old `dma_read!()` and `dma_write!()` series was the ability to read/write a large structs. However, the semantics was unclear to begin with, as there was no guarantee about their atomicity even for structs that were small enough to fit in u32. Suggested-by: Danilo Krummrich <dakr@kernel.org> Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078 Link: https://patch.msgid.link/20260706-io_projection-v6-0-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich <dakr@kernel.org>
-rw-r--r--drivers/gpu/nova-core/gsp.rs53
-rw-r--r--drivers/gpu/nova-core/gsp/cmdq.rs66
-rw-r--r--drivers/gpu/nova-core/gsp/fw.rs82
-rw-r--r--drivers/pwm/pwm_th1520.rs7
-rw-r--r--rust/helpers/io.c13
-rw-r--r--rust/kernel/devres.rs24
-rw-r--r--rust/kernel/dma.rs281
-rw-r--r--rust/kernel/io.rs1502
-rw-r--r--rust/kernel/io/mem.rs29
-rw-r--r--rust/kernel/io/poll.rs6
-rw-r--r--rust/kernel/io/register.rs44
-rw-r--r--rust/kernel/lib.rs3
-rw-r--r--rust/kernel/pci.rs1
-rw-r--r--rust/kernel/pci/io.rs166
-rw-r--r--rust/kernel/ptr.rs12
-rw-r--r--samples/rust/rust_dma.rs12
16 files changed, 1626 insertions, 675 deletions
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs
index 69175ca3315c..cfa7553cd820 100644
--- a/drivers/gpu/nova-core/gsp.rs
+++ b/drivers/gpu/nova-core/gsp.rs
@@ -9,14 +9,16 @@ use kernel::{
dma::{
Coherent,
CoherentBox,
+ CoherentView,
DmaAddress, //
},
+ io::{
+ io_project,
+ io_write,
+ Io, //
+ },
pci,
- prelude::*,
- transmute::{
- AsBytes,
- FromBytes, //
- }, //
+ prelude::*, //
};
pub(crate) mod cmdq;
@@ -48,21 +50,21 @@ const LOG_BUFFER_SIZE: usize = RM_LOG_BUFFER_NUM_PAGES * GSP_PAGE_SIZE;
/// Array of page table entries, as understood by the GSP bootloader.
#[repr(C)]
+#[derive(FromBytes, IntoBytes)]
struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]);
-/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one.
-unsafe impl<const NUM_ENTRIES: usize> FromBytes for PteArray<NUM_ENTRIES> {}
-
-/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one.
-unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}
-
impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
- /// Returns the page table entry for `index`, for a mapping starting at `start`.
- // TODO: Replace with `IoView` projection once available.
- fn entry(start: DmaAddress, index: usize) -> Result<u64> {
- start
- .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT)
- .ok_or(EOVERFLOW)
+ /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`.
+ fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> {
+ for i in 0..NUM_PAGES {
+ io_write!(view, .0[build: i],
+ start
+ .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT)
+ .ok_or(EOVERFLOW)?
+ );
+ }
+
+ Ok(())
}
}
@@ -89,17 +91,12 @@ impl LogBuffer {
let start_addr = obj.0.dma_handle();
- // SAFETY: `obj` has just been created and we are its sole user.
- let pte_region = unsafe {
- &mut obj.0.as_mut()[size_of::<u64>()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()]
- };
-
- // Write values one by one to avoid an on-stack instance of `PteArray`.
- for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() {
- let pte_value = PteArray::<0>::entry(start_addr, i)?;
-
- chunk.copy_from_slice(&pte_value.to_ne_bytes());
- }
+ let pte_view = io_project!(
+ obj.0,
+ [build: size_of::<u64>()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()]
+ )
+ .try_cast::<PteArray<RM_LOG_BUFFER_NUM_PAGES>>()?;
+ PteArray::init(pte_view, start_addr)?;
Ok(obj)
}
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 070de0731e95..c34b48961496 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -2,16 +2,23 @@
mod continuation;
-use core::mem;
+use core::{
+ mem,
+ sync::atomic::{
+ fence,
+ Ordering, //
+ },
+};
use kernel::{
device,
dma::{
Coherent,
+ CoherentBox,
DmaAddress, //
},
- dma_write,
io::{
+ io_project,
poll::read_poll_timeout,
Io, //
},
@@ -171,20 +178,18 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE);
#[repr(C)]
// There is no struct defined for this in the open-gpu-kernel-source headers.
// Instead it is defined by code in `GspMsgQueuesInit()`.
-// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
-pub(super) struct Msgq {
+struct Msgq {
/// Header for sending messages, including the write pointer.
- pub(super) tx: MsgqTxHeader,
+ tx: MsgqTxHeader,
/// Header for receiving messages, including the read pointer.
- pub(super) rx: MsgqRxHeader,
+ rx: MsgqRxHeader,
/// The message queue proper.
msgq: MsgqData,
}
/// Structure shared between the driver and the GSP and containing the command and message queues.
#[repr(C)]
-// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
-pub(super) struct GspMem {
+struct GspMem {
/// Self-mapping page table entries.
ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
/// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
@@ -192,13 +197,13 @@ pub(super) struct GspMem {
/// index into the GSP queue.
///
/// This member is read-only for the GSP.
- pub(super) cpuq: Msgq,
+ cpuq: Msgq,
/// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
/// write and read pointers that the GSP updates. This means that the read pointer here is an
/// index into the CPU queue.
///
/// This member is read-only for the driver.
- pub(super) gspq: Msgq,
+ gspq: Msgq,
}
impl GspMem {
@@ -232,20 +237,12 @@ impl DmaGspMem {
const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();
- let gsp_mem = Coherent::<GspMem>::zeroed(dev, GFP_KERNEL)?;
-
- let start = gsp_mem.dma_handle();
- // Write values one by one to avoid an on-stack instance of `PteArray`.
- for i in 0..GspMem::PTE_ARRAY_SIZE {
- dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?);
- }
+ let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?;
+ gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES);
+ gsp_mem.cpuq.rx = MsgqRxHeader::new();
- dma_write!(
- gsp_mem,
- .cpuq.tx,
- MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES)
- );
- dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new());
+ let gsp_mem: Coherent<_> = gsp_mem.into();
+ PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?;
Ok(Self(gsp_mem))
}
@@ -406,7 +403,7 @@ impl DmaGspMem {
//
// - The returned value is within `0..MSGQ_NUM_PAGES`.
fn gsp_write_ptr(&self) -> u32 {
- super::fw::gsp_mem::gsp_write_ptr(&self.0)
+ MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES
}
// Returns the index of the memory page the GSP will read the next command from.
@@ -415,7 +412,7 @@ impl DmaGspMem {
//
// - The returned value is within `0..MSGQ_NUM_PAGES`.
fn gsp_read_ptr(&self) -> u32 {
- super::fw::gsp_mem::gsp_read_ptr(&self.0)
+ MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES
}
// Returns the index of the memory page the CPU can read the next message from.
@@ -424,12 +421,18 @@ impl DmaGspMem {
//
// - The returned value is within `0..MSGQ_NUM_PAGES`.
fn cpu_read_ptr(&self) -> u32 {
- super::fw::gsp_mem::cpu_read_ptr(&self.0)
+ MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES
}
// Informs the GSP that it can send `elem_count` new pages into the message queue.
fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
- super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count)
+ let rx = io_project!(self.0, .cpuq.rx);
+ let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
+
+ // Ensure read pointer is properly ordered.
+ fence(Ordering::SeqCst);
+
+ MsgqRxHeader::set_read_ptr(rx, rptr)
}
// Returns the index of the memory page the CPU can write the next command to.
@@ -438,12 +441,17 @@ impl DmaGspMem {
//
// - The returned value is within `0..MSGQ_NUM_PAGES`.
fn cpu_write_ptr(&self) -> u32 {
- super::fw::gsp_mem::cpu_write_ptr(&self.0)
+ MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES
}
// Informs the GSP that it can process `elem_count` new pages from the command queue.
fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
- super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count)
+ let tx = io_project!(self.0, .cpuq.tx);
+ let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
+ MsgqTxHeader::set_write_ptr(tx, wptr);
+
+ // Ensure all command data is visible before triggering the GSP read.
+ fence(Ordering::SeqCst);
}
}
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 4db0cfa4dc4d..b0e7de328eaf 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -10,7 +10,14 @@ use r570_144 as bindings;
use core::ops::Range;
use kernel::{
- dma::Coherent,
+ dma::{
+ Coherent,
+ CoherentView, //
+ },
+ io::{
+ io_read,
+ io_write, //
+ },
prelude::*,
ptr::{
Alignable,
@@ -44,59 +51,6 @@ use crate::{
},
};
-// TODO: Replace with `IoView` projections once available.
-pub(super) mod gsp_mem {
- use core::sync::atomic::{
- fence,
- Ordering, //
- };
-
- use kernel::{
- dma::Coherent,
- dma_read,
- dma_write, //
- };
-
- use crate::gsp::cmdq::{
- GspMem,
- MSGQ_NUM_PAGES, //
- };
-
- pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 {
- dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES
- }
-
- pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 {
- dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES
- }
-
- pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 {
- dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES
- }
-
- pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) {
- let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
-
- // Ensure read pointer is properly ordered.
- fence(Ordering::SeqCst);
-
- dma_write!(qs, .cpuq.rx.0.readPtr, rptr);
- }
-
- pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 {
- dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES
- }
-
- pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) {
- let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
-
- dma_write!(qs, .cpuq.tx.0.writePtr, wptr);
-
- // Ensure all command data is visible before triggering the GSP read.
- fence(Ordering::SeqCst);
- }
-}
-
/// Maximum size of a single GSP message queue element in bytes.
pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize =
num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX);
@@ -720,6 +674,16 @@ impl MsgqTxHeader {
entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),
})
}
+
+ /// Returns the value of the write pointer for this queue.
+ pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 {
+ io_read!(this, .0.writePtr)
+ }
+
+ /// Sets the value of the write pointer for this queue.
+ pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) {
+ io_write!(this, .0.writePtr, val)
+ }
}
// SAFETY: Padding is explicit and does not contain uninitialized data.
@@ -735,6 +699,16 @@ impl MsgqRxHeader {
pub(crate) fn new() -> Self {
Self(Default::default())
}
+
+ /// Returns the value of the read pointer for this queue.
+ pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 {
+ io_read!(this, .0.readPtr)
+ }
+
+ /// Sets the value of the read pointer for this queue.
+ pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) {
+ io_write!(this, .0.readPtr, val)
+ }
}
// SAFETY: Padding is explicit and does not contain uninitialized data.
diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs
index 7102c60fd14b..9372cd3e6a6d 100644
--- a/drivers/pwm/pwm_th1520.rs
+++ b/drivers/pwm/pwm_th1520.rs
@@ -20,7 +20,6 @@
//! this method is not used in this driver.
//!
-use core::ops::Deref;
use kernel::{
clk::Clk,
device::{Bound, Core, Device},
@@ -212,8 +211,7 @@ impl pwm::PwmOps for Th1520PwmDriverData {
) -> Result<Self::WfHw> {
let data = chip.drvdata();
let hwpwm = pwm.hwpwm();
- let iomem_accessor = data.iomem.access(parent_dev)?;
- let iomap = iomem_accessor.deref();
+ let iomap = data.iomem.access(parent_dev)?;
let ctrl = iomap.try_read32(th1520_pwm_ctrl(hwpwm))?;
let period_cycles = iomap.try_read32(th1520_pwm_per(hwpwm))?;
@@ -247,8 +245,7 @@ impl pwm::PwmOps for Th1520PwmDriverData {
) -> Result {
let data = chip.drvdata();
let hwpwm = pwm.hwpwm();
- let iomem_accessor = data.iomem.access(parent_dev)?;
- let iomap = iomem_accessor.deref();
+ let iomap = data.iomem.access(parent_dev)?;
let duty_cycles = iomap.try_read32(th1520_pwm_fp(hwpwm))?;
let was_enabled = duty_cycles != 0;
diff --git a/rust/helpers/io.c b/rust/helpers/io.c
index 397810864a24..7ed9a4f77f1b 100644
--- a/rust/helpers/io.c
+++ b/rust/helpers/io.c
@@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr)
iounmap(addr);
}
+__rust_helper void rust_helper_memcpy_fromio(void *dst,
+ const volatile void __iomem *src,
+ size_t count)
+{
+ memcpy_fromio(dst, src, count);
+}
+
+__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst,
+ const void *src, size_t count)
+{
+ memcpy_toio(dst, src, count);
+}
+
__rust_helper u8 rust_helper_readb(const void __iomem *addr)
{
return readb(addr);
diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs
index b7c075a39ba4..20f94030f977 100644
--- a/rust/kernel/devres.rs
+++ b/rust/kernel/devres.rs
@@ -70,17 +70,19 @@ struct Inner<T> {
/// devres::Devres,
/// io::{
/// Io,
-/// IoKnownSize,
+/// IoBase,
/// Mmio,
/// MmioRaw,
-/// PhysAddr, //
+/// MmioBackend,
+/// PhysAddr,
+/// Region, //
/// },
/// prelude::*,
/// };
/// use core::ops::Deref;
///
/// // See also [`pci::Bar`] for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
+/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>);
///
/// impl<const SIZE: usize> IoMem<SIZE> {
/// /// # Safety
@@ -95,7 +97,7 @@ struct Inner<T> {
/// return Err(ENOMEM);
/// }
///
-/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
+/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?))
/// }
/// }
///
@@ -106,12 +108,13 @@ struct Inner<T> {
/// }
/// }
///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-/// type Target = Mmio<SIZE>;
+/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> {
+/// type Backend = MmioBackend;
+/// type Target = Region<SIZE>;
///
-/// fn deref(&self) -> &Self::Target {
+/// fn as_view(self) -> Mmio<'a, Region<SIZE>> {
/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-/// unsafe { Mmio::from_raw(&self.0) }
+/// unsafe { Mmio::from_raw(self.0) }
/// }
/// }
/// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> {
@@ -299,10 +302,7 @@ impl<T: Send + 'static> Devres<T> {
/// use kernel::{
/// device::Core,
/// devres::Devres,
- /// io::{
- /// Io,
- /// IoKnownSize, //
- /// },
+ /// io::Io,
/// pci, //
/// };
///
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 200def84fb69..e275f2562a5b 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -14,14 +14,22 @@ use crate::{
},
error::to_result,
fs::file,
+ io::{
+ IoBackend,
+ IoBase,
+ IoCapable,
+ IoCopyable,
+ SysMem,
+ SysMemBackend, //
+ },
prelude::*,
ptr::KnownSize,
sync::aref::ARef,
transmute::{
AsBytes,
FromBytes, //
- }, //
- uaccess::UserSliceWriter,
+ },
+ uaccess::UserSliceWriter, //
};
use core::{
ops::{
@@ -654,52 +662,6 @@ impl<T: KnownSize + ?Sized> Coherent<T> {
// SAFETY: per safety requirement.
unsafe { &mut *self.as_mut_ptr() }
}
-
- /// Reads the value of `field` and ensures that its type is [`FromBytes`].
- ///
- /// # Safety
- ///
- /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
- /// validated beforehand.
- ///
- /// Public but hidden since it should only be used from [`dma_read`] macro.
- #[doc(hidden)]
- pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
- // SAFETY:
- // - By the safety requirements field is valid.
- // - Using read_volatile() here is not sound as per the usual rules, the usage here is
- // a special exception with the following notes in place. When dealing with a potential
- // race from a hardware or code outside kernel (e.g. user-space program), we need that
- // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
- // rationale behind is that it should generate the same code as READ_ONCE() which the
- // kernel already relies on to avoid UB on data races. Note that the usage of
- // read_volatile() is limited to this particular case, it cannot be used to prevent
- // the UB caused by racing between two kernel functions nor do they provide atomicity.
- unsafe { field.read_volatile() }
- }
-
- /// Writes a value to `field` and ensures that its type is [`AsBytes`].
- ///
- /// # Safety
- ///
- /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
- /// validated beforehand.
- ///
- /// Public but hidden since it should only be used from [`dma_write`] macro.
- #[doc(hidden)]
- pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
- // SAFETY:
- // - By the safety requirements field is valid.
- // - Using write_volatile() here is not sound as per the usual rules, the usage here is
- // a special exception with the following notes in place. When dealing with a potential
- // race from a hardware or code outside kernel (e.g. user-space program), we need that
- // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
- // rationale behind is that it should generate the same code as WRITE_ONCE() which the
- // kernel already relies on to avoid UB on data races. Note that the usage of
- // write_volatile() is limited to this particular case, it cannot be used to prevent
- // the UB caused by racing between two kernel functions nor do they provide atomicity.
- unsafe { field.write_volatile(val) }
- }
}
impl<T: AsBytes + FromBytes> Coherent<T> {
@@ -1133,84 +1095,153 @@ unsafe impl Send for CoherentHandle {}
// plain `Copy` values.
unsafe impl Sync for CoherentHandle {}
-/// Reads a field of an item from an allocated region of structs.
+/// View type for `Coherent`.
///
-/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
-/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!).
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { field: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// let whole = kernel::dma_read!(alloc, [try: 2]);
-/// let field = kernel::dma_read!(alloc, [panic: 1].field);
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_read {
- ($dma:expr, $($proj:tt)*) => {{
- let dma = &$dma;
- let ptr = $crate::ptr::project!(
- $crate::dma::Coherent::as_ptr(dma), $($proj)*
- );
- // SAFETY: The pointer created by the projection is within the DMA region.
- unsafe { $crate::dma::Coherent::field_read(dma, ptr) }
- }};
+/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle.
+pub struct CoherentView<'a, T: ?Sized> {
+ cpu_addr: SysMem<'a, T>,
+ dma_handle: DmaAddress,
}
-/// Writes to a field of an item from an allocated region of structs.
-///
-/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
-/// evaluating to a [`Coherent`], `proj` is a
-/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
-/// projected location.
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { member: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// kernel::dma_write!(alloc, [try: 2].member, 0xf);
-/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf });
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_write {
- (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
- let dma = &$dma;
- let ptr = $crate::ptr::project!(
- mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)*
- );
- let val = $val;
- // SAFETY: The pointer created by the projection is within the DMA region.
- unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) }
- }};
- (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
- $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
- };
- (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
- $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*])
- };
- ($dma:expr, $($rest:tt)*) => {
- $crate::dma_write!(@parse [$dma] [] [$($rest)*])
- };
+impl<T: ?Sized> Copy for CoherentView<'_, T> {}
+impl<T: ?Sized> Clone for CoherentView<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+impl<'a, T: ?Sized> CoherentView<'a, T> {
+ /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region.
+ #[inline]
+ pub fn as_sys_mem(self) -> SysMem<'a, T> {
+ self.cpu_addr
+ }
+
+ /// Returns a DMA handle which may be given to the device as the DMA address base of the region.
+ #[inline]
+ pub fn dma_handle(self) -> DmaAddress {
+ self.dma_handle
+ }
+
+ /// Returns a reference to the data in the region.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that the device does not read/write to/from memory while the returned
+ /// reference is live.
+ /// * Callers must ensure that this call does not race with a write (including call to `as_mut`)
+ /// to the same region while the returned reference is live.
+ #[inline]
+ pub unsafe fn as_ref(self) -> &'a T {
+ // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+ // safety requirement.
+ unsafe { &*self.cpu_addr.as_ptr() }
+ }
+
+ /// Returns a mutable reference to the data in the region.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that the device does not read/write to/from memory while the returned
+ /// reference is live.
+ /// * Callers must ensure that this call does not race with a read (including call to `as_ref`)
+ /// or write (including call to `as_mut`) to the same region while the returned reference is
+ /// live.
+ #[inline]
+ pub unsafe fn as_mut(self) -> &'a mut T {
+ // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+ // safety requirement.
+ unsafe { &mut *self.cpu_addr.as_ptr() }
+ }
+}
+
+/// `IoBackend` implementation for `Coherent`.
+pub struct CoherentIoBackend;
+
+impl IoBackend for CoherentIoBackend {
+ type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ SysMemBackend::as_ptr(view.cpu_addr)
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ let offset = ptr.addr() - view.cpu_addr.as_ptr().addr();
+ // CAST: The offset DMA address can never overflow.
+ let dma_handle = view.dma_handle + offset as DmaAddress;
+ CoherentView {
+ dma_handle,
+ // SAFETY: Per safety requirement.
+ cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) },
+ }
+ }
+}
+
+impl<T> IoCapable<T> for CoherentIoBackend
+where
+ SysMemBackend: IoCapable<T>,
+{
+ #[inline]
+ fn io_read<'a>(view: Self::View<'a, T>) -> T {
+ SysMemBackend::io_read(view.cpu_addr)
+ }
+
+ #[inline]
+ fn io_write<'a>(view: Self::View<'a, T>, value: T) {
+ SysMemBackend::io_write(view.cpu_addr, value)
+ }
+}
+
+impl IoCopyable for CoherentIoBackend {
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ // SAFETY: Per safety requirement.
+ unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) }
+ }
+
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ // SAFETY: Per safety requirement.
+ unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) }
+ }
+
+ #[inline]
+ fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T {
+ SysMemBackend::copy_read(view.cpu_addr)
+ }
+
+ #[inline]
+ fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) {
+ SysMemBackend::copy_write(view.cpu_addr, value)
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
+ type Backend = CoherentIoBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> CoherentView<'a, Self::Target> {
+ self
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
+ type Backend = CoherentIoBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> CoherentView<'a, Self::Target> {
+ CoherentView {
+ // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory.
+ cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) },
+ dma_handle: self.dma_handle,
+ }
+ }
}
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index fcc7678fd9e3..95f46bb75f9e 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -4,9 +4,18 @@
//!
//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
+use core::{
+ marker::PhantomData,
+ mem::MaybeUninit, //
+};
+
use crate::{
bindings,
- prelude::*, //
+ prelude::*,
+ ptr::{
+ Alignment,
+ KnownSize, //
+ }, //
};
pub mod mem;
@@ -31,128 +40,226 @@ pub type PhysAddr = bindings::phys_addr_t;
/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
pub type ResourceSize = bindings::resource_size_t;
+/// Untyped I/O region.
+///
+/// This type can be used when an I/O region without known type information has a compile-time known
+/// minimum size (and a runtime known actual size).
+///
+/// # Invariants
+///
+/// - Size of the region is at least as large as the `SIZE` generic parameter.
+/// - Size of the region is multiple of 4.
+#[repr(C, align(4))]
+#[derive(FromBytes)]
+pub struct Region<const SIZE: usize = 0> {
+ inner: [u8],
+}
+
+impl<const SIZE: usize> Region<SIZE> {
+ /// Create a raw mutable pointer from given base address and size.
+ ///
+ /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should
+ /// be 4-byte aligned to uphold the type invariant.
+ ///
+ /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer
+ /// that does not uphold the type invariants. However such pointers are not valid.
+ #[inline]
+ pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self {
+ core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE>
+ }
+
+ /// Create a raw mutable pointer from given base address and size.
+ ///
+ /// The alignment of `base` is checked, and `size` is checked against the minimum size specified
+ /// via const generics.
+ #[inline]
+ pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> {
+ if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) {
+ return Err(EINVAL);
+ }
+
+ Ok(Self::ptr_from_raw_parts_mut(base, size))
+ }
+}
+
+impl<const SIZE: usize> KnownSize for Region<SIZE> {
+ const MIN_SIZE: usize = SIZE;
+ // Alignment of 4 is the most common; different base types can be added once required.
+ const MIN_ALIGN: Alignment = Alignment::new::<4>();
+
+ #[inline(always)]
+ fn size(p: *const Self) -> usize {
+ (p as *const [u8]).len()
+ }
+}
+
+// SAFETY:
+// - Values read from I/O are always treated as initialized.
+// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding
+// free.
+//
+// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type
+// invariant which the macro does not know.
+unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> {
+ #[inline]
+ #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings.
+ fn only_derive_is_allowed_to_implement_this_trait() {}
+}
+
/// Raw representation of an MMIO region.
///
+/// `MmioRaw<T>` is equivalent to `T __iomem *` in C.
+///
/// By itself, the existence of an instance of this structure does not provide any guarantees that
/// the represented MMIO region does exist or is properly mapped.
///
/// Instead, the bus specific MMIO implementation must convert this raw representation into an
/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio`
/// structure any guarantees are given.
-pub struct MmioRaw<const SIZE: usize = 0> {
- addr: usize,
- maxsize: usize,
+pub struct MmioRaw<T: ?Sized> {
+ /// Pointer is in I/O address space.
+ ///
+ /// The provenance does not matter, only the address and metadata do.
+ ptr: *mut T,
}
-impl<const SIZE: usize> MmioRaw<SIZE> {
- /// Returns a new `MmioRaw` instance on success, an error otherwise.
- pub fn new(addr: usize, maxsize: usize) -> Result<Self> {
- if maxsize < SIZE {
- return Err(EINVAL);
+impl<T: ?Sized> Copy for MmioRaw<T> {}
+impl<T: ?Sized> Clone for MmioRaw<T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Send for MmioRaw<T> {}
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Sync for MmioRaw<T> {}
+
+impl<T> MmioRaw<T> {
+ /// Create a `MmioRaw` from address.
+ #[inline]
+ pub fn new(addr: usize) -> Self {
+ Self {
+ ptr: core::ptr::without_provenance_mut(addr),
}
+ }
+}
- Ok(Self { addr, maxsize })
+impl<const SIZE: usize> MmioRaw<Region<SIZE>> {
+ /// Create a `MmioRaw` representing a I/O region with given size.
+ ///
+ /// The size is checked against the minimum size specified via const generics.
+ #[inline]
+ pub fn new_region(addr: usize, size: usize) -> Result<Self> {
+ Ok(Self {
+ ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?,
+ })
}
+}
+impl<T: ?Sized + KnownSize> MmioRaw<T> {
/// Returns the base address of the MMIO region.
#[inline]
pub fn addr(&self) -> usize {
- self.addr
+ self.ptr.addr()
}
- /// Returns the maximum size of the MMIO region.
+ /// Returns the size of the MMIO region.
#[inline]
- pub fn maxsize(&self) -> usize {
- self.maxsize
+ pub fn size(&self) -> usize {
+ KnownSize::size(self.ptr)
}
}
-/// IO-mapped memory region.
-///
-/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the
-/// mapping, performing an additional region request etc.
-///
-/// # Invariant
-///
-/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size
-/// `maxsize`.
-///
-/// # Examples
-///
-/// ```no_run
-/// use kernel::{
-/// bindings,
-/// ffi::c_void,
-/// io::{
-/// Io,
-/// IoKnownSize,
-/// Mmio,
-/// MmioRaw,
-/// PhysAddr,
-/// },
-/// };
-/// use core::ops::Deref;
-///
-/// // See also `pci::Bar` for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
-///
-/// impl<const SIZE: usize> IoMem<SIZE> {
-/// /// # Safety
-/// ///
-/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs
-/// /// virtual address space.
-/// unsafe fn new(paddr: usize) -> Result<Self>{
-/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is
-/// // valid for `ioremap`.
-/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) };
-/// if addr.is_null() {
-/// return Err(ENOMEM);
-/// }
-///
-/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
-/// }
-/// }
-///
-/// impl<const SIZE: usize> Drop for IoMem<SIZE> {
-/// fn drop(&mut self) {
-/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`.
-/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); };
-/// }
-/// }
-///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-/// type Target = Mmio<SIZE>;
-///
-/// fn deref(&self) -> &Self::Target {
-/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-/// unsafe { Mmio::from_raw(&self.0) }
-/// }
-/// }
-///
-///# fn no_run() -> Result<(), Error> {
-/// // SAFETY: Invalid usage for example purposes.
-/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? };
-/// iomem.write32(0x42, 0x0);
-/// assert!(iomem.try_write32(0x42, 0x0).is_ok());
-/// assert!(iomem.try_write32(0x42, 0x4).is_err());
-/// # Ok(())
-/// # }
-/// ```
-#[repr(transparent)]
-pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>);
-
-/// Checks whether an access of type `U` at the given `offset`
+/// Checks whether an access of type `U` at the given `base` and the given `offset`
/// is valid within this region.
+///
+/// The `base` is used for alignment checking only. This can be set to 0 to skip the check.
#[inline]
-const fn offset_valid<U>(offset: usize, size: usize) -> bool {
- let type_size = core::mem::size_of::<U>();
- if let Some(end) = offset.checked_add(type_size) {
- end <= size && offset % type_size == 0
+const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool {
+ if let Some(end) = offset.checked_add(size_of::<U>()) {
+ end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0)
} else {
false
}
}
+/// Returns a view for a given `offset`, performing compile-time bound checks.
+// Always inline to optimize out error path of `build_assert`.
+#[inline(always)]
+fn io_view_assert<'a, IO: Io<'a>, U>(
+ this: IO,
+ offset: usize,
+) -> <IO::Backend as IoBackend>::View<'a, U> {
+ // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and
+ // ensure alignment by checking that the alignment of `U` is smaller or equal to the
+ // alignment of `IO::Target`.
+ const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize());
+ build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE));
+
+ let view = this.as_view();
+ let ptr = IO::Backend::as_ptr(view);
+ let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+ // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+ // valid projection.
+ unsafe { IO::Backend::project_view(view, projected_ptr) }
+}
+
+/// Returns a view for a given `offset`, performing runtime bound checks.
+#[inline]
+fn io_view<'a, IO: Io<'a>, U>(
+ this: IO,
+ offset: usize,
+) -> Result<<IO::Backend as IoBackend>::View<'a, U>> {
+ let view = this.as_view();
+ let ptr = IO::Backend::as_ptr(view);
+
+ if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) {
+ return Err(EINVAL);
+ }
+
+ let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+ // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+ // valid projection.
+ Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
+}
+
+/// I/O backends.
+///
+/// This is an abstract representation to be implemented by arbitrary I/O
+/// backends (e.g. MMIO, PCI config space, etc.).
+///
+/// The base trait only defines the projection operations; which I/O methods are available depends
+/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions,
+/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI
+/// configuration space, u8, u16, and u32 are supported but u64 is not.
+///
+/// This trait is separate from the `Io` trait as multiple different I/O types may share the same
+/// operation.
+pub trait IoBackend {
+ /// View type for this I/O backend.
+ type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>;
+
+ /// Convert a `view` to a raw pointer for projection.
+ ///
+ /// The returned pointer is private implementation detail of the backend; it is likely not
+ /// valid. It should not be dereferenced.
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T;
+
+ /// Project `view` to its subregion indicated by `ptr`.
+ ///
+ /// If input `view` is valid, returned view must also be valid.
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a projection of `Self::as_ptr(view)`.
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U>;
+}
+
/// Trait indicating that an I/O backend supports operations of a certain type and providing an
/// implementation for these operations.
///
@@ -161,20 +268,75 @@ const fn offset_valid<U>(offset: usize, size: usize) -> bool {
/// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`,
/// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit
/// system might implement all four.
-pub trait IoCapable<T> {
- /// Performs an I/O read of type `T` at `address` and returns the result.
+pub trait IoCapable<T>: IoBackend {
+ /// Performs an I/O read of type `T` at `view` and returns the result.
+ fn io_read<'a>(view: Self::View<'a, T>) -> T;
+
+ /// Performs an I/O write of `value` at `view`.
+ fn io_write<'a>(view: Self::View<'a, T>, value: T);
+}
+
+/// Trait indicating that an I/O backend supports memory copy operations.
+pub trait IoCopyable: IoBackend {
+ /// Copy contents of `view` to `buffer`.
///
/// # Safety
///
- /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
- unsafe fn io_read(&self, address: usize) -> T;
+ /// - `buffer` is valid for volatile write for `view.size()` bytes.
+ /// - `buffer` should not overlap with `view`.
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8);
- /// Performs an I/O write of `value` at `address`.
+ /// Copy contents from `buffer` to `view`.
///
/// # Safety
///
- /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
- unsafe fn io_write(&self, value: T, address: usize);
+ /// - `buffer` is valid for volatile read for `view.size()` bytes.
+ /// - `buffer` should not overlap with `view`.
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8);
+
+ /// Copy from `view` and return the value.
+ #[inline]
+ fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+ // Project `self` to `[u8]`.
+ let ptr = Self::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+ )
+ };
+
+ let mut buf = MaybeUninit::<T>::uninit();
+ // SAFETY:
+ // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes.
+ // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`.
+ unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) };
+ // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid.
+ unsafe { buf.assume_init() }
+ }
+
+ /// Copy `value` to `view`.
+ ///
+ /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`].
+ #[inline]
+ fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+ // Project `self` to `[u8]`.
+ let ptr = Self::as_ptr(view);
+ // SAFETY: This is a identity projection.
+ let slice_view = unsafe {
+ Self::project_view(
+ view,
+ core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+ )
+ };
+
+ // SAFETY:
+ // - `&raw const value` is valid for read for `size_of::<T>()` bytes.
+ // - `value` is local so `&raw const value` cannot overlap with `slice_view`.
+ unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) };
+ core::mem::forget(value);
+ }
}
/// Describes a given I/O location: its offset, width, and type to convert the raw value from and
@@ -186,15 +348,16 @@ pub trait IoCapable<T> {
/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`]
/// macro).
///
-/// An `IoLoc<T>` carries three pieces of information:
+/// An `IoLoc<Base, T>` carries the following pieces of information:
///
+/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
/// - The offset to access (returned by [`IoLoc::offset`]),
/// - The width of the access (determined by [`IoLoc::IoType`]),
/// - The type `T` in which the raw data is returned or provided.
///
/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
-pub trait IoLoc<T> {
+pub trait IoLoc<Base: ?Sized, T> {
/// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
type IoType: Into<T> + From<T>;
@@ -202,12 +365,12 @@ pub trait IoLoc<T> {
fn offset(self) -> usize;
}
-/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of
-/// [`Io::read`] and [`Io::write`].
+/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a
+/// parameter of [`Io::read`] and [`Io::write`].
macro_rules! impl_usize_ioloc {
($($ty:ty),*) => {
$(
- impl IoLoc<$ty> for usize {
+ impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
type IoType = $ty;
#[inline(always)]
@@ -225,181 +388,414 @@ impl_usize_ioloc!(u8, u16, u32, u64);
/// Types implementing this trait (e.g. MMIO BARs or PCI config regions)
/// can perform I/O operations on regions of memory.
///
-/// This is an abstract representation to be implemented by arbitrary I/O
-/// backends (e.g. MMIO, PCI config space, etc.).
+/// This trait defines which backend shall be used for I/O operations and provides a method to
+/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual
+/// methods to perform I/O operations.
+///
+/// This should be implemented on cheaply copyable handles, such as references or view types.
+pub trait IoBase<'a>: Copy {
+ /// Type that defines all I/O operations.
+ type Backend: IoBackend;
+
+ /// Type of this I/O region. For untyped regions, [`Region`] can be used.
+ type Target: ?Sized + KnownSize;
+
+ /// Return a view that covers the full region.
+ fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>;
+}
+
+/// Extension trait to provide I/O operation methods to types that implement [`IoBase`].
///
-/// The [`Io`] trait provides:
-/// - Base address and size information
+/// This trait provides:
/// - Helper methods for offset validation and address calculation
/// - Fallible (runtime checked) accessors for different data widths
///
-/// Which I/O methods are available depends on which [`IoCapable<T>`] traits
-/// are implemented for the type.
-///
-/// # Examples
-///
-/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically
-/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not.
-pub trait Io {
- /// Returns the base address of this mapping.
- fn addr(&self) -> usize;
+/// Which I/O methods are available depends on the associated [`IoBackend`] implementation.
+pub trait Io<'a>: IoBase<'a> {
+ /// Returns the size of this I/O region.
+ #[inline]
+ fn size(self) -> usize {
+ KnownSize::size(Self::Backend::as_ptr(self.as_view()))
+ }
- /// Returns the maximum size of this mapping.
- fn maxsize(&self) -> usize;
+ /// Returns the length of the slice in number of elements.
+ #[inline]
+ fn len<T>(self) -> usize
+ where
+ Self: Io<'a, Target = [T]>,
+ {
+ Self::Backend::as_ptr(self.as_view()).len()
+ }
+
+ /// Returns `true` if the slice has a length of 0.
+ #[inline]
+ fn is_empty<T>(self) -> bool
+ where
+ Self: Io<'a, Target = [T]>,
+ {
+ self.len() == 0
+ }
- /// Returns the absolute I/O address for a given `offset`,
- /// performing runtime bound checks.
+ /// Try to convert into a different typed I/O view.
+ ///
+ /// A runtime check is performed to ensure that the target type is of same or smaller size to
+ /// current type, and the current view is properly aligned for the target type. Returns
+ /// `Err(EINVAL)` if the runtime check fails.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use kernel::io::{
+ /// io_project,
+ /// Mmio,
+ /// Io,
+ /// Region,
+ /// };
+ /// #[derive(FromBytes, IntoBytes)]
+ /// #[repr(C)]
+ /// struct MyStruct { field: u32, }
+ ///
+ /// # fn test(mmio: &Mmio<'_, Region>) -> Result {
+ /// // let mmio: Mmio<'_, Region>;
+ /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?;
+ /// # Ok::<(), Error>(()) }
+ /// ```
#[inline]
- fn io_addr<U>(&self, offset: usize) -> Result<usize> {
- if !offset_valid::<U>(offset, self.maxsize()) {
+ fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>>
+ where
+ Self::Target: FromBytes + IntoBytes,
+ U: FromBytes + IntoBytes,
+ {
+ let view = self.as_view();
+ let ptr = Self::Backend::as_ptr(view);
+
+ if size_of::<U>() > KnownSize::size(ptr) {
return Err(EINVAL);
}
- // Probably no need to check, since the safety requirements of `Self::new` guarantee that
- // this can't overflow.
- self.addr().checked_add(offset).ok_or(EINVAL)
+ if ptr.addr() % align_of::<U>() != 0 {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: We have checked bounds and alignment, so this is a valid projection.
+ Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) })
+ }
+
+ /// Read a value from I/O.
+ ///
+ /// This only works for primitives supported by the I/O backend.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_read_val(mmio: Mmio<'_, u32>) {
+ /// // let mmio: Mmio<'_, u32>;
+ /// let val: u32 = mmio.read_val();
+ /// # }
+ /// ```
+ #[inline]
+ fn read_val(self) -> Self::Target
+ where
+ Self::Backend: IoCapable<Self::Target>,
+ Self::Target: Sized,
+ {
+ Self::Backend::io_read(self.as_view())
+ }
+
+ /// Write a value to I/O.
+ ///
+ /// This only works for primitives supported by the I/O backend.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_write_val(mmio: Mmio<'_, u32>) {
+ /// // let mmio: Mmio<'_, u32>;
+ /// mmio.write_val(1u32);
+ /// # }
+ /// ```
+ #[inline]
+ fn write_val(self, value: Self::Target)
+ where
+ Self::Backend: IoCapable<Self::Target>,
+ Self::Target: Sized,
+ {
+ Self::Backend::io_write(self.as_view(), value)
+ }
+
+ /// Copy-read from I/O memory.
+ ///
+ /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual
+ /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
+ /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as
+ /// byte-swapping is not performed.
+ ///
+ /// [`read_val`]: Io::read_val
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) {
+ /// // let mmio: Mmio<'_, [u8; 6]>;
+ /// let val: [u8; 6] = mmio.copy_read();
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_read(self) -> Self::Target
+ where
+ Self::Backend: IoCopyable,
+ Self::Target: Sized + FromBytes,
+ {
+ Self::Backend::copy_read(self.as_view())
+ }
+
+ /// Copy-write to I/O memory.
+ ///
+ /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual
+ /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
+ /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as
+ /// byte-swapping is not performed.
+ ///
+ /// [`write_val`]: Io::write_val
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) {
+ /// // let mmio: Mmio<'_, [u8; 6]>;
+ /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_write(self, value: Self::Target)
+ where
+ Self::Backend: IoCopyable,
+ Self::Target: Sized + IntoBytes,
+ {
+ Self::Backend::copy_write(self.as_view(), value);
+ }
+
+ /// Copy bytes from `data` to I/O memory.
+ ///
+ /// # Panics
+ ///
+ /// This function will panic if the length of `self` differs from the length of `data`, similar
+ /// to [`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+ /// // let mmio: Mmio<'_, [u8]>;
+ /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_from_slice(self, data: &[u8])
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ {
+ assert_eq!(self.len(), data.len());
+
+ // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes.
+ unsafe {
+ Self::Backend::copy_to_io(self.as_view(), data.as_ptr());
+ }
+ }
+
+ /// Copy bytes from I/O memory to `data`.
+ ///
+ /// # Panics
+ ///
+ /// This function will panic if the length of `self` differs from the length of `data`, similar
+ /// to [`[u8]::copy_from_slice`].
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use kernel::io::*;
+ /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+ /// // let mmio: Mmio<'_, [u8]>;
+ /// let mut buf = [0; 6];
+ /// mmio.copy_to_slice(&mut buf);
+ /// # }
+ /// ```
+ #[inline]
+ fn copy_to_slice(self, data: &mut [u8])
+ where
+ Self::Backend: IoCopyable,
+ Self: Io<'a, Target = [u8]>,
+ {
+ assert_eq!(self.len(), data.len());
+
+ // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes.
+ unsafe {
+ Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr());
+ }
}
/// Fallible 8-bit read with runtime bounds check.
#[inline(always)]
- fn try_read8(&self, offset: usize) -> Result<u8>
+ fn try_read8(self, offset: usize) -> Result<u8>
where
- Self: IoCapable<u8>,
+ usize: IoLoc<Self::Target, u8, IoType = u8>,
+ Self::Backend: IoCapable<u8>,
{
self.try_read(offset)
}
/// Fallible 16-bit read with runtime bounds check.
#[inline(always)]
- fn try_read16(&self, offset: usize) -> Result<u16>
+ fn try_read16(self, offset: usize) -> Result<u16>
where
- Self: IoCapable<u16>,
+ usize: IoLoc<Self::Target, u16, IoType = u16>,
+ Self::Backend: IoCapable<u16>,
{
self.try_read(offset)
}
/// Fallible 32-bit read with runtime bounds check.
#[inline(always)]
- fn try_read32(&self, offset: usize) -> Result<u32>
+ fn try_read32(self, offset: usize) -> Result<u32>
where
- Self: IoCapable<u32>,
+ usize: IoLoc<Self::Target, u32, IoType = u32>,
+ Self::Backend: IoCapable<u32>,
{
self.try_read(offset)
}
/// Fallible 64-bit read with runtime bounds check.
#[inline(always)]
- fn try_read64(&self, offset: usize) -> Result<u64>
+ fn try_read64(self, offset: usize) -> Result<u64>
where
- Self: IoCapable<u64>,
+ usize: IoLoc<Self::Target, u64, IoType = u64>,
+ Self::Backend: IoCapable<u64>,
{
self.try_read(offset)
}
/// Fallible 8-bit write with runtime bounds check.
#[inline(always)]
- fn try_write8(&self, value: u8, offset: usize) -> Result
+ fn try_write8(self, value: u8, offset: usize) -> Result
where
- Self: IoCapable<u8>,
+ usize: IoLoc<Self::Target, u8, IoType = u8>,
+ Self::Backend: IoCapable<u8>,
{
self.try_write(offset, value)
}
/// Fallible 16-bit write with runtime bounds check.
#[inline(always)]
- fn try_write16(&self, value: u16, offset: usize) -> Result
+ fn try_write16(self, value: u16, offset: usize) -> Result
where
- Self: IoCapable<u16>,
+ usize: IoLoc<Self::Target, u16, IoType = u16>,
+ Self::Backend: IoCapable<u16>,
{
self.try_write(offset, value)
}
/// Fallible 32-bit write with runtime bounds check.
#[inline(always)]
- fn try_write32(&self, value: u32, offset: usize) -> Result
+ fn try_write32(self, value: u32, offset: usize) -> Result
where
- Self: IoCapable<u32>,
+ usize: IoLoc<Self::Target, u32, IoType = u32>,
+ Self::Backend: IoCapable<u32>,
{
self.try_write(offset, value)
}
/// Fallible 64-bit write with runtime bounds check.
#[inline(always)]
- fn try_write64(&self, value: u64, offset: usize) -> Result
+ fn try_write64(self, value: u64, offset: usize) -> Result
where
- Self: IoCapable<u64>,
+ usize: IoLoc<Self::Target, u64, IoType = u64>,
+ Self::Backend: IoCapable<u64>,
{
self.try_write(offset, value)
}
/// Infallible 8-bit read with compile-time bounds check.
#[inline(always)]
- fn read8(&self, offset: usize) -> u8
+ fn read8(self, offset: usize) -> u8
where
- Self: IoKnownSize + IoCapable<u8>,
+ usize: IoLoc<Self::Target, u8, IoType = u8>,
+ Self::Backend: IoCapable<u8>,
{
self.read(offset)
}
/// Infallible 16-bit read with compile-time bounds check.
#[inline(always)]
- fn read16(&self, offset: usize) -> u16
+ fn read16(self, offset: usize) -> u16
where
- Self: IoKnownSize + IoCapable<u16>,
+ usize: IoLoc<Self::Target, u16, IoType = u16>,
+ Self::Backend: IoCapable<u16>,
{
self.read(offset)
}
/// Infallible 32-bit read with compile-time bounds check.
#[inline(always)]
- fn read32(&self, offset: usize) -> u32
+ fn read32(self, offset: usize) -> u32
where
- Self: IoKnownSize + IoCapable<u32>,
+ usize: IoLoc<Self::Target, u32, IoType = u32>,
+ Self::Backend: IoCapable<u32>,
{
self.read(offset)
}
/// Infallible 64-bit read with compile-time bounds check.
#[inline(always)]
- fn read64(&self, offset: usize) -> u64
+ fn read64(self, offset: usize) -> u64
where
- Self: IoKnownSize + IoCapable<u64>,
+ usize: IoLoc<Self::Target, u64, IoType = u64>,
+ Self::Backend: IoCapable<u64>,
{
self.read(offset)
}
/// Infallible 8-bit write with compile-time bounds check.
#[inline(always)]
- fn write8(&self, value: u8, offset: usize)
+ fn write8(self, value: u8, offset: usize)
where
- Self: IoKnownSize + IoCapable<u8>,
+ usize: IoLoc<Self::Target, u8, IoType = u8>,
+ Self::Backend: IoCapable<u8>,
{
self.write(offset, value)
}
/// Infallible 16-bit write with compile-time bounds check.
#[inline(always)]
- fn write16(&self, value: u16, offset: usize)
+ fn write16(self, value: u16, offset: usize)
where
- Self: IoKnownSize + IoCapable<u16>,
+ usize: IoLoc<Self::Target, u16, IoType = u16>,
+ Self::Backend: IoCapable<u16>,
{
self.write(offset, value)
}
/// Infallible 32-bit write with compile-time bounds check.
#[inline(always)]
- fn write32(&self, value: u32, offset: usize)
+ fn write32(self, value: u32, offset: usize)
where
- Self: IoKnownSize + IoCapable<u32>,
+ usize: IoLoc<Self::Target, u32, IoType = u32>,
+ Self::Backend: IoCapable<u32>,
{
self.write(offset, value)
}
/// Infallible 64-bit write with compile-time bounds check.
#[inline(always)]
- fn write64(&self, value: u64, offset: usize)
+ fn write64(self, value: u64, offset: usize)
where
- Self: IoKnownSize + IoCapable<u64>,
+ usize: IoLoc<Self::Target, u64, IoType = u64>,
+ Self::Backend: IoCapable<u64>,
{
self.write(offset, value)
}
@@ -414,9 +810,10 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_reads(io: &Mmio) -> Result {
+ /// fn do_reads(io: Mmio<'_, Region>) -> Result {
/// // 32-bit read from address `0x10`.
/// let v: u32 = io.try_read(0x10)?;
///
@@ -427,15 +824,13 @@ pub trait Io {
/// }
/// ```
#[inline(always)]
- fn try_read<T, L>(&self, location: L) -> Result<T>
+ fn try_read<T, L>(self, location: L) -> Result<T>
where
- L: IoLoc<T>,
- Self: IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
{
- let address = self.io_addr::<L::IoType>(location.offset())?;
-
- // SAFETY: `address` has been validated by `io_addr`.
- Ok(unsafe { self.io_read(address) }.into())
+ let view = io_view::<Self, L::IoType>(self, location.offset())?;
+ Ok(Self::Backend::io_read(view).into())
}
/// Generic fallible write with runtime bounds check.
@@ -448,9 +843,10 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_writes(io: &Mmio) -> Result {
+ /// fn do_writes(io: Mmio<'_, Region>) -> Result {
/// // 32-bit write of value `1` at address `0x10`.
/// io.try_write(0x10, 1u32)?;
///
@@ -461,17 +857,14 @@ pub trait Io {
/// }
/// ```
#[inline(always)]
- fn try_write<T, L>(&self, location: L, value: T) -> Result
+ fn try_write<T, L>(self, location: L, value: T) -> Result
where
- L: IoLoc<T>,
- Self: IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
{
- let address = self.io_addr::<L::IoType>(location.offset())?;
+ let view = io_view::<Self, L::IoType>(self, location.offset())?;
let io_value = value.into();
-
- // SAFETY: `address` has been validated by `io_addr`.
- unsafe { self.io_write(io_value, address) }
-
+ Self::Backend::io_write(view, io_value);
Ok(())
}
@@ -486,6 +879,7 @@ pub trait Io {
/// register,
/// Io,
/// Mmio,
+ /// Region,
/// };
///
/// register! {
@@ -501,17 +895,17 @@ pub trait Io {
/// }
/// }
///
- /// fn do_write_reg(io: &Mmio) -> Result {
+ /// fn do_write_reg(io: Mmio<'_, Region>) -> Result {
///
/// io.try_write_reg(VERSION::new(1, 0))
/// }
/// ```
#[inline(always)]
- fn try_write_reg<T, L, V>(&self, value: V) -> Result
+ fn try_write_reg<T, L, V>(self, value: V) -> Result
where
- L: IoLoc<T>,
- V: LocatedRegister<Location = L, Value = T>,
- Self: IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ V: LocatedRegister<Self::Target, Location = L, Value = T>,
+ Self::Backend: IoCapable<L::IoType>,
{
let (location, value) = value.into_io_op();
@@ -531,29 +925,27 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_update(io: &Mmio<0x1000>) -> Result {
+ /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result {
/// io.try_update(0x10, |v: u32| {
/// v + 1
/// })
/// }
/// ```
#[inline(always)]
- fn try_update<T, L, F>(&self, location: L, f: F) -> Result
+ fn try_update<T, L, F>(self, location: L, f: F) -> Result
where
- L: IoLoc<T>,
- Self: IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
F: FnOnce(T) -> T,
{
- let address = self.io_addr::<L::IoType>(location.offset())?;
+ let view = io_view::<Self, L::IoType>(self, location.offset())?;
- // SAFETY: `address` has been validated by `io_addr`.
- let value: T = unsafe { self.io_read(address) }.into();
+ let value: T = Self::Backend::io_read(view).into();
let io_value = f(value).into();
-
- // SAFETY: `address` has been validated by `io_addr`.
- unsafe { self.io_write(io_value, address) }
+ Self::Backend::io_write(view, io_value);
Ok(())
}
@@ -568,9 +960,10 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_reads(io: &Mmio<0x1000>) {
+ /// fn do_reads(io: Mmio<'_, Region<0x1000>>) {
/// // 32-bit read from address `0x10`.
/// let v: u32 = io.read(0x10);
///
@@ -579,15 +972,13 @@ pub trait Io {
/// }
/// ```
#[inline(always)]
- fn read<T, L>(&self, location: L) -> T
+ fn read<T, L>(self, location: L) -> T
where
- L: IoLoc<T>,
- Self: IoKnownSize + IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
{
- let address = self.io_addr_assert::<L::IoType>(location.offset());
-
- // SAFETY: `address` has been validated by `io_addr_assert`.
- unsafe { self.io_read(address) }.into()
+ let view = io_view_assert::<Self, L::IoType>(self, location.offset());
+ Self::Backend::io_read(view).into()
}
/// Generic infallible write with compile-time bounds check.
@@ -600,9 +991,10 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_writes(io: &Mmio<0x1000>) {
+ /// fn do_writes(io: Mmio<'_, Region<0x1000>>) {
/// // 32-bit write of value `1` at address `0x10`.
/// io.write(0x10, 1u32);
///
@@ -611,16 +1003,14 @@ pub trait Io {
/// }
/// ```
#[inline(always)]
- fn write<T, L>(&self, location: L, value: T)
+ fn write<T, L>(self, location: L, value: T)
where
- L: IoLoc<T>,
- Self: IoKnownSize + IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
{
- let address = self.io_addr_assert::<L::IoType>(location.offset());
+ let view = io_view_assert::<Self, L::IoType>(self, location.offset());
let io_value = value.into();
-
- // SAFETY: `address` has been validated by `io_addr_assert`.
- unsafe { self.io_write(io_value, address) }
+ Self::Backend::io_write(view, io_value);
}
/// Generic infallible write of a fully-located register value.
@@ -634,6 +1024,7 @@ pub trait Io {
/// register,
/// Io,
/// Mmio,
+ /// Region,
/// };
///
/// register! {
@@ -649,16 +1040,16 @@ pub trait Io {
/// }
/// }
///
- /// fn do_write_reg(io: &Mmio<0x1000>) {
+ /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) {
/// io.write_reg(VERSION::new(1, 0));
/// }
/// ```
#[inline(always)]
- fn write_reg<T, L, V>(&self, value: V)
+ fn write_reg<T, L, V>(self, value: V)
where
- L: IoLoc<T>,
- V: LocatedRegister<Location = L, Value = T>,
- Self: IoKnownSize + IoCapable<L::IoType>,
+ L: IoLoc<Self::Target, T>,
+ V: LocatedRegister<Self::Target, Location = L, Value = T>,
+ Self::Backend: IoCapable<L::IoType>,
{
let (location, value) = value.into_io_op();
@@ -678,143 +1069,208 @@ pub trait Io {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// };
///
- /// fn do_update(io: &Mmio<0x1000>) {
+ /// fn do_update(io: Mmio<'_, Region<0x1000>>) {
/// io.update(0x10, |v: u32| {
/// v + 1
/// })
/// }
/// ```
#[inline(always)]
- fn update<T, L, F>(&self, location: L, f: F)
+ fn update<T, L, F>(self, location: L, f: F)
where
- L: IoLoc<T>,
- Self: IoKnownSize + IoCapable<L::IoType> + Sized,
+ L: IoLoc<Self::Target, T>,
+ Self::Backend: IoCapable<L::IoType>,
F: FnOnce(T) -> T,
{
- let address = self.io_addr_assert::<L::IoType>(location.offset());
-
- // SAFETY: `address` has been validated by `io_addr_assert`.
- let value: T = unsafe { self.io_read(address) }.into();
+ let view = io_view_assert::<Self, L::IoType>(self, location.offset());
+ let value: T = Self::Backend::io_read(view).into();
let io_value = f(value).into();
-
- // SAFETY: `address` has been validated by `io_addr_assert`.
- unsafe { self.io_write(io_value, address) }
+ Self::Backend::io_write(view, io_value);
}
}
-/// Trait for types with a known size at compile time.
+// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by
+// implementers, which is relied upon for correctness and soundness.
+impl<'a, T: IoBase<'a>> Io<'a> for T {}
+
+/// A view of memory-mapped I/O region.
///
-/// This trait is implemented by I/O backends that have a compile-time known size,
-/// enabling the use of infallible I/O accessors with compile-time bounds checking.
+/// # Invariant
///
-/// Types implementing this trait can use the infallible methods in [`Io`] trait
-/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound.
-pub trait IoKnownSize: Io {
- /// Minimum usable size of this region.
- const MIN_SIZE: usize;
+/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`.
+pub struct Mmio<'a, T: ?Sized> {
+ ptr: *mut T,
+ phantom: PhantomData<&'a ()>,
+}
- /// Returns the absolute I/O address for a given `offset`,
- /// performing compile-time bound checks.
- // Always inline to optimize out error path of `build_assert`.
- #[inline(always)]
- fn io_addr_assert<U>(&self, offset: usize) -> usize {
- build_assert!(offset_valid::<U>(offset, Self::MIN_SIZE));
+impl<T: ?Sized> Copy for Mmio<'_, T> {}
+impl<T: ?Sized> Clone for Mmio<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+impl<'a, T: ?Sized> Mmio<'a, T> {
+ /// Create a `Mmio`, providing the accessors to the MMIO mapping.
+ ///
+ /// # Safety
+ ///
+ /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive.
+ #[inline]
+ pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self {
+ // INVARIANT: Per safety requirement.
+ Self {
+ ptr: raw.ptr,
+ phantom: PhantomData,
+ }
+ }
+}
+
+// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {}
- self.addr() + offset
+// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> {
+ type Backend = MmioBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> Mmio<'a, T> {
+ self
+ }
+}
+
+/// I/O Backend for memory-mapped I/O.
+pub struct MmioBackend;
+
+impl IoBackend for MmioBackend {
+ type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ view.ptr
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ _view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
+ // memory-mapped I/O region.
+ Mmio {
+ ptr,
+ phantom: PhantomData,
+ }
}
}
-/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`.
+/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`.
macro_rules! impl_mmio_io_capable {
- ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => {
- $(#[$attr])*
- impl<const SIZE: usize> IoCapable<$ty> for $mmio<SIZE> {
- unsafe fn io_read(&self, address: usize) -> $ty {
- // SAFETY: By the trait invariant `address` is a valid address for MMIO operations.
- unsafe { bindings::$read_fn(address as *const c_void) }
+ ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => {
+ impl IoCapable<$ty> for $backend {
+ #[inline]
+ fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty {
+ // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
+ // `MmioBackend` and `RelaxedMmioBackend`.
+ unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) }
}
- unsafe fn io_write(&self, value: $ty, address: usize) {
- // SAFETY: By the trait invariant `address` is a valid address for MMIO operations.
- unsafe { bindings::$write_fn(value, address as *mut c_void) }
+ #[inline]
+ fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) {
+ // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
+ // `MmioBackend` and `RelaxedMmioBackend`.
+ unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) }
}
}
};
}
// MMIO regions support 8, 16, and 32-bit accesses.
-impl_mmio_io_capable!(Mmio, u8, readb, writeb);
-impl_mmio_io_capable!(Mmio, u16, readw, writew);
-impl_mmio_io_capable!(Mmio, u32, readl, writel);
+impl_mmio_io_capable!(MmioBackend, u8, readb, writeb);
+impl_mmio_io_capable!(MmioBackend, u16, readw, writew);
+impl_mmio_io_capable!(MmioBackend, u32, readl, writel);
// MMIO regions on 64-bit systems also support 64-bit accesses.
-impl_mmio_io_capable!(
- Mmio,
- #[cfg(CONFIG_64BIT)]
- u64,
- readq,
- writeq
-);
+#[cfg(CONFIG_64BIT)]
+impl_mmio_io_capable!(MmioBackend, u64, readq, writeq);
-impl<const SIZE: usize> Io for Mmio<SIZE> {
- /// Returns the base address of this mapping.
+impl IoCopyable for MmioBackend {
#[inline]
- fn addr(&self) -> usize {
- self.0.addr()
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ // SAFETY:
+ // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+ // - `buffer` is valid for write for `view.size()` bytes.
+ unsafe {
+ bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size());
+ }
}
- /// Returns the maximum size of this mapping.
#[inline]
- fn maxsize(&self) -> usize {
- self.0.maxsize()
- }
-}
-
-impl<const SIZE: usize> IoKnownSize for Mmio<SIZE> {
- const MIN_SIZE: usize = SIZE;
-}
-
-impl<const SIZE: usize> Mmio<SIZE> {
- /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping.
- ///
- /// # Safety
- ///
- /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size
- /// `maxsize`.
- pub unsafe fn from_raw(raw: &MmioRaw<SIZE>) -> &Self {
- // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`.
- unsafe { &*core::ptr::from_ref(raw).cast() }
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ // SAFETY:
+ // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+ // - `buffer` is valid for read for `view.size()` bytes.
+ unsafe {
+ bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size());
+ }
}
}
-/// [`Mmio`] wrapper using relaxed accessors.
+/// [`Mmio`] but using relaxed accessors.
///
/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of
/// the regular ones.
///
/// See [`Mmio::relaxed`] for a usage example.
-#[repr(transparent)]
-pub struct RelaxedMmio<const SIZE: usize = 0>(Mmio<SIZE>);
+pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>);
+
+impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {}
+impl<T: ?Sized> Clone for RelaxedMmio<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+/// I/O Backend for memory-mapped I/O, with relaxed access semantics.
+pub struct RelaxedMmioBackend;
+
+impl IoBackend for RelaxedMmioBackend {
+ type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>;
-impl<const SIZE: usize> Io for RelaxedMmio<SIZE> {
#[inline]
- fn addr(&self) -> usize {
- self.0.addr()
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ MmioBackend::as_ptr(view.0)
}
#[inline]
- fn maxsize(&self) -> usize {
- self.0.maxsize()
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ // SAFETY: Per safety requirement.
+ RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) })
}
}
-impl<const SIZE: usize> IoKnownSize for RelaxedMmio<SIZE> {
- const MIN_SIZE: usize = SIZE;
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> {
+ type Backend = RelaxedMmioBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> RelaxedMmio<'a, T> {
+ self
+ }
}
-impl<const SIZE: usize> Mmio<SIZE> {
- /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations.
+impl<'a, T: ?Sized> Mmio<'a, T> {
+ /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations.
///
/// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses
/// and can be used when such ordering is not required.
@@ -825,31 +1281,457 @@ impl<const SIZE: usize> Mmio<SIZE> {
/// use kernel::io::{
/// Io,
/// Mmio,
+ /// Region,
/// RelaxedMmio,
/// };
///
- /// fn do_io(io: &Mmio<0x100>) {
+ /// fn do_io(io: Mmio<'_, Region<0x100>>) {
/// // The access is performed using `readl_relaxed` instead of `readl`.
/// let v = io.relaxed().read32(0x10);
/// }
///
/// ```
- pub fn relaxed(&self) -> &RelaxedMmio<SIZE> {
- // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio<SIZE>` and
- // `RelaxedMmio<SIZE>` have identical layout.
- unsafe { core::mem::transmute(self) }
+ #[inline]
+ pub fn relaxed(self) -> RelaxedMmio<'a, T> {
+ RelaxedMmio(self)
}
}
// MMIO regions support 8, 16, and 32-bit accesses.
-impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed);
-impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed);
-impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed);
// MMIO regions on 64-bit systems also support 64-bit accesses.
-impl_mmio_io_capable!(
- RelaxedMmio,
- #[cfg(CONFIG_64BIT)]
- u64,
- readq_relaxed,
- writeq_relaxed
-);
+#[cfg(CONFIG_64BIT)]
+impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed);
+
+/// I/O Backend for system memory.
+pub struct SysMemBackend;
+
+impl IoBackend for SysMemBackend {
+ type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ view.ptr
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ _view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
+ // kernel accessible memory region.
+ SysMem {
+ ptr,
+ phantom: PhantomData,
+ }
+ }
+}
+
+/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and
+/// `write_volatile`.
+macro_rules! impl_sysmem_io_capable {
+ ($ty:ty) => {
+ impl IoCapable<$ty> for SysMemBackend {
+ #[inline]
+ fn io_read(view: SysMem<'_, $ty>) -> $ty {
+ // SAFETY:
+ // - Per type invariant, `ptr` is valid and aligned.
+ // - Using read_volatile() here so that race with hardware is well-defined.
+ // - Using read_volatile() here is not sound if it races with other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ // - The macro is only used on primitives so all bit patterns are valid.
+ unsafe { view.ptr.read_volatile() }
+ }
+
+ #[inline]
+ fn io_write(view: SysMem<'_, $ty>, value: $ty) {
+ // SAFETY:
+ // - Per type invariant, `ptr` is valid and aligned.
+ // - Using write_volatile() here so that race with hardware is well-defined.
+ // - Using write_volatile() here is not sound if it races with other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ unsafe { view.ptr.write_volatile(value) }
+ }
+ }
+ };
+}
+
+impl_sysmem_io_capable!(u8);
+impl_sysmem_io_capable!(u16);
+impl_sysmem_io_capable!(u32);
+#[cfg(CONFIG_64BIT)]
+impl_sysmem_io_capable!(u64);
+
+impl IoCopyable for SysMemBackend {
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
+ // SAFETY:
+ // - `view.ptr` is in CPU address space and valid for read.
+ // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`.
+ unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) };
+ }
+
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
+ // SAFETY:
+ // - `view.ptr` is in CPU address space and valid for write.
+ // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`.
+ unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) };
+ }
+
+ #[inline]
+ fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+ // SAFETY:
+ // - Per type invariant, `ptr` is valid and aligned.
+ // - Using read_volatile() here so that race with hardware is well-defined.
+ // - Using read_volatile() here is not sound if it races with other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ // - `T: FromBytes` so all bit patterns are valid.
+ unsafe { view.ptr.read_volatile() }
+ }
+
+ #[inline]
+ fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+ // SAFETY:
+ // - Per type invariant, `ptr` is valid and aligned.
+ // - Using write_volatile() here so that race with hardware is well-defined.
+ // - Using write_volatile() here is not sound if it races with other CPU per Rust
+ // rules, but this is allowed per LKMM.
+ unsafe { view.ptr.write_volatile(value) }
+ }
+}
+
+/// A view of a system memory region.
+///
+/// Provides `Io` trait implementation for kernel virtual address ranges,
+/// using volatile read/write to safely access shared memory that may be
+/// concurrently accessed by external hardware.
+///
+/// # Invariants
+///
+/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel
+/// accessible memory region for the lifetime `'a`.
+pub struct SysMem<'a, T: ?Sized> {
+ ptr: *mut T,
+ phantom: PhantomData<&'a ()>,
+}
+
+impl<T: ?Sized> Copy for SysMem<'_, T> {}
+impl<T: ?Sized> Clone for SysMem<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
+unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {}
+
+// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
+unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {}
+
+impl<'a, T: ?Sized> SysMem<'a, T> {
+ /// Create a `SysMem` from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel
+ /// accessible memory region for the lifetime `'a`.
+ #[inline]
+ pub unsafe fn new(ptr: *mut T) -> Self {
+ // INVARIANT: Per safety requirement.
+ Self {
+ ptr,
+ phantom: PhantomData,
+ }
+ }
+
+ /// Obtain the raw pointer to the memory.
+ #[inline]
+ pub fn as_ptr(self) -> *mut T {
+ self.ptr
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> {
+ type Backend = SysMemBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> {
+ self
+ }
+}
+
+/// I/O Backend for [`IoSysMap`].
+pub struct IoSysMapBackend;
+
+/// Either [`Mmio`] or [`SysMem`].
+///
+/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does
+/// not want or cannot be generic over I/O backends. This serves a similar purpose to
+/// [`include/linux/iosys-map.h`] in C.
+///
+/// This type can be used like any other types that implements [`Io`]; this also include
+/// [`io_project!`], [`io_read!`], [`io_write!`].
+///
+/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h
+pub enum IoSysMap<'a, T: ?Sized> {
+ /// The view is I/O memory.
+ Io(Mmio<'a, T>),
+ /// The view is system memory.
+ Sys(SysMem<'a, T>),
+}
+
+impl<T: ?Sized> Copy for IoSysMap<'_, T> {}
+impl<T: ?Sized> Clone for IoSysMap<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> {
+ #[inline]
+ fn from(value: Mmio<'a, T>) -> Self {
+ IoSysMap::Io(value)
+ }
+}
+
+impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> {
+ #[inline]
+ fn from(value: SysMem<'a, T>) -> Self {
+ IoSysMap::Sys(value)
+ }
+}
+
+impl IoBackend for IoSysMapBackend {
+ type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+ match view {
+ IoSysMap::Io(l) => MmioBackend::as_ptr(l),
+ IoSysMap::Sys(r) => SysMemBackend::as_ptr(r),
+ }
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ match view {
+ // SAFETY: Per safety requirement.
+ IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }),
+ // SAFETY: Per safety requirement.
+ IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }),
+ }
+ }
+}
+
+impl<T> IoCapable<T> for IoSysMapBackend
+where
+ MmioBackend: IoCapable<T>,
+ SysMemBackend: IoCapable<T>,
+{
+ #[inline]
+ fn io_read(view: Self::View<'_, T>) -> T {
+ match view {
+ IoSysMap::Io(l) => MmioBackend::io_read(l),
+ IoSysMap::Sys(r) => SysMemBackend::io_read(r),
+ }
+ }
+
+ #[inline]
+ fn io_write<'a>(view: Self::View<'a, T>, value: T) {
+ match view {
+ IoSysMap::Io(l) => MmioBackend::io_write(l, value),
+ IoSysMap::Sys(r) => SysMemBackend::io_write(r, value),
+ }
+ }
+}
+
+impl IoCopyable for IoSysMapBackend {
+ #[inline]
+ unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+ match view {
+ // SAFETY: Per safety requirement.
+ IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) },
+ // SAFETY: Per safety requirement.
+ IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) },
+ }
+ }
+
+ #[inline]
+ unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+ match view {
+ // SAFETY: Per safety requirement.
+ IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) },
+ // SAFETY: Per safety requirement.
+ IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) },
+ }
+ }
+
+ #[inline]
+ fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+ match view {
+ IoSysMap::Io(l) => MmioBackend::copy_read(l),
+ IoSysMap::Sys(r) => SysMemBackend::copy_read(r),
+ }
+ }
+
+ #[inline]
+ fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+ match view {
+ IoSysMap::Io(l) => MmioBackend::copy_write(l, value),
+ IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value),
+ }
+ }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> {
+ type Backend = IoSysMapBackend;
+ type Target = T;
+
+ #[inline]
+ fn as_view(self) -> IoSysMap<'a, T> {
+ self
+ }
+}
+
+// This helper turns associated functions to methods so it can be invoked in macro.
+// Used by `io_project!()` only.
+#[doc(hidden)]
+#[derive(Clone, Copy)]
+pub struct ProjectHelper<T>(pub T);
+
+impl<'a, T> ProjectHelper<T>
+where
+ T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>,
+{
+ // These helper methods must not have symbols present in the binary to avoid confusion.
+ #[inline(always)]
+ pub fn as_ptr(self) -> *mut T::Target {
+ T::Backend::as_ptr(self.0)
+ }
+
+ /// # Safety
+ ///
+ /// Same as `IoBackend::project_view`
+ #[inline(always)]
+ pub unsafe fn project_view<U: ?Sized + KnownSize>(
+ self,
+ ptr: *mut U,
+ ) -> <T::Backend as IoBackend>::View<'a, U> {
+ // SAFETY: Per safety requirement.
+ unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
+ }
+}
+
+/// Project an I/O type to a subview of it.
+///
+/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
+///
+/// # Examples
+///
+/// ```
+/// use kernel::io::{
+/// io_project,
+/// Mmio,
+/// };
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<[MyStruct]>;
+/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
+/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
+/// let nested: Mmio<'_, u32> = io_project!(whole, .field);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_project {
+ ($io:expr, $($proj:tt)*) => {{
+ #[allow(unused)]
+ use $crate::io::IoBase as _;
+ let view = $crate::io::ProjectHelper($io.as_view());
+ let ptr = $crate::ptr::project!(
+ mut view.as_ptr(), $($proj)*
+ );
+ #[allow(unused_unsafe)]
+ // SAFETY: `ptr` is a projection.
+ unsafe { view.project_view(ptr) }
+ }};
+}
+#[doc(inline)]
+pub use crate::io_project;
+
+/// Read from I/O memory.
+///
+/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
+///
+/// # Examples
+///
+/// ```
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<'_, [MyStruct]>;
+/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_read {
+ ($io:expr, $($proj:tt)*) => {
+ $crate::io::Io::read_val($crate::io_project!($io, $($proj)*))
+ };
+}
+#[doc(inline)]
+pub use crate::io_read;
+
+/// Writes to I/O memory.
+///
+/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!),
+/// and `val` is the value to be written to the projected location.
+///
+/// # Examples
+///
+/// ```
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<'_, [MyStruct]>;
+/// kernel::io::io_write!(mmio, [try: 2].field, 10);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_write {
+ (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => {
+ $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val)
+ };
+ (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
+ $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*])
+ };
+ (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
+ $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
+ };
+ ($io:expr, $($rest:tt)*) => {
+ $crate::io_write!(@parse [$io] [] [$($rest)*])
+ };
+}
+#[doc(inline)]
+pub use crate::io_write;
diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs
index 931f2fa3bb10..32a919099dcd 100644
--- a/rust/kernel/io/mem.rs
+++ b/rust/kernel/io/mem.rs
@@ -2,8 +2,6 @@
//! Generic memory-mapped IO.
-use core::ops::Deref;
-
use crate::{
device::{
Bound,
@@ -16,7 +14,9 @@ use crate::{
Region,
Resource, //
},
+ IoBase,
Mmio,
+ MmioBackend,
MmioRaw, //
},
prelude::*,
@@ -225,11 +225,13 @@ impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> {
}
}
-impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> {
- type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> {
+ type Backend = MmioBackend;
+ type Target = super::Region<SIZE>;
- fn deref(&self) -> &Self::Target {
- &self.iomem
+ #[inline]
+ fn as_view(self) -> Mmio<'a, Self::Target> {
+ self.iomem.as_view()
}
}
@@ -244,7 +246,7 @@ impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> {
/// start of the I/O memory mapped region.
pub struct IoMem<'a, const SIZE: usize = 0> {
dev: &'a Device<Bound>,
- io: MmioRaw<SIZE>,
+ io: MmioRaw<super::Region<SIZE>>,
}
impl<const SIZE: usize> ForLt for IoMem<'static, SIZE> {
@@ -292,8 +294,7 @@ impl<'a, const SIZE: usize> IoMem<'a, SIZE> {
return Err(ENOMEM);
}
- let io = MmioRaw::new(addr as usize, size)?;
-
+ let io = MmioRaw::new_region(addr as usize, size)?;
Ok(IoMem { dev, io })
}
@@ -316,11 +317,13 @@ impl<const SIZE: usize> Drop for IoMem<'_, SIZE> {
}
}
-impl<const SIZE: usize> Deref for IoMem<'_, SIZE> {
- type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> {
+ type Backend = MmioBackend;
+ type Target = super::Region<SIZE>;
- fn deref(&self) -> &Self::Target {
+ #[inline]
+ fn as_view(self) -> Mmio<'a, Self::Target> {
// SAFETY: Safe as by the invariant of `IoMem`.
- unsafe { Mmio::from_raw(&self.io) }
+ unsafe { Mmio::from_raw(self.io) }
}
}
diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs
index 75d1b3e8596c..d75f2fcf46f2 100644
--- a/rust/kernel/io/poll.rs
+++ b/rust/kernel/io/poll.rs
@@ -48,13 +48,14 @@ use crate::{
/// use kernel::io::{
/// Io,
/// Mmio,
+/// Region,
/// poll::read_poll_timeout, //
/// };
/// use kernel::time::Delta;
///
/// const HW_READY: u16 = 0x01;
///
-/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result {
+/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result {
/// read_poll_timeout(
/// // The `op` closure reads the value of a specific status register.
/// || io.try_read16(0x1000),
@@ -135,13 +136,14 @@ where
/// use kernel::io::{
/// Io,
/// Mmio,
+/// Region,
/// poll::read_poll_timeout_atomic, //
/// };
/// use kernel::time::Delta;
///
/// const HW_READY: u16 = 0x01;
///
-/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result {
+/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result {
/// read_poll_timeout_atomic(
/// // The `op` closure reads the value of a specific status register.
/// || io.try_read16(0x1000),
diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs
index 66449377a0b7..6cb07fc92cc3 100644
--- a/rust/kernel/io/register.rs
+++ b/rust/kernel/io/register.rs
@@ -58,7 +58,7 @@
//! },
//! num::Bounded,
//! };
-//! # use kernel::io::Mmio;
+//! # use kernel::io::{Mmio, Region};
//! # register! {
//! # pub BOOT_0(u32) @ 0x00000100 {
//! # 15:8 vendor_id;
@@ -66,7 +66,7 @@
//! # 3:0 minor_revision;
//! # }
//! # }
-//! # fn test(io: &Mmio<0x1000>) {
+//! # fn test(io: Mmio<'_, Region<0x1000>>) {
//! # fn obtain_vendor_id() -> u8 { 0xff }
//!
//! // Read from the register's defined offset (0x100).
@@ -113,6 +113,8 @@ use crate::{
io::IoLoc, //
};
+use super::Region;
+
/// Trait implemented by all registers.
pub trait Register: Sized {
/// Backing primitive type of the register.
@@ -129,7 +131,7 @@ pub trait FixedRegister: Register {}
/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
/// passing a [`FixedRegister`] value.
-impl<T> IoLoc<T> for ()
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for ()
where
T: FixedRegister,
{
@@ -143,7 +145,7 @@ where
/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used
/// as an [`IoLoc`].
-impl<T> IoLoc<T> for T
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T
where
T: FixedRegister,
{
@@ -168,7 +170,7 @@ impl<T: FixedRegister> FixedRegisterLoc<T> {
}
}
-impl<T> IoLoc<T> for FixedRegisterLoc<T>
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T>
where
T: FixedRegister,
{
@@ -239,7 +241,7 @@ where
}
}
-impl<T, B> IoLoc<T> for RelativeRegisterLoc<T, B>
+impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B>
where
T: RelativeRegister,
B: RegisterBase<T::BaseFamily> + ?Sized,
@@ -283,7 +285,7 @@ impl<T: RegisterArray> RegisterArrayLoc<T> {
}
}
-impl<T> IoLoc<T> for RegisterArrayLoc<T>
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T>
where
T: RegisterArray,
{
@@ -370,7 +372,7 @@ where
}
}
-impl<T, B> IoLoc<T> for RelativeRegisterArrayLoc<T, B>
+impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B>
where
T: RelativeRegisterArray,
B: RegisterBase<T::BaseFamily> + ?Sized,
@@ -387,18 +389,18 @@ where
/// which to write it.
///
/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
-pub trait LocatedRegister {
+pub trait LocatedRegister<Base: ?Sized> {
/// Register value to write.
type Value: Register;
/// Full location information at which to write the value.
- type Location: IoLoc<Self::Value>;
+ type Location: IoLoc<Base, Self::Value>;
/// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write
/// operation.
fn into_io_op(self) -> (Self::Location, Self::Value);
}
-impl<T> LocatedRegister for T
+impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T
where
T: FixedRegister,
{
@@ -444,7 +446,7 @@ where
/// Io,
/// },
/// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
///
/// register! {
/// FIXED_REG(u32) @ 0x100 {
@@ -453,7 +455,7 @@ where
/// }
/// }
///
-/// # fn test(io: &Mmio<0x1000>) {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
/// let val = io.read(FIXED_REG);
///
/// // Write from an already-existing value.
@@ -557,7 +559,7 @@ where
/// Io,
/// },
/// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
///
/// // Type used to identify the base.
/// pub struct CpuCtlBase;
@@ -582,7 +584,7 @@ where
/// }
/// }
///
-/// # fn test(io: Mmio<0x1000>) {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
/// // Read the status of `Cpu0`.
/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>());
///
@@ -599,7 +601,7 @@ where
/// }
/// }
///
-/// # fn test2(io: Mmio<0x1000>) {
+/// # fn test2(io: Mmio<'_, Region<0x1000>>) {
/// // Start the aliased `CPU0`, leaving its other fields untouched.
/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true));
/// # }
@@ -636,7 +638,7 @@ where
/// Io,
/// },
/// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
/// # fn get_scratch_idx() -> usize {
/// # 0x15
/// # }
@@ -649,7 +651,7 @@ where
/// }
/// }
///
-/// # fn test(io: &Mmio<0x1000>)
+/// # fn test(io: Mmio<'_, Region<0x1000>>)
/// # -> Result<(), Error>{
/// // Read scratch register 0, i.e. I/O address `0x80`.
/// let scratch_0 = io.read(SCRATCH::at(0)).value();
@@ -722,7 +724,7 @@ where
/// Io,
/// },
/// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
/// # fn get_scratch_idx() -> usize {
/// # 0x15
/// # }
@@ -750,7 +752,7 @@ where
/// }
/// }
///
-/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
/// // Read scratch register 0 of CPU0.
/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0));
///
@@ -792,7 +794,7 @@ where
/// }
/// }
///
-/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> {
+/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status();
/// # Ok(())
/// # }
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 9512af7156df..68f4d9a3425d 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -16,6 +16,9 @@
// Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
// the unstable features in use.
//
+// Stable since Rust 1.87.0.
+#![feature(unsigned_is_multiple_of)]
+//
// Stable since Rust 1.89.0.
#![feature(generic_arg_infer)]
//
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index f783b9d9fa26..ee25ae56bb92 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -43,7 +43,6 @@ pub use self::id::{
pub use self::io::{
Bar,
ConfigSpace,
- ConfigSpaceKind,
ConfigSpaceSize,
DevresBar,
Extended,
diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs
index 6ebfbf368cd3..953e16735c6e 100644
--- a/rust/kernel/pci/io.rs
+++ b/rust/kernel/pci/io.rs
@@ -8,22 +8,21 @@ use crate::{
device,
devres::DevresLt,
io::{
- Io,
+ IoBackend,
+ IoBase,
IoCapable,
- IoKnownSize,
Mmio,
- MmioRaw, //
+ MmioBackend,
+ MmioRaw,
+ Region, //
},
prelude::*,
+ ptr::KnownSize,
types::{
CovariantForLt,
ForLt, //
}, //
};
-use core::{
- marker::PhantomData,
- ops::Deref, //
-};
/// Represents the size of a PCI configuration space.
///
@@ -50,68 +49,95 @@ impl ConfigSpaceSize {
}
}
-/// Marker type for normal (256-byte) PCI configuration space.
-pub struct Normal;
+/// Alias for normal (256-byte) PCI configuration space.
+pub type Normal = Region<256>;
-/// Marker type for extended (4096-byte) PCIe configuration space.
-pub struct Extended;
+/// Alias for extended (4096-byte) PCIe configuration space.
+pub type Extended = Region<4096>;
-/// Trait for PCI configuration space size markers.
+/// A view of PCI configuration space of a device.
+///
+/// Provides typed read and write accessors for configuration registers
+/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers.
+///
+/// The generic parameter `T` is the type of the view. The full configuration space is also a
+/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration
+/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default).
///
-/// This trait is implemented by [`Normal`] and [`Extended`] to provide
-/// compile-time knowledge of the configuration space size.
-pub trait ConfigSpaceKind {
- /// The size of this configuration space in bytes.
- const SIZE: usize;
+/// # Invariants
+///
+/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within
+/// `0..pdev.cfg_size().into_raw()`.
+pub struct ConfigSpace<'a, T: ?Sized = Extended> {
+ pub(crate) pdev: &'a Device<device::Bound>,
+ ptr: *mut T,
}
-impl ConfigSpaceKind for Normal {
- const SIZE: usize = 256;
+impl<T: ?Sized> Copy for ConfigSpace<'_, T> {}
+impl<T: ?Sized> Clone for ConfigSpace<'_, T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ *self
+ }
}
-impl ConfigSpaceKind for Extended {
- const SIZE: usize = 4096;
-}
+// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Send for ConfigSpace<'_, T> {}
-/// The PCI configuration space of a device.
-///
-/// Provides typed read and write accessors for configuration registers
-/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers.
-///
-/// The generic parameter `S` indicates the maximum size of the configuration space.
-/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for
-/// 4096-byte PCIe extended configuration space (default).
-pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> {
- pub(crate) pdev: &'a Device<device::Bound>,
- _marker: PhantomData<S>,
+// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Sync for ConfigSpace<'_, T> {}
+
+/// I/O Backend for PCI configuration space.
+pub struct ConfigSpaceBackend;
+
+impl IoBackend for ConfigSpaceBackend {
+ type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>;
+
+ #[inline]
+ fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T {
+ view.ptr
+ }
+
+ #[inline]
+ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+ view: Self::View<'a, T>,
+ ptr: *mut U,
+ ) -> Self::View<'a, U> {
+ // INVARIANT: Per safety requirement.
+ ConfigSpace {
+ pdev: view.pdev,
+ ptr,
+ }
+ }
}
/// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`.
macro_rules! impl_config_space_io_capable {
($ty:ty, $read_fn:ident, $write_fn:ident) => {
- impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> {
- unsafe fn io_read(&self, address: usize) -> $ty {
+ impl IoCapable<$ty> for ConfigSpaceBackend {
+ fn io_read(view: ConfigSpace<'_, $ty>) -> $ty {
+ // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
+ // signed offset parameter. PCI configuration space size is at most 4096 bytes,
+ // so the value always fits within `i32` without truncation or sign change.
+ let addr = view.ptr.addr() as i32;
+
let mut val: $ty = 0;
// Return value from C function is ignored in infallible accessors.
- let _ret =
- // SAFETY: By the type invariant `self.pdev` is a valid address.
- // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
- // signed offset parameter. PCI configuration space size is at most 4096 bytes,
- // so the value always fits within `i32` without truncation or sign change.
- unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) };
-
+ // SAFETY: By the type invariant `pdev` is a valid address.
+ let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) };
val
}
- unsafe fn io_write(&self, value: $ty, address: usize) {
+ fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) {
+ // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
+ // signed offset parameter. PCI configuration space size is at most 4096 bytes,
+ // so the value always fits within `i32` without truncation or sign change.
+ let addr = view.ptr.addr() as i32;
+
// Return value from C function is ignored in infallible accessors.
- let _ret =
- // SAFETY: By the type invariant `self.pdev` is a valid address.
- // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
- // signed offset parameter. PCI configuration space size is at most 4096 bytes,
- // so the value always fits within `i32` without truncation or sign change.
- unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) };
+ // SAFETY: By the type invariant `pdev` is a valid address.
+ let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) };
}
}
};
@@ -122,24 +148,16 @@ impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte);
impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word);
impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword);
-impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> {
- /// Returns the base address of the I/O region. It is always 0 for configuration space.
- #[inline]
- fn addr(&self) -> usize {
- 0
- }
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> {
+ type Backend = ConfigSpaceBackend;
+ type Target = T;
- /// Returns the maximum size of the configuration space.
#[inline]
- fn maxsize(&self) -> usize {
- self.pdev.cfg_size().into_raw()
+ fn as_view(self) -> ConfigSpace<'a, T> {
+ self
}
}
-impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> {
- const MIN_SIZE: usize = S::SIZE;
-}
-
/// A PCI BAR to perform I/O-Operations on.
///
/// I/O backend assumes that the device is little-endian and will automatically
@@ -151,7 +169,7 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> {
/// memory mapped PCI BAR and its size.
pub struct Bar<'a, const SIZE: usize = 0> {
pdev: &'a Device<device::Bound>,
- io: MmioRaw<SIZE>,
+ io: MmioRaw<crate::io::Region<SIZE>>,
num: i32,
}
@@ -204,7 +222,7 @@ impl<'a, const SIZE: usize> Bar<'a, SIZE> {
return Err(ENOMEM);
}
- let io = match MmioRaw::new(ioptr, len as usize) {
+ let io = match MmioRaw::new_region(ioptr, len as usize) {
Ok(io) => io,
Err(err) => {
// SAFETY:
@@ -264,12 +282,14 @@ impl<const SIZE: usize> Drop for Bar<'_, SIZE> {
}
}
-impl<const SIZE: usize> Deref for Bar<'_, SIZE> {
- type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> {
+ type Backend = MmioBackend;
+ type Target = crate::io::Region<SIZE>;
- fn deref(&self) -> &Self::Target {
+ #[inline]
+ fn as_view(self) -> Mmio<'a, Self::Target> {
// SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped.
- unsafe { Mmio::from_raw(&self.io) }
+ unsafe { Mmio::from_raw(self.io) }
}
}
@@ -304,23 +324,25 @@ impl Device<device::Bound> {
}
}
- /// Return an initialized normal (256-byte) config space object.
+ /// Return a view of the normal (256-byte) config space.
pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> {
+ // INVARIANT: null is aligned and the range is within config space.
ConfigSpace {
pdev: self,
- _marker: PhantomData,
+ ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()),
}
}
- /// Return an initialized extended (4096-byte) config space object.
+ /// Return a view of the extended (4096-byte) config space.
pub fn config_space_extended<'a>(&'a self) -> Result<ConfigSpace<'a, Extended>> {
if self.cfg_size() != ConfigSpaceSize::Extended {
return Err(EINVAL);
}
+ // INVARIANT: null is aligned and we just checked the `cfg_size`.
Ok(ConfigSpace {
pdev: self,
- _marker: PhantomData,
+ ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096),
})
}
}
diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs
index 3f3e529e9f58..82acb531b17b 100644
--- a/rust/kernel/ptr.rs
+++ b/rust/kernel/ptr.rs
@@ -235,11 +235,20 @@ impl_alignable_uint!(u8, u16, u32, u64, usize);
///
/// This is a generalization of [`size_of`] that works for dynamically sized types.
pub trait KnownSize {
+ /// Minimum size of this type known at compile-time.
+ const MIN_SIZE: usize;
+
+ /// Minimum alignment of this type known at compile-time.
+ const MIN_ALIGN: Alignment;
+
/// Get the size of an object of this type in bytes, with the metadata of the given pointer.
fn size(p: *const Self) -> usize;
}
impl<T> KnownSize for T {
+ const MIN_SIZE: usize = size_of::<T>();
+ const MIN_ALIGN: Alignment = Alignment::of::<T>();
+
#[inline(always)]
fn size(_: *const Self) -> usize {
size_of::<T>()
@@ -247,6 +256,9 @@ impl<T> KnownSize for T {
}
impl<T> KnownSize for [T] {
+ const MIN_SIZE: usize = 0;
+ const MIN_ALIGN: Alignment = Alignment::of::<T>();
+
#[inline(always)]
fn size(p: *const Self) -> usize {
p.len() * size_of::<T>()
diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs
index 5046b4628d0e..b629acc6d915 100644
--- a/samples/rust/rust_dma.rs
+++ b/samples/rust/rust_dma.rs
@@ -12,6 +12,11 @@ use kernel::{
Device,
DmaMask, //
},
+ io::{
+ io_project,
+ io_read,
+ Io, //
+ },
page, pci,
prelude::*,
scatterlist::{Owned, SGTable},
@@ -34,6 +39,7 @@ const TEST_VALUES: [(u32, u32); 5] = [
(0xcd, 0xef),
];
+#[derive(FromBytes, IntoBytes)]
struct MyStruct {
h: u32,
b: u32,
@@ -77,7 +83,7 @@ impl pci::Driver for DmaSampleDriver {
Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
for (i, value) in TEST_VALUES.into_iter().enumerate() {
- kernel::dma_write!(ca, [try: i], MyStruct::new(value.0, value.1));
+ io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0, value.1));
}
let size = 4 * page::PAGE_SIZE;
@@ -97,8 +103,8 @@ impl pci::Driver for DmaSampleDriver {
impl DmaSampleDriver {
fn check_dma(&self) {
for (i, value) in TEST_VALUES.into_iter().enumerate() {
- let val0 = kernel::dma_read!(self.ca, [panic: i].h);
- let val1 = kernel::dma_read!(self.ca, [panic: i].b);
+ let val0 = io_read!(self.ca, [panic: i].h);
+ let val1 = io_read!(self.ca, [panic: i].b);
assert_eq!(val0, value.0);
assert_eq!(val1, value.1);