Messages in this thread Patch in this message | | | From | AllenX <> | Subject | [PATCH 2/2] rust: add infiniband mlx4 driver basic structure | Date | Sun, 1 Oct 2023 00:14:43 +0800 |
| |
add infiniband mlx4 device net driver basic structure and Rust security abstration interface
Signed-off-by: Allen Xu <xubo3006@163.com> --- rust/bindings/bindings_helper.h | 1 + rust/kernel/lib.rs | 1 + rust/kernel/mlx4.rs | 288 ++++++++++++++++++++++++++++++++ rust/kernel/workqueue.rs | 9 +- samples/rust/Kconfig | 7 + samples/rust/Makefile | 1 + samples/rust/rust_mlx4.rs | 49 ++++++ 7 files changed, 353 insertions(+), 3 deletions(-) create mode 100644 rust/kernel/mlx4.rs create mode 100644 samples/rust/rust_mlx4.rs
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index cb63710af1..c9b794a3d3 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -39,6 +39,7 @@ #include <net/udp_tunnel.h> #include <rdma/rdma_netlink.h> #include <rdma/ib_verbs.h> +#include <linux/mlx4/driver.h> /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 39e0b17778..60f83427d8 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -80,6 +80,7 @@ mod raw_list; pub mod rbtree; pub mod unsafe_list; +pub mod mlx4; pub mod rxe; #[doc(hidden)] diff --git a/rust/kernel/mlx4.rs b/rust/kernel/mlx4.rs new file mode 100644 index 0000000000..7f53e0e9e6 --- /dev/null +++ b/rust/kernel/mlx4.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infiniband mlx4 devices. +//! + +use alloc::boxed::Box; +use core::pin::Pin; +use core::{marker, ptr}; +use macros::vtable; + +use crate::bindings; +use crate::error::{code::*, Result}; +use crate::str::CStr; +use crate::workqueue::{BoxedQueue, Queue}; + +/// Infiband mlx4 device registration. +/// +pub struct Registration<T: Mlx4Operation> { + registered: bool, + #[allow(dead_code)] + name: &'static CStr, + wq: Mlx4WorkQueue, + cm_wq: CmWorkQueue, + qp_wq: QpWorkQueue, + mcg_wq: McgWorkQueue, + phantom: marker::PhantomData<T>, +} + +impl<T: Mlx4Operation> Registration<T> { + /// Creates a new [`Registration`] but does not register it yet. + /// + /// It is allowed to move. + pub fn new(name: &'static CStr) -> Self { + // INVARIANT: `registered` is `false` + Self { + registered: false, + name, + wq: Mlx4WorkQueue::new(), + cm_wq: CmWorkQueue::new(), + qp_wq: QpWorkQueue::new(), + mcg_wq: McgWorkQueue::new(), + phantom: marker::PhantomData, + } + } + + /// Registers a infiband mlx4 device. + /// + /// Returns a pinned heap-allocated representation of the registration. + pub fn new_pinned(name: &'static CStr) -> Result<Pin<Box<Self>>> { + let mut r = Pin::from(Box::try_new(Self::new(name))?); + r.as_mut().register()?; + Ok(r) + } + + // Registers a infiband mlx4 device with the rest of the kernel. + /// + /// It must be pinned because the memory block that represents the registration is + /// self-referential. + pub fn register(self: Pin<&mut Self>) -> Result { + // SAFETY: We must ensure that we never move out of `this`. + let this = unsafe { self.get_unchecked_mut() }; + if this.registered { + // Already registered. + return Err(EINVAL); + } + + match this.wq.init() { + Ok(()) => {} + Err(e) => return Err(e), + } + + match this.qp_wq.init() { + Ok(()) => {} + Err(e) => { + this.wq.clean(); + return Err(e); + } + } + + match this.cm_wq.init() { + Ok(()) => {} + Err(e) => { + this.wq.clean(); + this.qp_wq.clean(); + return Err(e); + } + } + + match this.mcg_wq.init() { + Ok(()) => {} + Err(e) => { + this.wq.clean(); + this.cm_wq.clean(); + this.qp_wq.clean(); + return Err(e); + } + } + + // SAFETY: The adapter is compatible with the mlx4 register + unsafe { + bindings::mlx4_register_interface(Mlx4OperationTable::<T>::build()); + } + + this.registered = true; + Ok(()) + } +} + +impl<T: Mlx4Operation> Drop for Registration<T> { + /// Removes the registration from the kernel if it has completed successfully before. + fn drop(&mut self) { + if self.registered { + self.mcg_wq.clean(); + self.cm_wq.clean(); + self.qp_wq.clean(); + self.wq.clean(); + } + } +} + +/// Build kernel's `struct mlx4_interface` type with mlx4 device operation. +pub struct Mlx4OperationTable<T>(marker::PhantomData<T>); + +impl<T: Mlx4Operation> Mlx4OperationTable<T> { + /// Builds an instance of [`struct mlx4_interface`]. + /// + /// # Safety + /// + /// The caller must ensure that the adapter is compatible with the way the device is registered. + pub fn build() -> *mut bindings::mlx4_interface { + return &mut bindings::mlx4_interface { + add: Some(Self::add_callback), + remove: Some(Self::remove_callback), + event: Some(Self::event_callback), + get_dev: None, + activate: None, + list: bindings::list_head { + next: ptr::null_mut(), + prev: ptr::null_mut(), + }, + // MLX4_PROT_IB_IPV6 + protocol: 0, + // MLX4_INTFF_BONDING + flags: 1, + }; + } + + unsafe extern "C" fn add_callback(_dev: *mut bindings::mlx4_dev) -> *mut core::ffi::c_void { + let _ = T::add(); + return ptr::null_mut(); + } + + unsafe extern "C" fn remove_callback( + _dev: *mut bindings::mlx4_dev, + _context: *mut core::ffi::c_void, + ) { + let _ = T::remove(); + } + + unsafe extern "C" fn event_callback( + _dev: *mut bindings::mlx4_dev, + _context: *mut core::ffi::c_void, + _event: bindings::mlx4_dev_event, + _param: core::ffi::c_ulong, + ) { + let _ = T::event(); + } +} + +/// Corresponds to the kernel's `struct mlx4_interface`. +/// +/// You implement this trait whenever you would create a `struct mlx4_interface`. +#[vtable] +pub trait Mlx4Operation { + /// Add a new mlx4 ib device. + fn add() -> Result; + /// Remove mlx4 ib device. + fn remove() -> Result; + /// Respond to specific mlx4 ib device event + fn event() -> Result; +} + +pub(crate) struct Mlx4WorkQueue { + wq: Option<BoxedQueue>, +} + +impl Mlx4WorkQueue { + pub(crate) fn new() -> Self { + Self { wq: None } + } + + pub(crate) fn init(&mut self) -> Result { + let wq_tmp = Queue::try_new(format_args!("mlx4_ib"), 655369, 1); + self.wq = match wq_tmp { + Ok(wq) => Some(wq), + Err(e) => return Err(e), + }; + + Ok(()) + } + + pub(crate) fn clean(&mut self) { + if self.wq.is_some() { + drop(self.wq.take().unwrap()); + } + } +} + +pub(crate) struct CmWorkQueue { + cm_wq: Option<BoxedQueue>, +} + +impl CmWorkQueue { + pub(crate) fn new() -> Self { + Self { cm_wq: None } + } + + pub(crate) fn init(&mut self) -> Result { + let cm_wq_tmp = Queue::try_new(format_args!("mlx4_ib_cm"), 0, 0); + self.cm_wq = match cm_wq_tmp { + Ok(cm_wq) => Some(cm_wq), + Err(e) => return Err(e), + }; + + Ok(()) + } + + pub(crate) fn clean(&mut self) { + if self.cm_wq.is_some() { + drop(self.cm_wq.take().unwrap()); + } + } +} + +pub(crate) struct McgWorkQueue { + clean_wq: Option<BoxedQueue>, +} + +impl McgWorkQueue { + pub(crate) fn new() -> Self { + Self { clean_wq: None } + } + + pub(crate) fn init(&mut self) -> Result { + let clean_wq_tmp = Queue::try_new(format_args!("mlx4_ib_mcg"), 655369, 1); + self.clean_wq = match clean_wq_tmp { + Ok(clean_wq) => Some(clean_wq), + Err(e) => return Err(e), + }; + + Ok(()) + } + + pub(crate) fn clean(&mut self) { + if self.clean_wq.is_some() { + drop(self.clean_wq.take().unwrap()); + } + } +} + +pub(crate) struct QpWorkQueue { + mlx4_ib_qp_event_wq: Option<BoxedQueue>, +} + +impl QpWorkQueue { + pub(crate) fn new() -> Self { + Self { + mlx4_ib_qp_event_wq: None, + } + } + + pub(crate) fn init(&mut self) -> Result { + let mlx4_ib_qp_event_wq_tmp = + Queue::try_new(format_args!("mlx4_ib_qp_event_wq"), 655361, 1); + self.mlx4_ib_qp_event_wq = match mlx4_ib_qp_event_wq_tmp { + Ok(mlx4_ib_qp_event_wq) => Some(mlx4_ib_qp_event_wq), + Err(e) => return Err(e), + }; + + Ok(()) + } + + pub(crate) fn clean(&mut self) { + if self.mlx4_ib_qp_event_wq.is_some() { + drop(self.mlx4_ib_qp_event_wq.take().unwrap()); + } + } +} diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index d87dfe4156..fa289af097 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -209,14 +209,14 @@ impl Queue { /// /// Callers should first consider using one of the existing ones (e.g. [`system`]) before /// deciding to create a new one. - pub fn try_new(name: fmt::Arguments<'_>) -> Result<BoxedQueue> { + pub fn try_new(name: fmt::Arguments<'_>, flags: u32, max_active: i32) -> Result<BoxedQueue> { // SAFETY: We use a format string that requires an `fmt::Arguments` pointer as the first // and only argument. let ptr = unsafe { bindings::alloc_workqueue( c_str!("%pA").as_char_ptr(), - 0, - 0, + flags, + max_active, &name as *const _ as *const core::ffi::c_void, ) }; @@ -408,6 +408,9 @@ pub struct BoxedQueue { ptr: NonNull<Queue>, } +// SAFETY: Kernel workqueues are usable from any thread. +unsafe impl Sync for BoxedQueue {} + impl BoxedQueue { /// Creates a new instance of [`BoxedQueue`]. /// diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig index 84c68786d9..07b7e2e6e2 100644 --- a/samples/rust/Kconfig +++ b/samples/rust/Kconfig @@ -170,4 +170,11 @@ config SAMPLE_RUST_RXE If unsure, say N. +config SAMPLE_RUST_MLX4 + tristate "infiniband mlx4" + help + This option builds the infiniband mlx4 driver cases for Rust. + + If unsure, say N. + endif # SAMPLES_RUST diff --git a/samples/rust/Makefile b/samples/rust/Makefile index 56ec1dcee6..c6617394db 100644 --- a/samples/rust/Makefile +++ b/samples/rust/Makefile @@ -16,5 +16,6 @@ obj-$(CONFIG_SAMPLE_RUST_ECHO_SERVER) += rust_echo_server.o obj-$(CONFIG_SAMPLE_RUST_FS) += rust_fs.o obj-$(CONFIG_SAMPLE_RUST_SELFTESTS) += rust_selftests.o obj-$(CONFIG_SAMPLE_RUST_RXE) += rust_rxe.o +obj-$(CONFIG_SAMPLE_RUST_MLX4) += rust_mlx4.o subdir-$(CONFIG_SAMPLE_RUST_HOSTPROGS) += hostprogs diff --git a/samples/rust/rust_mlx4.rs b/samples/rust/rust_mlx4.rs new file mode 100644 index 0000000000..280838071e --- /dev/null +++ b/samples/rust/rust_mlx4.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust infiniband mls4 device sample. + +use kernel::mlx4; +use kernel::prelude::*; + +module! { + type: RustMlx4, + name: "rust_mlx4", + author: "Rust for Linux Contributors", + description: "Rust infiniband mlx4 device sample", + license: "GPL", +} + +struct RustMlx4Ops; + +#[vtable] +impl mlx4::Mlx4Operation for RustMlx4Ops { + fn add() -> Result { + Ok(()) + } + fn remove() -> Result { + Ok(()) + } + fn event() -> Result { + Ok(()) + } +} + +struct RustMlx4 { + _dev: Pin<Box<mlx4::Registration<RustMlx4Ops>>>, +} + +impl kernel::Module for RustMlx4 { + fn init(name: &'static CStr, _module: &'static ThisModule) -> Result<Self> { + pr_info!("Rust infiniband mlx4 driver sample (init)\n"); + + Ok(RustMlx4 { + _dev: mlx4::Registration::new_pinned(name)?, + }) + } +} + +impl Drop for RustMlx4 { + fn drop(&mut self) { + pr_info!("Rust infiniband mlx4 driver sample (exit)\n"); + } +} -- 2.39.3
| |