From 1c71ddb310ad04ccec70441c7f17089514e3bab8 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Wed, 4 Sep 2024 22:43:47 +0200 Subject: [PATCH 01/16] rust: std_vendor: simplify `{ .. macro! .. }` with inner attributes It is cleaner to have a single inner attribute rather than needing several hidden lines to wrap the macro invocations. Thus simplify them. Reviewed-by: Vincenzo Palazzo Reviewed-by: Alice Ryhl Tested-by: Gary Guo Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20240904204347.168520-20-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/std_vendor.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 8b4872b48e97..8f27d447a7fc 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -73,8 +73,7 @@ /// Naive factorial implementation: /// /// ```rust -/// # #[expect(clippy::disallowed_macros)] -/// # { +/// # #![expect(clippy::disallowed_macros)] /// fn factorial(n: u32) -> u32 { /// if dbg!(n <= 1) { /// dbg!(1) @@ -84,7 +83,6 @@ /// } /// /// dbg!(factorial(4)); -/// # } /// ``` /// /// This prints to the kernel log: @@ -129,11 +127,9 @@ /// invocations. You can use a 1-tuple directly if you need one: /// /// ``` -/// # #[expect(clippy::disallowed_macros)] -/// # { +/// # #![expect(clippy::disallowed_macros)] /// assert_eq!(1, dbg!(1u32,)); // trailing comma ignored /// assert_eq!((1,), dbg!((1u32,))); // 1-tuple -/// # } /// ``` /// /// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html -- 2.51.0 From c95bbb59a9b22f9b838b15d28319185c1c884329 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sun, 15 Sep 2024 14:26:31 +0100 Subject: [PATCH 02/16] rust: enable arbitrary_self_types and remove `Receiver` The term "receiver" means that a type can be used as the type of `self`, and thus enables method call syntax `foo.bar()` instead of `Foo::bar(foo)`. Stable Rust as of today (1.81) enables a limited selection of types (primitives and types in std, e.g. `Box` and `Arc`) to be used as receivers, while custom types cannot. We want the kernel `Arc` type to have the same functionality as the Rust std `Arc`, so we use the `Receiver` trait (gated behind `receiver_trait` unstable feature) to gain the functionality. The `arbitrary_self_types` RFC [1] (tracking issue [2]) is accepted and it will allow all types that implement a new `Receiver` trait (different from today's unstable trait) to be used as receivers. This trait will be automatically implemented for all `Deref` types, which include our `Arc` type, so we no longer have to opt-in to be used as receiver. To prepare us for the change, remove the `Receiver` implementation and the associated feature. To still allow `Arc` and others to be used as method receivers, turn on `arbitrary_self_types` feature instead. This feature gate is introduced in 1.23.0. It used to enable both `Deref` types and raw pointer types to be used as receivers, but the latter is now split into a different feature gate in Rust 1.83 nightly. We do not need receivers on raw pointers so this change would not affect us and usage of `arbitrary_self_types` feature would work for all Rust versions that we support (>=1.78). Cc: Adrian Taylor Link: https://github.com/rust-lang/rfcs/pull/3519 [1] Link: https://github.com/rust-lang/rust/issues/44874 [2] Signed-off-by: Gary Guo Reviewed-by: Benno Lossin Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20240915132734.1653004-1-gary@garyguo.net Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 2 +- rust/kernel/list/arc.rs | 3 --- rust/kernel/sync/arc.rs | 6 ------ scripts/Makefile.build | 2 +- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index f329638fc5e0..3e4bd6e57bee 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -12,11 +12,11 @@ //! do so first instead of bypassing this crate. #![no_std] +#![feature(arbitrary_self_types)] #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] #![feature(lint_reasons)] #![feature(new_uninit)] -#![feature(receiver_trait)] #![feature(unsize)] // Ensure conditional compilation based on the kernel configuration works; diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index d801b9dc6291..3483d8c232c4 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -441,9 +441,6 @@ where } } -// This is to allow [`ListArc`] (and variants) to be used as the type of `self`. -impl core::ops::Receiver for ListArc where T: ListArcSafe + ?Sized {} - // This is to allow coercion from `ListArc` to `ListArc` if `T` can be converted to the // dynamically-sized type (DST) `U`. impl core::ops::CoerceUnsized> for ListArc diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 84c7cbfb096d..9325cc5a16a4 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -171,9 +171,6 @@ impl ArcInner { } } -// This is to allow [`Arc`] (and variants) to be used as the type of `self`. -impl core::ops::Receiver for Arc {} - // This is to allow coercion from `Arc` to `Arc` if `T` can be converted to the // dynamically-sized type (DST) `U`. impl, U: ?Sized> core::ops::CoerceUnsized> for Arc {} @@ -480,9 +477,6 @@ pub struct ArcBorrow<'a, T: ?Sized + 'a> { _p: PhantomData<&'a ()>, } -// This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`. -impl core::ops::Receiver for ArcBorrow<'_, T> {} - // This is to allow `ArcBorrow` to be dispatched on when `ArcBorrow` can be coerced into // `ArcBorrow`. impl, U: ?Sized> core::ops::DispatchFromDyn> diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 0a9ea56db100..64518c2f3d9c 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -248,7 +248,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := lint_reasons,new_uninit +rust_allowed_features := arbitrary_self_types,lint_reasons,new_uninit # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree -- 2.51.0 From 5ed147473458f8c20f908a03227d8f5bb3cb8f7d Mon Sep 17 00:00:00 2001 From: Filipe Xavier Date: Fri, 13 Sep 2024 07:19:56 -0300 Subject: [PATCH 03/16] rust: error: make conversion functions public Change visibility to public of functions in error.rs: from_err_ptr, from_errno, from_result and to_ptr. Additionally, remove dead_code annotations. Link: https://github.com/Rust-for-Linux/linux/issues/1105 Reviewed-by: Alice Ryhl Signed-off-by: Filipe Xavier Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/DM4PR14MB7276E6948E67B3B23D8EA847E9652@DM4PR14MB7276.namprd14.prod.outlook.com Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index a681acda87ce..2f1e4b783bfb 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -95,7 +95,7 @@ impl Error { /// /// It is a bug to pass an out-of-range `errno`. `EINVAL` would /// be returned in such a case. - pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error { + pub fn from_errno(errno: core::ffi::c_int) -> Error { if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { // TODO: Make it a `WARN_ONCE` once available. crate::pr_warn!( @@ -133,8 +133,7 @@ impl Error { } /// Returns the error encoded as a pointer. - #[expect(dead_code)] - pub(crate) fn to_ptr(self) -> *mut T { + pub fn to_ptr(self) -> *mut T { #[cfg_attr(target_pointer_width = "32", allow(clippy::useless_conversion))] // SAFETY: `self.0` is a valid error due to its invariant. unsafe { @@ -270,9 +269,7 @@ pub fn to_result(err: core::ffi::c_int) -> Result { /// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) /// } /// ``` -// TODO: Remove `dead_code` marker once an in-kernel client is available. -#[allow(dead_code)] -pub(crate) fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { +pub fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid. let const_ptr: *const core::ffi::c_void = ptr.cast(); // SAFETY: The FFI function does not deref the pointer. @@ -318,9 +315,7 @@ pub(crate) fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { /// }) /// } /// ``` -// TODO: Remove `dead_code` marker once an in-kernel client is available. -#[allow(dead_code)] -pub(crate) fn from_result(f: F) -> T +pub fn from_result(f: F) -> T where T: From, F: FnOnce() -> Result, -- 2.51.0 From 3566362dd4dc61f770e42045a92cc3f17df61df0 Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Fri, 4 Oct 2024 14:56:16 +0200 Subject: [PATCH 04/16] rust: std_vendor: update dbg macro from Rust upstream `dbg!` contains adapted code from Rust upstream. Compare the kernel code with the Rust upstream one and update missing column numbers in `dbg!` outputs. Column numbers are not copied but adjusted for the kernel's examples. Suggested-by: Miguel Ojeda Link: https://github.com/Rust-for-Linux/linux/issues/1124 Signed-off-by: Deepak Thukral Link: https://lore.kernel.org/r/20241004125616.49886-1-iapain@gmail.com [ Fixed typo and slightly reworded. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/std_vendor.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 8f27d447a7fc..279bd353687a 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -18,7 +18,7 @@ /// let a = 2; /// # #[expect(clippy::disallowed_macros)] /// let b = dbg!(a * 2) + 1; -/// // ^-- prints: [src/main.rs:2] a * 2 = 4 +/// // ^-- prints: [src/main.rs:3:9] a * 2 = 4 /// assert_eq!(b, 5); /// ``` /// @@ -67,7 +67,7 @@ /// This prints to the kernel log: /// /// ```text,ignore -/// [src/main.rs:4] n.checked_sub(4) = None +/// [src/main.rs:3:8] n.checked_sub(4) = None /// ``` /// /// Naive factorial implementation: @@ -88,15 +88,15 @@ /// This prints to the kernel log: /// /// ```text,ignore -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = true -/// [src/main.rs:4] 1 = 1 -/// [src/main.rs:5] n * factorial(n - 1) = 2 -/// [src/main.rs:5] n * factorial(n - 1) = 6 -/// [src/main.rs:5] n * factorial(n - 1) = 24 -/// [src/main.rs:11] factorial(4) = 24 +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = true +/// [src/main.rs:4:9] 1 = 1 +/// [src/main.rs:5:9] n * factorial(n - 1) = 2 +/// [src/main.rs:5:9] n * factorial(n - 1) = 6 +/// [src/main.rs:5:9] n * factorial(n - 1) = 24 +/// [src/main.rs:11:1] factorial(4) = 24 /// ``` /// /// The `dbg!(..)` macro moves the input: -- 2.51.0 From f4c2c90bb7b4ae1812dbaca15d9637eecaac2c9f Mon Sep 17 00:00:00 2001 From: Filipe Xavier Date: Thu, 26 Sep 2024 17:50:37 -0300 Subject: [PATCH 05/16] rust: lock: add trylock method support for lock backend Add a non-blocking trylock method to lock backend interface, mutex and spinlock implementations. It includes a C helper for spin_trylock. Rust Binder will use this method together with the new shrinker abstractions to avoid deadlocks in the memory shrinker. Link: https://lore.kernel.org/all/20240912-shrinker-v1-1-18b7f1253553@google.com Signed-off-by: Filipe Xavier Reviewed-by: Fiona Behrens Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Link: https://lore.kernel.org/r/BL0PR02MB4914579914884B5D7473B3D6E96A2@BL0PR02MB4914.namprd02.prod.outlook.com [ Slightly reworded. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/helpers/spinlock.c | 5 +++++ rust/kernel/sync/lock.rs | 16 ++++++++++++++++ rust/kernel/sync/lock/mutex.rs | 11 +++++++++++ rust/kernel/sync/lock/spinlock.rs | 11 +++++++++++ 4 files changed, 43 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index acc1376b833c..775ed4d549ae 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -22,3 +22,8 @@ void rust_helper_spin_unlock(spinlock_t *lock) { spin_unlock(lock); } + +int rust_helper_spin_trylock(spinlock_t *lock) +{ + return spin_trylock(lock); +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 07fcf2d8efc6..90cc5416529b 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -58,6 +58,13 @@ pub unsafe trait Backend { #[must_use] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; + /// Tries to acquire the lock. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn try_lock(ptr: *mut Self::State) -> Option; + /// Releases the lock, giving up its ownership. /// /// # Safety @@ -128,6 +135,15 @@ impl Lock { // SAFETY: The lock was just acquired. unsafe { Guard::new(self, state) } } + + /// Tries to acquire the lock. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + pub fn try_lock(&self) -> Option> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + unsafe { B::try_lock(self.state.get()).map(|state| Guard::new(self, state)) } + } } /// A lock guard. diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 30632070ee67..c4f3b6cbfe48 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -115,4 +115,15 @@ unsafe impl super::Backend for MutexBackend { // caller is the owner of the mutex. unsafe { bindings::mutex_unlock(ptr) }; } + + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::mutex_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index ea5c5bc1ce12..c900ae23db76 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -114,4 +114,15 @@ unsafe impl super::Backend for SpinLockBackend { // caller is the owner of the spinlock. unsafe { bindings::spin_unlock(ptr) } } + + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } } -- 2.51.0 From e9759c5b9ea555d09f426c70c880e9522e9b0576 Mon Sep 17 00:00:00 2001 From: Filipe Xavier Date: Sat, 5 Oct 2024 19:51:23 +0000 Subject: [PATCH 06/16] rust: error: optimize error type to use nonzero Optimize `Result<(), Error>` size by changing `Error` type to `NonZero*` for niche optimization. This reduces the space used by the `Result` type, as the `NonZero*` type enables the compiler to apply more efficient memory layout. For example, the `Result<(), Error>` changes size from 8 to 4 bytes. Link: https://github.com/Rust-for-Linux/linux/issues/1120 Signed-off-by: Filipe Xavier Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Fiona Behrens Link: https://lore.kernel.org/r/BL0PR02MB4914B9B088865CF237731207E9732@BL0PR02MB4914.namprd02.prod.outlook.com [ Removed unneeded block around `match`, added backticks in panic message and added intra-doc link. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 2f1e4b783bfb..be6509d5f4a4 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -9,6 +9,7 @@ use crate::{alloc::AllocError, str::CStr}; use alloc::alloc::LayoutError; use core::fmt; +use core::num::NonZeroI32; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -20,7 +21,11 @@ pub mod code { $( #[doc = $doc] )* - pub const $err: super::Error = super::Error(-(crate::bindings::$err as i32)); + pub const $err: super::Error = + match super::Error::try_from_errno(-(crate::bindings::$err as i32)) { + Some(err) => err, + None => panic!("Invalid errno in `declare_err!`"), + }; }; } @@ -88,7 +93,7 @@ pub mod code { /// /// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). #[derive(Clone, Copy, PartialEq, Eq)] -pub struct Error(core::ffi::c_int); +pub struct Error(NonZeroI32); impl Error { /// Creates an [`Error`] from a kernel error code. @@ -107,7 +112,20 @@ impl Error { // INVARIANT: The check above ensures the type invariant // will hold. - Error(errno) + // SAFETY: `errno` is checked above to be in a valid range. + unsafe { Error::from_errno_unchecked(errno) } + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// Returns [`None`] if `errno` is out-of-range. + const fn try_from_errno(errno: core::ffi::c_int) -> Option { + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + return None; + } + + // SAFETY: `errno` is checked above to be in a valid range. + Some(unsafe { Error::from_errno_unchecked(errno) }) } /// Creates an [`Error`] from a kernel error code. @@ -115,21 +133,22 @@ impl Error { /// # Safety /// /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). - unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { + const unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { // INVARIANT: The contract ensures the type invariant // will hold. - Error(errno) + // SAFETY: The caller guarantees `errno` is non-zero. + Error(unsafe { NonZeroI32::new_unchecked(errno) }) } /// Returns the kernel error code. pub fn to_errno(self) -> core::ffi::c_int { - self.0 + self.0.get() } #[cfg(CONFIG_BLOCK)] pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { // SAFETY: `self.0` is a valid error due to its invariant. - unsafe { bindings::errno_to_blk_status(self.0) } + unsafe { bindings::errno_to_blk_status(self.0.get()) } } /// Returns the error encoded as a pointer. @@ -137,7 +156,7 @@ impl Error { #[cfg_attr(target_pointer_width = "32", allow(clippy::useless_conversion))] // SAFETY: `self.0` is a valid error due to its invariant. unsafe { - bindings::ERR_PTR(self.0.into()) as *mut _ + bindings::ERR_PTR(self.0.get().into()) as *mut _ } } @@ -145,7 +164,7 @@ impl Error { #[cfg(not(testlib))] pub fn name(&self) -> Option<&'static CStr> { // SAFETY: Just an FFI call, there are no extra safety requirements. - let ptr = unsafe { bindings::errname(-self.0) }; + let ptr = unsafe { bindings::errname(-self.0.get()) }; if ptr.is_null() { None } else { -- 2.51.0 From ce1c54fdff7c4556b08f5b875a331d8952e8b6b7 Mon Sep 17 00:00:00 2001 From: Aliet Exposito Garcia Date: Wed, 18 Sep 2024 18:51:14 -0400 Subject: [PATCH 07/16] rust: kernel: move `FromBytes` and `AsBytes` traits to a new `transmute` module Refactor the `FromBytes` and `AsBytes` traits from `types.rs` into a new `transmute.rs` module: - Add `rust/kernel/transmute.rs` with the definitions of `FromBytes` and `AsBytes`. - Remove the same trait definitions from `rust/kernel/types.rs`. - Update `rust/kernel/uaccess.rs` to import `AsBytes` and `FromBytes` from `transmute.rs`. The traits and their implementations remain unchanged. Suggested-by: Benno Lossin Link: https://github.com/Rust-for-Linux/linux/issues/1117 Signed-off-by: Aliet Exposito Garcia Reviewed-by: Fiona Behrens Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20240918225115.2309224-2-aliet.exposito@gmail.com [ Rebased on top of the lints series and slightly reworded. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + rust/kernel/transmute.rs | 71 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/types.rs | 68 -------------------------------------- rust/kernel/uaccess.rs | 2 +- 4 files changed, 73 insertions(+), 69 deletions(-) create mode 100644 rust/kernel/transmute.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3e4bd6e57bee..dc37aef6a008 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -54,6 +54,7 @@ pub mod str; pub mod sync; pub mod task; pub mod time; +pub mod transmute; pub mod types; pub mod uaccess; pub mod workqueue; diff --git a/rust/kernel/transmute.rs b/rust/kernel/transmute.rs new file mode 100644 index 000000000000..1c7d43771a37 --- /dev/null +++ b/rust/kernel/transmute.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Traits for transmuting types. + +/// Types for which any bit pattern is valid. +/// +/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so +/// reading arbitrary bytes into something that contains a `bool` is not okay. +/// +/// It's okay for the type to have padding, as initializing those bytes has no effect. +/// +/// # Safety +/// +/// All bit-patterns must be valid for this type. This type must not have interior mutability. +pub unsafe trait FromBytes {} + +macro_rules! impl_frombytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? FromBytes for $t {})* + }; +} + +impl_frombytes! { + // SAFETY: All bit patterns are acceptable values of the types below. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + + // SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit + // patterns are also acceptable for arrays of that type. + {} [T], + {} [T; N], +} + +/// Types that can be viewed as an immutable slice of initialized bytes. +/// +/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This +/// means that it should not have any padding, as padding bytes are uninitialized. Reading +/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive +/// information on the stack to userspace. +/// +/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered +/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so +/// this is a correctness requirement, but not a safety requirement. +/// +/// # Safety +/// +/// Values of this type may not contain any uninitialized bytes. This type must not have interior +/// mutability. +pub unsafe trait AsBytes {} + +macro_rules! impl_asbytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? AsBytes for $t {})* + }; +} + +impl_asbytes! { + // SAFETY: Instances of the following types have no uninitialized portions. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + bool, + char, + str, + + // SAFETY: If individual values in an array have no uninitialized portions, then the array + // itself does not have any uninitialized portions either. + {} [T], + {} [T; N], +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 28d9e5ea3df4..085e8076f078 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -479,71 +479,3 @@ pub enum Either { /// Constructs an instance of [`Either`] containing a value of type `R`. Right(R), } - -/// Types for which any bit pattern is valid. -/// -/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so -/// reading arbitrary bytes into something that contains a `bool` is not okay. -/// -/// It's okay for the type to have padding, as initializing those bytes has no effect. -/// -/// # Safety -/// -/// All bit-patterns must be valid for this type. This type must not have interior mutability. -pub unsafe trait FromBytes {} - -macro_rules! impl_frombytes { - ($($({$($generics:tt)*})? $t:ty, )*) => { - // SAFETY: Safety comments written in the macro invocation. - $(unsafe impl$($($generics)*)? FromBytes for $t {})* - }; -} - -impl_frombytes! { - // SAFETY: All bit patterns are acceptable values of the types below. - u8, u16, u32, u64, usize, - i8, i16, i32, i64, isize, - - // SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit - // patterns are also acceptable for arrays of that type. - {} [T], - {} [T; N], -} - -/// Types that can be viewed as an immutable slice of initialized bytes. -/// -/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This -/// means that it should not have any padding, as padding bytes are uninitialized. Reading -/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive -/// information on the stack to userspace. -/// -/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered -/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so -/// this is a correctness requirement, but not a safety requirement. -/// -/// # Safety -/// -/// Values of this type may not contain any uninitialized bytes. This type must not have interior -/// mutability. -pub unsafe trait AsBytes {} - -macro_rules! impl_asbytes { - ($($({$($generics:tt)*})? $t:ty, )*) => { - // SAFETY: Safety comments written in the macro invocation. - $(unsafe impl$($($generics)*)? AsBytes for $t {})* - }; -} - -impl_asbytes! { - // SAFETY: Instances of the following types have no uninitialized portions. - u8, u16, u32, u64, usize, - i8, i16, i32, i64, isize, - bool, - char, - str, - - // SAFETY: If individual values in an array have no uninitialized portions, then the array - // itself does not have any uninitialized portions either. - {} [T], - {} [T; N], -} diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index e9347cff99ab..0af243482eac 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -9,7 +9,7 @@ use crate::{ bindings, error::Result, prelude::*, - types::{AsBytes, FromBytes}, + transmute::{AsBytes, FromBytes}, }; use alloc::vec::Vec; use core::ffi::{c_ulong, c_void}; -- 2.51.0 From b7a084ba4fbb8f416ce8d19c93a3a2bee63c9c89 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:05 +0200 Subject: [PATCH 08/16] rust: alloc: add `Allocator` trait Add a kernel specific `Allocator` trait, that in contrast to the one in Rust's core library doesn't require unstable features and supports GFP flags. Subsequent patches add the following trait implementors: `Kmalloc`, `Vmalloc` and `KVmalloc`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-2-dakr@kernel.org [ Fixed typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 101 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 1966bd407017..998779cc6976 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -11,6 +11,7 @@ pub mod vec_ext; /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; +use core::{alloc::Layout, ptr::NonNull}; /// Flags to be used when allocating memory. /// @@ -86,3 +87,103 @@ pub mod flags { /// small allocations. pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); } + +/// The kernel's [`Allocator`] trait. +/// +/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described +/// via [`Layout`]. +/// +/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on +/// an object instance. +/// +/// In order to be able to support `#[derive(SmartPointer)]` later on, we need to avoid a design +/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind +/// of `self` parameter. +/// +/// # Safety +/// +/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed. +/// +/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`] +/// function of the same type. +/// +/// - Implementers must ensure that all trait functions abide by the guarantees documented in the +/// `# Guarantees` sections. +pub unsafe trait Allocator { + /// Allocate memory based on `layout` and `flags`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`] or [`Allocator::realloc`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// . + fn alloc(layout: Layout, flags: Flags) -> Result, AllocError> { + // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a + // new memory allocation. + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } + } + + /// Re-allocate an existing memory allocation to satisfy the requested `layout`. + /// + /// If the requested size is zero, `realloc` behaves equivalent to `free`. + /// + /// If the requested size is larger than the size of the existing allocation, a successful call + /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but + /// may also be larger. + /// + /// If the requested size is smaller than the size of the existing allocation, `realloc` may or + /// may not shrink the buffer; this is implementation specific to the allocator. + /// + /// On allocation failure, the existing buffer, if any, remains valid. + /// + /// The buffer is represented as `NonNull<[u8]>`. + /// + /// # Safety + /// + /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation + /// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a + /// pointer returned by this [`Allocator`]. + /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and + /// `old_layout` is ignored. + /// - `old_layout` must match the `Layout` the allocation has been created with. + /// + /// # Guarantees + /// + /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then + /// it additionally guarantees that: + /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new + /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == + /// p[0..min(layout.size(), old_layout.size())]`. + /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError>; + + /// Free an existing memory allocation. + /// + /// # Safety + /// + /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`]; + /// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this + /// [`Allocator`]. + /// - `layout` must match the `Layout` the allocation has been created with. + /// - The memory allocation at `ptr` must never again be read from or written to. + unsafe fn free(ptr: NonNull, layout: Layout) { + // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this + // allocator. We are passing a `Layout` with the smallest possible alignment, so it is + // smaller than or equal to the alignment previously used with this allocation. + let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; + } +} -- 2.51.0 From a654a6e09644266e38ac05415ef7737d299c4497 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:06 +0200 Subject: [PATCH 09/16] rust: alloc: separate `aligned_size` from `krealloc_aligned` Separate `aligned_size` from `krealloc_aligned`. Subsequent patches implement `Allocator` derivates, such as `Kmalloc`, that require `aligned_size` and replace the original `krealloc_aligned`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-3-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/allocator.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 91216b36af69..765f72d5bc21 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -8,6 +8,17 @@ use core::ptr; struct KernelAllocator; +/// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. +fn aligned_size(new_layout: Layout) -> usize { + // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. + let layout = new_layout.pad_to_align(); + + // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` + // which together with the slab guarantees means the `krealloc` will return a properly aligned + // object (see comments in `kmalloc()` for more information). + layout.size() +} + /// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. /// /// # Safety @@ -15,13 +26,7 @@ struct KernelAllocator; /// - `ptr` can be either null or a pointer which has been allocated by this allocator. /// - `new_layout` must have a non-zero size. pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { - // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. - let layout = new_layout.pad_to_align(); - - // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` - // which together with the slab guarantees means the `krealloc` will return a properly aligned - // object (see comments in `kmalloc()` for more information). - let size = layout.size(); + let size = aligned_size(new_layout); // SAFETY: // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the -- 2.51.0 From 941e65531446c1eb5d573c5d30172117ebe96112 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:07 +0200 Subject: [PATCH 10/16] rust: alloc: rename `KernelAllocator` to `Kmalloc` Subsequent patches implement `Vmalloc` and `KVmalloc` allocators, hence align `KernelAllocator` to this naming scheme. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-4-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/allocator.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 765f72d5bc21..3b1c735ba409 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -6,7 +6,7 @@ use super::{flags::*, Flags}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr; -struct KernelAllocator; +struct Kmalloc; /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { @@ -37,7 +37,7 @@ pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: F } // SAFETY: TODO. -unsafe impl GlobalAlloc for KernelAllocator { +unsafe impl GlobalAlloc for Kmalloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety // requirement. @@ -74,7 +74,7 @@ unsafe impl GlobalAlloc for KernelAllocator { } #[global_allocator] -static ALLOCATOR: KernelAllocator = KernelAllocator; +static ALLOCATOR: Kmalloc = Kmalloc; // See . #[no_mangle] -- 2.51.0 From 8a799831fc63c988eec90d334fdd68ff5f2c7eb5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:08 +0200 Subject: [PATCH 11/16] rust: alloc: implement `ReallocFunc` `ReallocFunc` is an abstraction for the kernel's realloc derivates, such as `krealloc`, `vrealloc` and `kvrealloc`. All of the named functions share the same function signature and implement the same semantics. The `ReallocFunc` abstractions provides a generalized wrapper around those, to trivialize the implementation of `Kmalloc`, `Vmalloc` and `KVmalloc` in subsequent patches. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-5-dakr@kernel.org [ Added temporary `allow(dead_code)` for `dangling_from_layout` to clean warning in `rusttest` target as discussed in the list (but it is needed earlier, i.e. in this patch already). Added colon. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 9 +++++ rust/kernel/alloc/allocator.rs | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 998779cc6976..95d402feb6ff 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -187,3 +187,12 @@ pub unsafe trait Allocator { let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; } } + +#[allow(dead_code)] +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 3b1c735ba409..9ed122401e8a 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,10 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 //! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: use super::{flags::*, Flags}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr; +use core::ptr::NonNull; + +use crate::alloc::AllocError; +use crate::bindings; struct Kmalloc; @@ -36,6 +46,66 @@ pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: F unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } } +/// # Invariants +/// +/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. +struct ReallocFunc( + unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, +); + +#[expect(dead_code)] +impl ReallocFunc { + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + unsafe fn call( + &self, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + let size = aligned_size(layout); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; + + // SAFETY: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, flags.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } +} + // SAFETY: TODO. unsafe impl GlobalAlloc for Kmalloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { -- 2.51.0 From a87a36f0bf517dae22f3e3790b05c979070f776a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:09 +0200 Subject: [PATCH 12/16] rust: alloc: make `allocator` module public Subsequent patches implement allocators such as `Kmalloc`, `Vmalloc`, `KVmalloc`; we need them to be available outside of the kernel crate as well. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-6-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 95d402feb6ff..2203852c3712 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -4,7 +4,7 @@ #[cfg(not(test))] #[cfg(not(testlib))] -mod allocator; +pub mod allocator; pub mod box_ext; pub mod vec_ext; -- 2.51.0 From a34822d1c4c93085f635b922441a017bd7e959b0 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:10 +0200 Subject: [PATCH 13/16] rust: alloc: implement `Allocator` for `Kmalloc` Implement `Allocator` for `Kmalloc`, the kernel's default allocator, typically used for objects smaller than page size. All memory allocations made with `Kmalloc` end up in `krealloc()`. It serves as allocator for the subsequently introduced types `KBox` and `KVec`. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-7-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/allocator.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 9ed122401e8a..272c581e24a0 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -13,10 +13,16 @@ use core::alloc::{GlobalAlloc, Layout}; use core::ptr; use core::ptr::NonNull; -use crate::alloc::AllocError; +use crate::alloc::{AllocError, Allocator}; use crate::bindings; -struct Kmalloc; +/// The contiguous kernel allocator. +/// +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. +/// +/// For more details see [self]. +pub struct Kmalloc; /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { @@ -53,8 +59,10 @@ struct ReallocFunc( unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, ); -#[expect(dead_code)] impl ReallocFunc { + // INVARIANT: `krealloc` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -106,6 +114,23 @@ impl ReallocFunc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } + } +} + // SAFETY: TODO. unsafe impl GlobalAlloc for Kmalloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { -- 2.51.0 From 5a888c28e3b4ff6f54a53fca33951537d135e7f1 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:11 +0200 Subject: [PATCH 14/16] rust: alloc: add module `allocator_test` `Allocator`s, such as `Kmalloc`, will be used by e.g. `Box` and `Vec` in subsequent patches, and hence this dependency propagates throughout the whole kernel. Add the `allocator_test` module that provides an empty implementation for all `Allocator`s in the kernel, such that we don't break the `rusttest` make target in subsequent patches. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-8-dakr@kernel.org [ Added missing `_old_layout` parameter as discussed. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 9 +++++++-- rust/kernel/alloc/allocator_test.rs | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 rust/kernel/alloc/allocator_test.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 2203852c3712..b5605aab182d 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -2,12 +2,17 @@ //! Extensions to the [`alloc`] crate. -#[cfg(not(test))] -#[cfg(not(testlib))] +#[cfg(not(any(test, testlib)))] pub mod allocator; pub mod box_ext; pub mod vec_ext; +#[cfg(any(test, testlib))] +pub mod allocator_test; + +#[cfg(any(test, testlib))] +pub use self::allocator_test as allocator; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs new file mode 100644 index 000000000000..c5d325506f0c --- /dev/null +++ b/rust/kernel/alloc/allocator_test.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#![allow(missing_docs)] + +use super::{AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::ptr::NonNull; + +pub struct Kmalloc; + +unsafe impl Allocator for Kmalloc { + unsafe fn realloc( + _ptr: Option>, + _layout: Layout, + _old_layout: Layout, + _flags: Flags, + ) -> Result, AllocError> { + panic!(); + } +} -- 2.51.0 From 61c004781d6b928443052e7a6cf84b35d4f61401 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:12 +0200 Subject: [PATCH 15/16] rust: alloc: implement `Vmalloc` allocator Implement `Allocator` for `Vmalloc`, the kernel's virtually contiguous allocator, typically used for larger objects, (much) larger than page size. All memory allocations made with `Vmalloc` end up in `vrealloc()`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-9-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/helpers/helpers.c | 1 + rust/helpers/vmalloc.c | 9 +++++++ rust/kernel/alloc/allocator.rs | 37 +++++++++++++++++++++++++++++ rust/kernel/alloc/allocator_test.rs | 1 + 4 files changed, 48 insertions(+) create mode 100644 rust/helpers/vmalloc.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 30f40149f3a9..20a0c69d5cc7 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -22,5 +22,6 @@ #include "spinlock.c" #include "task.c" #include "uaccess.c" +#include "vmalloc.c" #include "wait.c" #include "workqueue.c" diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 000000000000..80d34501bbc0 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void * __must_check __realloc_size(2) +rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +{ + return vrealloc(p, size, flags); +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 272c581e24a0..ee4c828cbd58 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -15,6 +15,7 @@ use core::ptr::NonNull; use crate::alloc::{AllocError, Allocator}; use crate::bindings; +use crate::pr_warn; /// The contiguous kernel allocator. /// @@ -24,6 +25,15 @@ use crate::bindings; /// For more details see [self]. pub struct Kmalloc; +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. @@ -63,6 +73,9 @@ impl ReallocFunc { // INVARIANT: `krealloc` satisfies the type invariants. const KREALLOC: Self = Self(bindings::krealloc); + // INVARIANT: `vrealloc` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -168,6 +181,30 @@ unsafe impl GlobalAlloc for Kmalloc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + } +} + #[global_allocator] static ALLOCATOR: Kmalloc = Kmalloc; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index c5d325506f0c..32a1450c4d39 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -7,6 +7,7 @@ use core::alloc::Layout; use core::ptr::NonNull; pub struct Kmalloc; +pub type Vmalloc = Kmalloc; unsafe impl Allocator for Kmalloc { unsafe fn realloc( -- 2.51.0 From 8362c2608ba1be635ffa22a256dfcfe51c6238cc Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:13 +0200 Subject: [PATCH 16/16] rust: alloc: implement `KVmalloc` allocator Implement `Allocator` for `KVmalloc`, an `Allocator` that tries to allocate memory with `kmalloc` first and, on failure, falls back to `vmalloc`. All memory allocations made with `KVmalloc` end up in `kvrealloc_noprof()`; all frees in `kvfree()`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-10-dakr@kernel.org [ Reworded typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/helpers/slab.c | 6 +++++ rust/kernel/alloc/allocator.rs | 36 +++++++++++++++++++++++++++++ rust/kernel/alloc/allocator_test.rs | 1 + 3 files changed, 43 insertions(+) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index f043e087f9d6..a842bfbddcba 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -7,3 +7,9 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) { return krealloc(objp, new_size, flags); } + +void * __must_check __realloc_size(2) +rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +{ + return kvrealloc(p, size, flags); +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index ee4c828cbd58..f8a1f0feb25d 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -34,6 +34,15 @@ pub struct Kmalloc; /// For more details see [self]. pub struct Vmalloc; +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; + /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. @@ -76,6 +85,9 @@ impl ReallocFunc { // INVARIANT: `vrealloc` satisfies the type invariants. const VREALLOC: Self = Self(bindings::vrealloc); + // INVARIANT: `kvrealloc` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -205,6 +217,30 @@ unsafe impl Allocator for Vmalloc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + } +} + #[global_allocator] static ALLOCATOR: Kmalloc = Kmalloc; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index 32a1450c4d39..bd0cbcd93e52 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -8,6 +8,7 @@ use core::ptr::NonNull; pub struct Kmalloc; pub type Vmalloc = Kmalloc; +pub type KVmalloc = Kmalloc; unsafe impl Allocator for Kmalloc { unsafe fn realloc( -- 2.51.0