--- /dev/null
- unsafe { bindings::auxiliary_set_drvdata(adev.as_raw(), data.into_foreign()) };
+// SPDX-License-Identifier: GPL-2.0
+
+//! Abstractions for the auxiliary bus.
+//!
+//! C header: [`include/linux/auxiliary_bus.h`](srctree/include/linux/auxiliary_bus.h)
+
+use crate::{
+ bindings, container_of, device,
+ device_id::RawDeviceId,
+ driver,
+ error::{to_result, Result},
+ prelude::*,
+ str::CStr,
+ types::{ForeignOwnable, Opaque},
+ ThisModule,
+};
+use core::{
+ marker::PhantomData,
+ ptr::{addr_of_mut, NonNull},
+};
+
+/// An adapter for the registration of auxiliary drivers.
+pub struct Adapter<T: Driver>(T);
+
+// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if
+// a preceding call to `register` has been successful.
+unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
+ type RegType = bindings::auxiliary_driver;
+
+ unsafe fn register(
+ adrv: &Opaque<Self::RegType>,
+ name: &'static CStr,
+ module: &'static ThisModule,
+ ) -> Result {
+ // SAFETY: It's safe to set the fields of `struct auxiliary_driver` on initialization.
+ unsafe {
+ (*adrv.get()).name = name.as_char_ptr();
+ (*adrv.get()).probe = Some(Self::probe_callback);
+ (*adrv.get()).remove = Some(Self::remove_callback);
+ (*adrv.get()).id_table = T::ID_TABLE.as_ptr();
+ }
+
+ // SAFETY: `adrv` is guaranteed to be a valid `RegType`.
+ to_result(unsafe {
+ bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr())
+ })
+ }
+
+ unsafe fn unregister(adrv: &Opaque<Self::RegType>) {
+ // SAFETY: `adrv` is guaranteed to be a valid `RegType`.
+ unsafe { bindings::auxiliary_driver_unregister(adrv.get()) }
+ }
+}
+
+impl<T: Driver + 'static> Adapter<T> {
+ extern "C" fn probe_callback(
+ adev: *mut bindings::auxiliary_device,
+ id: *const bindings::auxiliary_device_id,
+ ) -> kernel::ffi::c_int {
+ // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a
+ // `struct auxiliary_device`.
+ //
+ // INVARIANT: `adev` is valid for the duration of `probe_callback()`.
+ let adev = unsafe { &*adev.cast::<Device<device::Core>>() };
+
+ // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id`
+ // and does not add additional invariants, so it's safe to transmute.
+ let id = unsafe { &*id.cast::<DeviceId>() };
+ let info = T::ID_TABLE.info(id.index());
+
+ match T::probe(adev, info) {
+ Ok(data) => {
+ // Let the `struct auxiliary_device` own a reference of the driver's private data.
+ // SAFETY: By the type invariant `adev.as_raw` returns a valid pointer to a
+ // `struct auxiliary_device`.
- drop(unsafe { KBox::<T>::from_foreign(ptr) });
++ unsafe {
++ bindings::auxiliary_set_drvdata(adev.as_raw(), data.into_foreign().cast())
++ };
+ }
+ Err(err) => return Error::to_errno(err),
+ }
+
+ 0
+ }
+
+ extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) {
+ // SAFETY: The auxiliary bus only ever calls the remove callback with a valid pointer to a
+ // `struct auxiliary_device`.
+ let ptr = unsafe { bindings::auxiliary_get_drvdata(adev) };
+
+ // SAFETY: `remove_callback` is only ever called after a successful call to
+ // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized
+ // `KBox<T>` pointer created through `KBox::into_foreign`.
++ drop(unsafe { KBox::<T>::from_foreign(ptr.cast()) });
+ }
+}
+
+/// Declares a kernel module that exposes a single auxiliary driver.
+#[macro_export]
+macro_rules! module_auxiliary_driver {
+ ($($f:tt)*) => {
+ $crate::module_driver!(<T>, $crate::auxiliary::Adapter<T>, { $($f)* });
+ };
+}
+
+/// Abstraction for `bindings::auxiliary_device_id`.
+#[repr(transparent)]
+#[derive(Clone, Copy)]
+pub struct DeviceId(bindings::auxiliary_device_id);
+
+impl DeviceId {
+ /// Create a new [`DeviceId`] from name.
+ pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
+ let name = name.as_bytes_with_nul();
+ let modname = modname.as_bytes_with_nul();
+
+ // TODO: Replace with `bindings::auxiliary_device_id::default()` once stabilized for
+ // `const`.
+ //
+ // SAFETY: FFI type is valid to be zero-initialized.
+ let mut id: bindings::auxiliary_device_id = unsafe { core::mem::zeroed() };
+
+ let mut i = 0;
+ while i < modname.len() {
+ id.name[i] = modname[i];
+ i += 1;
+ }
+
+ // Reuse the space of the NULL terminator.
+ id.name[i - 1] = b'.';
+
+ let mut j = 0;
+ while j < name.len() {
+ id.name[i] = name[j];
+ i += 1;
+ j += 1;
+ }
+
+ Self(id)
+ }
+}
+
+// SAFETY:
+// * `DeviceId` is a `#[repr(transparent)`] wrapper of `auxiliary_device_id` and does not add
+// additional invariants, so it's safe to transmute to `RawType`.
+// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field.
+unsafe impl RawDeviceId for DeviceId {
+ type RawType = bindings::auxiliary_device_id;
+
+ const DRIVER_DATA_OFFSET: usize =
+ core::mem::offset_of!(bindings::auxiliary_device_id, driver_data);
+
+ fn index(&self) -> usize {
+ self.0.driver_data
+ }
+}
+
+/// IdTable type for auxiliary drivers.
+pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
+
+/// Create a auxiliary `IdTable` with its alias for modpost.
+#[macro_export]
+macro_rules! auxiliary_device_table {
+ ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
+ const $table_name: $crate::device_id::IdArray<
+ $crate::auxiliary::DeviceId,
+ $id_info_type,
+ { $table_data.len() },
+ > = $crate::device_id::IdArray::new($table_data);
+
+ $crate::module_device_table!("auxiliary", $module_table_name, $table_name);
+ };
+}
+
+/// The auxiliary driver trait.
+///
+/// Drivers must implement this trait in order to get an auxiliary driver registered.
+pub trait Driver {
+ /// The type holding information about each device id supported by the driver.
+ ///
+ /// TODO: Use associated_type_defaults once stabilized:
+ ///
+ /// type IdInfo: 'static = ();
+ type IdInfo: 'static;
+
+ /// The table of device ids supported by the driver.
+ const ID_TABLE: IdTable<Self::IdInfo>;
+
+ /// Auxiliary driver probe.
+ ///
+ /// Called when an auxiliary device is matches a corresponding driver.
+ fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>;
+}
+
+/// The auxiliary device representation.
+///
+/// This structure represents the Rust abstraction for a C `struct auxiliary_device`. The
+/// implementation abstracts the usage of an already existing C `struct auxiliary_device` within
+/// Rust code that we get passed from the C side.
+///
+/// # Invariants
+///
+/// A [`Device`] instance represents a valid `struct auxiliary_device` created by the C portion of
+/// the kernel.
+#[repr(transparent)]
+pub struct Device<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::auxiliary_device>,
+ PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Device<Ctx> {
+ fn as_raw(&self) -> *mut bindings::auxiliary_device {
+ self.0.get()
+ }
+
+ /// Returns the auxiliary device' id.
+ pub fn id(&self) -> u32 {
+ // SAFETY: By the type invariant `self.as_raw()` is a valid pointer to a
+ // `struct auxiliary_device`.
+ unsafe { (*self.as_raw()).id }
+ }
+
+ /// Returns a reference to the parent [`device::Device`], if any.
+ pub fn parent(&self) -> Option<&device::Device> {
+ let ptr: *const Self = self;
+ // CAST: `Device<Ctx: DeviceContext>` types are transparent to each other.
+ let ptr: *const Device = ptr.cast();
+ // SAFETY: `ptr` was derived from `&self`.
+ let this = unsafe { &*ptr };
+
+ this.as_ref().parent()
+ }
+}
+
+impl Device {
+ extern "C" fn release(dev: *mut bindings::device) {
+ // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device`
+ // embedded in `struct auxiliary_device`.
+ let adev = unsafe { container_of!(dev, bindings::auxiliary_device, dev) }.cast_mut();
+
+ // SAFETY: `adev` points to the memory that has been allocated in `Registration::new`, via
+ // `KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)`.
+ let _ = unsafe { KBox::<Opaque<bindings::auxiliary_device>>::from_raw(adev.cast()) };
+ }
+}
+
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
+// SAFETY: Instances of `Device` are always reference-counted.
+unsafe impl crate::types::AlwaysRefCounted for Device {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::get_device(self.as_ref().as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // CAST: `Self` a transparent wrapper of `bindings::auxiliary_device`.
+ let adev: *mut bindings::auxiliary_device = obj.cast().as_ptr();
+
+ // SAFETY: By the type invariant of `Self`, `adev` is a pointer to a valid
+ // `struct auxiliary_device`.
+ let dev = unsafe { addr_of_mut!((*adev).dev) };
+
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::put_device(dev) }
+ }
+}
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct auxiliary_device`.
+ let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::as_ref(dev) }
+ }
+}
+
+// SAFETY: A `Device` is always reference-counted and can be released from any thread.
+unsafe impl Send for Device {}
+
+// SAFETY: `Device` can be shared among threads because all methods of `Device`
+// (i.e. `Device<Normal>) are thread safe.
+unsafe impl Sync for Device {}
+
+/// The registration of an auxiliary device.
+///
+/// This type represents the registration of a [`struct auxiliary_device`]. When an instance of this
+/// type is dropped, its respective auxiliary device will be unregistered from the system.
+///
+/// # Invariants
+///
+/// `self.0` always holds a valid pointer to an initialized and registered
+/// [`struct auxiliary_device`].
+pub struct Registration(NonNull<bindings::auxiliary_device>);
+
+impl Registration {
+ /// Create and register a new auxiliary device.
+ pub fn new(parent: &device::Device, name: &CStr, id: u32, modname: &CStr) -> Result<Self> {
+ let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?;
+ let adev = boxed.get();
+
+ // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization.
+ unsafe {
+ (*adev).dev.parent = parent.as_raw();
+ (*adev).dev.release = Some(Device::release);
+ (*adev).name = name.as_char_ptr();
+ (*adev).id = id;
+ }
+
+ // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
+ // which has not been initialized yet.
+ unsafe { bindings::auxiliary_device_init(adev) };
+
+ // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be freed
+ // by `Device::release` when the last reference to the `struct auxiliary_device` is dropped.
+ let _ = KBox::into_raw(boxed);
+
+ // SAFETY:
+ // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which has
+ // been initialialized,
+ // - `modname.as_char_ptr()` is a NULL terminated string.
+ let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) };
+ if ret != 0 {
+ // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
+ // which has been initialialized.
+ unsafe { bindings::auxiliary_device_uninit(adev) };
+
+ return Err(Error::from_errno(ret));
+ }
+
+ // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated successfully.
+ //
+ // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is called,
+ // which happens in `Self::drop()`.
+ Ok(Self(unsafe { NonNull::new_unchecked(adev) }))
+ }
+}
+
+impl Drop for Registration {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+ // `struct auxiliary_device`.
+ unsafe { bindings::auxiliary_device_delete(self.0.as_ptr()) };
+
+ // This drops the reference we acquired through `auxiliary_device_init()`.
+ //
+ // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+ // `struct auxiliary_device`.
+ unsafe { bindings::auxiliary_device_uninit(self.0.as_ptr()) };
+ }
+}
+
+// SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread.
+unsafe impl Send for Registration {}
+
+// SAFETY: `Registration` does not expose any methods or fields that need synchronization.
+unsafe impl Sync for Registration {}
--- /dev/null
- Some(unsafe { T::borrow(self.as_ref().driver_data) })
+// SPDX-License-Identifier: GPL-2.0
+
+//! CPU frequency scaling.
+//!
+//! This module provides rust abstractions for interacting with the cpufreq subsystem.
+//!
+//! C header: [`include/linux/cpufreq.h`](srctree/include/linux/cpufreq.h)
+//!
+//! Reference: <https://docs.kernel.org/admin-guide/pm/cpufreq.html>
+
+use crate::{
+ clk::{Clk, Hertz},
+ cpumask,
+ device::{Bound, Device},
+ devres::Devres,
+ error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR},
+ ffi::{c_char, c_ulong},
+ prelude::*,
+ types::ForeignOwnable,
+ types::Opaque,
+};
+
+use core::{
+ cell::UnsafeCell,
+ marker::PhantomData,
+ mem::MaybeUninit,
+ ops::{Deref, DerefMut},
+ pin::Pin,
+ ptr,
+};
+
+use macros::vtable;
+
+/// Maximum length of CPU frequency driver's name.
+const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize;
+
+/// Default transition latency value in nanoseconds.
+pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32;
+
+/// CPU frequency driver flags.
+pub mod flags {
+ /// Driver needs to update internal limits even if frequency remains unchanged.
+ pub const NEED_UPDATE_LIMITS: u16 = 1 << 0;
+
+ /// Platform where constants like `loops_per_jiffy` are unaffected by frequency changes.
+ pub const CONST_LOOPS: u16 = 1 << 1;
+
+ /// Register driver as a thermal cooling device automatically.
+ pub const IS_COOLING_DEV: u16 = 1 << 2;
+
+ /// Supports multiple clock domains with per-policy governors in `cpu/cpuN/cpufreq/`.
+ pub const HAVE_GOVERNOR_PER_POLICY: u16 = 1 << 3;
+
+ /// Allows post-change notifications outside of the `target()` routine.
+ pub const ASYNC_NOTIFICATION: u16 = 1 << 4;
+
+ /// Ensure CPU starts at a valid frequency from the driver's freq-table.
+ pub const NEED_INITIAL_FREQ_CHECK: u16 = 1 << 5;
+
+ /// Disallow governors with `dynamic_switching` capability.
+ pub const NO_AUTO_DYNAMIC_SWITCHING: u16 = 1 << 6;
+}
+
+/// Relations from the C code.
+const CPUFREQ_RELATION_L: u32 = 0;
+const CPUFREQ_RELATION_H: u32 = 1;
+const CPUFREQ_RELATION_C: u32 = 2;
+
+/// Can be used with any of the above values.
+const CPUFREQ_RELATION_E: u32 = 1 << 2;
+
+/// CPU frequency selection relations.
+///
+/// CPU frequency selection relations, each optionally marked as "efficient".
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum Relation {
+ /// Select the lowest frequency at or above target.
+ Low(bool),
+ /// Select the highest frequency below or at target.
+ High(bool),
+ /// Select the closest frequency to the target.
+ Close(bool),
+}
+
+impl Relation {
+ // Construct from a C-compatible `u32` value.
+ fn new(val: u32) -> Result<Self> {
+ let efficient = val & CPUFREQ_RELATION_E != 0;
+
+ Ok(match val & !CPUFREQ_RELATION_E {
+ CPUFREQ_RELATION_L => Self::Low(efficient),
+ CPUFREQ_RELATION_H => Self::High(efficient),
+ CPUFREQ_RELATION_C => Self::Close(efficient),
+ _ => return Err(EINVAL),
+ })
+ }
+}
+
+impl From<Relation> for u32 {
+ // Convert to a C-compatible `u32` value.
+ fn from(rel: Relation) -> Self {
+ let (mut val, efficient) = match rel {
+ Relation::Low(e) => (CPUFREQ_RELATION_L, e),
+ Relation::High(e) => (CPUFREQ_RELATION_H, e),
+ Relation::Close(e) => (CPUFREQ_RELATION_C, e),
+ };
+
+ if efficient {
+ val |= CPUFREQ_RELATION_E;
+ }
+
+ val
+ }
+}
+
+/// Policy data.
+///
+/// Rust abstraction for the C `struct cpufreq_policy_data`.
+///
+/// # Invariants
+///
+/// A [`PolicyData`] instance always corresponds to a valid C `struct cpufreq_policy_data`.
+///
+/// The callers must ensure that the `struct cpufreq_policy_data` is valid for access and remains
+/// valid for the lifetime of the returned reference.
+#[repr(transparent)]
+pub struct PolicyData(Opaque<bindings::cpufreq_policy_data>);
+
+impl PolicyData {
+ /// Creates a mutable reference to an existing `struct cpufreq_policy_data` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy_data) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Returns a raw pointer to the underlying C `cpufreq_policy_data`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::cpufreq_policy_data {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ /// Wrapper for `cpufreq_generic_frequency_table_verify`.
+ #[inline]
+ pub fn generic_verify(&self) -> Result<()> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ to_result(unsafe { bindings::cpufreq_generic_frequency_table_verify(self.as_raw()) })
+ }
+}
+
+/// The frequency table index.
+///
+/// Represents index with a frequency table.
+///
+/// # Invariants
+///
+/// The index must correspond to a valid entry in the [`Table`] it is used for.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct TableIndex(usize);
+
+impl TableIndex {
+ /// Creates an instance of [`TableIndex`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `index` correspond to a valid entry in the [`Table`] it is used
+ /// for.
+ pub unsafe fn new(index: usize) -> Self {
+ // INVARIANT: The caller ensures that `index` correspond to a valid entry in the [`Table`].
+ Self(index)
+ }
+}
+
+impl From<TableIndex> for usize {
+ #[inline]
+ fn from(index: TableIndex) -> Self {
+ index.0
+ }
+}
+
+/// CPU frequency table.
+///
+/// Rust abstraction for the C `struct cpufreq_frequency_table`.
+///
+/// # Invariants
+///
+/// A [`Table`] instance always corresponds to a valid C `struct cpufreq_frequency_table`.
+///
+/// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and
+/// remains valid for the lifetime of the returned reference.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to read a frequency value from [`Table`].
+///
+/// ```
+/// use kernel::cpufreq::{Policy, TableIndex};
+///
+/// fn show_freq(policy: &Policy) {
+/// let table = policy.freq_table().unwrap();
+///
+/// // SAFETY: Index is a valid entry in the table.
+/// let index = unsafe { TableIndex::new(0) };
+///
+/// pr_info!("The frequency at index 0 is: {:?}\n", table.freq(index).unwrap());
+/// pr_info!("The flags at index 0 is: {}\n", table.flags(index));
+/// pr_info!("The data at index 0 is: {}\n", table.data(index));
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Table(Opaque<bindings::cpufreq_frequency_table>);
+
+impl Table {
+ /// Creates a reference to an existing C `struct cpufreq_frequency_table` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_frequency_table) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Returns the raw mutable pointer to the C `struct cpufreq_frequency_table`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::cpufreq_frequency_table {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ /// Returns frequency at `index` in the [`Table`].
+ #[inline]
+ pub fn freq(&self, index: TableIndex) -> Result<Hertz> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ Ok(Hertz::from_khz(unsafe {
+ (*self.as_raw().add(index.into())).frequency.try_into()?
+ }))
+ }
+
+ /// Returns flags at `index` in the [`Table`].
+ #[inline]
+ pub fn flags(&self, index: TableIndex) -> u32 {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ unsafe { (*self.as_raw().add(index.into())).flags }
+ }
+
+ /// Returns data at `index` in the [`Table`].
+ #[inline]
+ pub fn data(&self, index: TableIndex) -> u32 {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ unsafe { (*self.as_raw().add(index.into())).driver_data }
+ }
+}
+
+/// CPU frequency table owned and pinned in memory, created from a [`TableBuilder`].
+pub struct TableBox {
+ entries: Pin<KVec<bindings::cpufreq_frequency_table>>,
+}
+
+impl TableBox {
+ /// Constructs a new [`TableBox`] from a [`KVec`] of entries.
+ ///
+ /// # Errors
+ ///
+ /// Returns `EINVAL` if the entries list is empty.
+ #[inline]
+ fn new(entries: KVec<bindings::cpufreq_frequency_table>) -> Result<Self> {
+ if entries.is_empty() {
+ return Err(EINVAL);
+ }
+
+ Ok(Self {
+ // Pin the entries to memory, since we are passing its pointer to the C code.
+ entries: Pin::new(entries),
+ })
+ }
+
+ /// Returns a raw pointer to the underlying C `cpufreq_frequency_table`.
+ #[inline]
+ fn as_raw(&self) -> *const bindings::cpufreq_frequency_table {
+ // The pointer is valid until the table gets dropped.
+ self.entries.as_ptr()
+ }
+}
+
+impl Deref for TableBox {
+ type Target = Table;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: The caller owns TableBox, it is safe to deref.
+ unsafe { Self::Target::from_raw(self.as_raw()) }
+ }
+}
+
+/// CPU frequency table builder.
+///
+/// This is used by the CPU frequency drivers to build a frequency table dynamically.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create a CPU frequency table.
+///
+/// ```
+/// use kernel::cpufreq::{TableBuilder, TableIndex};
+/// use kernel::clk::Hertz;
+///
+/// let mut builder = TableBuilder::new();
+///
+/// // Adds few entries to the table.
+/// builder.add(Hertz::from_mhz(700), 0, 1).unwrap();
+/// builder.add(Hertz::from_mhz(800), 2, 3).unwrap();
+/// builder.add(Hertz::from_mhz(900), 4, 5).unwrap();
+/// builder.add(Hertz::from_ghz(1), 6, 7).unwrap();
+///
+/// let table = builder.to_table().unwrap();
+///
+/// // SAFETY: Index values correspond to valid entries in the table.
+/// let (index0, index2) = unsafe { (TableIndex::new(0), TableIndex::new(2)) };
+///
+/// assert_eq!(table.freq(index0), Ok(Hertz::from_mhz(700)));
+/// assert_eq!(table.flags(index0), 0);
+/// assert_eq!(table.data(index0), 1);
+///
+/// assert_eq!(table.freq(index2), Ok(Hertz::from_mhz(900)));
+/// assert_eq!(table.flags(index2), 4);
+/// assert_eq!(table.data(index2), 5);
+/// ```
+#[derive(Default)]
+#[repr(transparent)]
+pub struct TableBuilder {
+ entries: KVec<bindings::cpufreq_frequency_table>,
+}
+
+impl TableBuilder {
+ /// Creates a new instance of [`TableBuilder`].
+ #[inline]
+ pub fn new() -> Self {
+ Self {
+ entries: KVec::new(),
+ }
+ }
+
+ /// Adds a new entry to the table.
+ pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result<()> {
+ // Adds the new entry at the end of the vector.
+ Ok(self.entries.push(
+ bindings::cpufreq_frequency_table {
+ flags,
+ driver_data,
+ frequency: freq.as_khz() as u32,
+ },
+ GFP_KERNEL,
+ )?)
+ }
+
+ /// Consumes the [`TableBuilder`] and returns [`TableBox`].
+ pub fn to_table(mut self) -> Result<TableBox> {
+ // Add last entry to the table.
+ self.add(Hertz(c_ulong::MAX), 0, 0)?;
+
+ TableBox::new(self.entries)
+ }
+}
+
+/// CPU frequency policy.
+///
+/// Rust abstraction for the C `struct cpufreq_policy`.
+///
+/// # Invariants
+///
+/// A [`Policy`] instance always corresponds to a valid C `struct cpufreq_policy`.
+///
+/// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid
+/// for the lifetime of the returned reference.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create a CPU frequency table.
+///
+/// ```
+/// use kernel::cpufreq::{ETERNAL_LATENCY_NS, Policy};
+///
+/// fn update_policy(policy: &mut Policy) {
+/// policy
+/// .set_dvfs_possible_from_any_cpu(true)
+/// .set_fast_switch_possible(true)
+/// .set_transition_latency_ns(ETERNAL_LATENCY_NS);
+///
+/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur()));
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Policy(Opaque<bindings::cpufreq_policy>);
+
+impl Policy {
+ /// Creates a reference to an existing `struct cpufreq_policy` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_policy) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Creates a mutable reference to an existing `struct cpufreq_policy` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Returns a raw mutable pointer to the C `struct cpufreq_policy`.
+ #[inline]
+ fn as_raw(&self) -> *mut bindings::cpufreq_policy {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ #[inline]
+ fn as_ref(&self) -> &bindings::cpufreq_policy {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { &*self.as_raw() }
+ }
+
+ #[inline]
+ fn as_mut_ref(&mut self) -> &mut bindings::cpufreq_policy {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { &mut *self.as_raw() }
+ }
+
+ /// Returns the primary CPU for the [`Policy`].
+ #[inline]
+ pub fn cpu(&self) -> u32 {
+ self.as_ref().cpu
+ }
+
+ /// Returns the minimum frequency for the [`Policy`].
+ #[inline]
+ pub fn min(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().min as usize)
+ }
+
+ /// Set the minimum frequency for the [`Policy`].
+ #[inline]
+ pub fn set_min(&mut self, min: Hertz) -> &mut Self {
+ self.as_mut_ref().min = min.as_khz() as u32;
+ self
+ }
+
+ /// Returns the maximum frequency for the [`Policy`].
+ #[inline]
+ pub fn max(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().max as usize)
+ }
+
+ /// Set the maximum frequency for the [`Policy`].
+ #[inline]
+ pub fn set_max(&mut self, max: Hertz) -> &mut Self {
+ self.as_mut_ref().max = max.as_khz() as u32;
+ self
+ }
+
+ /// Returns the current frequency for the [`Policy`].
+ #[inline]
+ pub fn cur(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().cur as usize)
+ }
+
+ /// Returns the suspend frequency for the [`Policy`].
+ #[inline]
+ pub fn suspend_freq(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().suspend_freq as usize)
+ }
+
+ /// Sets the suspend frequency for the [`Policy`].
+ #[inline]
+ pub fn set_suspend_freq(&mut self, freq: Hertz) -> &mut Self {
+ self.as_mut_ref().suspend_freq = freq.as_khz() as u32;
+ self
+ }
+
+ /// Provides a wrapper to the generic suspend routine.
+ #[inline]
+ pub fn generic_suspend(&mut self) -> Result<()> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ to_result(unsafe { bindings::cpufreq_generic_suspend(self.as_mut_ref()) })
+ }
+
+ /// Provides a wrapper to the generic get routine.
+ #[inline]
+ pub fn generic_get(&self) -> Result<u32> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ Ok(unsafe { bindings::cpufreq_generic_get(self.cpu()) })
+ }
+
+ /// Provides a wrapper to the register with energy model using the OPP core.
+ #[cfg(CONFIG_PM_OPP)]
+ #[inline]
+ pub fn register_em_opp(&mut self) {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { bindings::cpufreq_register_em_with_opp(self.as_mut_ref()) };
+ }
+
+ /// Gets [`cpumask::Cpumask`] for a cpufreq [`Policy`].
+ #[inline]
+ pub fn cpus(&mut self) -> &mut cpumask::Cpumask {
+ // SAFETY: The pointer to `cpus` is valid for writing and remains valid for the lifetime of
+ // the returned reference.
+ unsafe { cpumask::CpumaskVar::as_mut_ref(&mut self.as_mut_ref().cpus) }
+ }
+
+ /// Sets clock for the [`Policy`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must guarantee that the returned [`Clk`] is not dropped while it is getting used
+ /// by the C code.
+ pub unsafe fn set_clk(&mut self, dev: &Device, name: Option<&CStr>) -> Result<Clk> {
+ let clk = Clk::get(dev, name)?;
+ self.as_mut_ref().clk = clk.as_raw();
+ Ok(clk)
+ }
+
+ /// Allows / disallows frequency switching code to run on any CPU.
+ #[inline]
+ pub fn set_dvfs_possible_from_any_cpu(&mut self, val: bool) -> &mut Self {
+ self.as_mut_ref().dvfs_possible_from_any_cpu = val;
+ self
+ }
+
+ /// Returns if fast switching of frequencies is possible or not.
+ #[inline]
+ pub fn fast_switch_possible(&self) -> bool {
+ self.as_ref().fast_switch_possible
+ }
+
+ /// Enables / disables fast frequency switching.
+ #[inline]
+ pub fn set_fast_switch_possible(&mut self, val: bool) -> &mut Self {
+ self.as_mut_ref().fast_switch_possible = val;
+ self
+ }
+
+ /// Sets transition latency (in nanoseconds) for the [`Policy`].
+ #[inline]
+ pub fn set_transition_latency_ns(&mut self, latency_ns: u32) -> &mut Self {
+ self.as_mut_ref().cpuinfo.transition_latency = latency_ns;
+ self
+ }
+
+ /// Sets cpuinfo `min_freq`.
+ #[inline]
+ pub fn set_cpuinfo_min_freq(&mut self, min_freq: Hertz) -> &mut Self {
+ self.as_mut_ref().cpuinfo.min_freq = min_freq.as_khz() as u32;
+ self
+ }
+
+ /// Sets cpuinfo `max_freq`.
+ #[inline]
+ pub fn set_cpuinfo_max_freq(&mut self, max_freq: Hertz) -> &mut Self {
+ self.as_mut_ref().cpuinfo.max_freq = max_freq.as_khz() as u32;
+ self
+ }
+
+ /// Set `transition_delay_us`, i.e. the minimum time between successive frequency change
+ /// requests.
+ #[inline]
+ pub fn set_transition_delay_us(&mut self, transition_delay_us: u32) -> &mut Self {
+ self.as_mut_ref().transition_delay_us = transition_delay_us;
+ self
+ }
+
+ /// Returns reference to the CPU frequency [`Table`] for the [`Policy`].
+ pub fn freq_table(&self) -> Result<&Table> {
+ if self.as_ref().freq_table.is_null() {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: The `freq_table` is guaranteed to be valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ Ok(unsafe { Table::from_raw(self.as_ref().freq_table) })
+ }
+
+ /// Sets the CPU frequency [`Table`] for the [`Policy`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must guarantee that the [`Table`] is not dropped while it is getting used by the
+ /// C code.
+ #[inline]
+ pub unsafe fn set_freq_table(&mut self, table: &Table) -> &mut Self {
+ self.as_mut_ref().freq_table = table.as_raw();
+ self
+ }
+
+ /// Returns the [`Policy`]'s private data.
+ pub fn data<T: ForeignOwnable>(&mut self) -> Option<<T>::Borrowed<'_>> {
+ if self.as_ref().driver_data.is_null() {
+ None
+ } else {
+ // SAFETY: The data is earlier set from [`set_data`].
- unsafe { <T as ForeignOwnable>::from_foreign(self.as_ref().driver_data) },
++ Some(unsafe { T::borrow(self.as_ref().driver_data.cast()) })
+ }
+ }
+
+ /// Sets the private data of the [`Policy`] using a foreign-ownable wrapper.
+ ///
+ /// # Errors
+ ///
+ /// Returns `EBUSY` if private data is already set.
+ fn set_data<T: ForeignOwnable>(&mut self, data: T) -> Result<()> {
+ if self.as_ref().driver_data.is_null() {
+ // Transfer the ownership of the data to the foreign interface.
+ self.as_mut_ref().driver_data = <T as ForeignOwnable>::into_foreign(data) as _;
+ Ok(())
+ } else {
+ Err(EBUSY)
+ }
+ }
+
+ /// Clears and returns ownership of the private data.
+ fn clear_data<T: ForeignOwnable>(&mut self) -> Option<T> {
+ if self.as_ref().driver_data.is_null() {
+ None
+ } else {
+ let data = Some(
+ // SAFETY: The data is earlier set by us from [`set_data`]. It is safe to take
+ // back the ownership of the data from the foreign interface.
++ unsafe { <T as ForeignOwnable>::from_foreign(self.as_ref().driver_data.cast()) },
+ );
+ self.as_mut_ref().driver_data = ptr::null_mut();
+ data
+ }
+ }
+}
+
+/// CPU frequency policy created from a CPU number.
+///
+/// This struct represents the CPU frequency policy obtained for a specific CPU, providing safe
+/// access to the underlying `cpufreq_policy` and ensuring proper cleanup when the `PolicyCpu` is
+/// dropped.
+struct PolicyCpu<'a>(&'a mut Policy);
+
+impl<'a> PolicyCpu<'a> {
+ fn from_cpu(cpu: u32) -> Result<Self> {
+ // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU.
+ let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(cpu) })?;
+
+ Ok(Self(
+ // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of
+ // the returned reference.
+ unsafe { Policy::from_raw_mut(ptr) },
+ ))
+ }
+}
+
+impl<'a> Deref for PolicyCpu<'a> {
+ type Target = Policy;
+
+ fn deref(&self) -> &Self::Target {
+ self.0
+ }
+}
+
+impl<'a> DerefMut for PolicyCpu<'a> {
+ fn deref_mut(&mut self) -> &mut Policy {
+ self.0
+ }
+}
+
+impl<'a> Drop for PolicyCpu<'a> {
+ fn drop(&mut self) {
+ // SAFETY: The underlying pointer is guaranteed to be valid for the lifetime of `self`.
+ unsafe { bindings::cpufreq_cpu_put(self.0.as_raw()) };
+ }
+}
+
+/// CPU frequency driver.
+///
+/// Implement this trait to provide a CPU frequency driver and its callbacks.
+///
+/// Reference: <https://docs.kernel.org/cpu-freq/cpu-drivers.html>
+#[vtable]
+pub trait Driver {
+ /// Driver's name.
+ const NAME: &'static CStr;
+
+ /// Driver's flags.
+ const FLAGS: u16;
+
+ /// Boost support.
+ const BOOST_ENABLED: bool;
+
+ /// Policy specific data.
+ ///
+ /// Require that `PData` implements `ForeignOwnable`. We guarantee to never move the underlying
+ /// wrapped data structure.
+ type PData: ForeignOwnable;
+
+ /// Driver's `init` callback.
+ fn init(policy: &mut Policy) -> Result<Self::PData>;
+
+ /// Driver's `exit` callback.
+ fn exit(_policy: &mut Policy, _data: Option<Self::PData>) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `online` callback.
+ fn online(_policy: &mut Policy) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `offline` callback.
+ fn offline(_policy: &mut Policy) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `suspend` callback.
+ fn suspend(_policy: &mut Policy) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `resume` callback.
+ fn resume(_policy: &mut Policy) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `ready` callback.
+ fn ready(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `verify` callback.
+ fn verify(data: &mut PolicyData) -> Result<()>;
+
+ /// Driver's `setpolicy` callback.
+ fn setpolicy(_policy: &mut Policy) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target` callback.
+ fn target(_policy: &mut Policy, _target_freq: u32, _relation: Relation) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target_index` callback.
+ fn target_index(_policy: &mut Policy, _index: TableIndex) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `fast_switch` callback.
+ fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `adjust_perf` callback.
+ fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `get_intermediate` callback.
+ fn get_intermediate(_policy: &mut Policy, _index: TableIndex) -> u32 {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target_intermediate` callback.
+ fn target_intermediate(_policy: &mut Policy, _index: TableIndex) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `get` callback.
+ fn get(_policy: &mut Policy) -> Result<u32> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `update_limits` callback.
+ fn update_limits(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `bios_limit` callback.
+ fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `set_boost` callback.
+ fn set_boost(_policy: &mut Policy, _state: i32) -> Result<()> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `register_em` callback.
+ fn register_em(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+}
+
+/// CPU frequency driver Registration.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to register a cpufreq driver.
+///
+/// ```
+/// use kernel::{
+/// cpu, cpufreq,
+/// c_str,
+/// device::{Bound, Device},
+/// macros::vtable,
+/// sync::Arc,
+/// };
+/// struct FooDevice;
+///
+/// #[derive(Default)]
+/// struct FooDriver;
+///
+/// #[vtable]
+/// impl cpufreq::Driver for FooDriver {
+/// const NAME: &'static CStr = c_str!("cpufreq-foo");
+/// const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV;
+/// const BOOST_ENABLED: bool = true;
+///
+/// type PData = Arc<FooDevice>;
+///
+/// fn init(policy: &mut cpufreq::Policy) -> Result<Self::PData> {
+/// // Initialize here
+/// Ok(Arc::new(FooDevice, GFP_KERNEL)?)
+/// }
+///
+/// fn exit(_policy: &mut cpufreq::Policy, _data: Option<Self::PData>) -> Result<()> {
+/// Ok(())
+/// }
+///
+/// fn suspend(policy: &mut cpufreq::Policy) -> Result<()> {
+/// policy.generic_suspend()
+/// }
+///
+/// fn verify(data: &mut cpufreq::PolicyData) -> Result<()> {
+/// data.generic_verify()
+/// }
+///
+/// fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result<()> {
+/// // Update CPU frequency
+/// Ok(())
+/// }
+///
+/// fn get(policy: &mut cpufreq::Policy) -> Result<u32> {
+/// policy.generic_get()
+/// }
+/// }
+///
+/// fn foo_probe(dev: &Device<Bound>) {
+/// cpufreq::Registration::<FooDriver>::new_foreign_owned(dev).unwrap();
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Registration<T: Driver>(KBox<UnsafeCell<bindings::cpufreq_driver>>, PhantomData<T>);
+
+/// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads
+/// or CPUs, so it is safe to share it.
+unsafe impl<T: Driver> Sync for Registration<T> {}
+
+#[allow(clippy::non_send_fields_in_send_ty)]
+/// SAFETY: Registration with and unregistration from the cpufreq subsystem can happen from any
+/// thread.
+unsafe impl<T: Driver> Send for Registration<T> {}
+
+impl<T: Driver> Registration<T> {
+ const VTABLE: bindings::cpufreq_driver = bindings::cpufreq_driver {
+ name: Self::copy_name(T::NAME),
+ boost_enabled: T::BOOST_ENABLED,
+ flags: T::FLAGS,
+
+ // Initialize mandatory callbacks.
+ init: Some(Self::init_callback),
+ verify: Some(Self::verify_callback),
+
+ // Initialize optional callbacks based on the traits of `T`.
+ setpolicy: if T::HAS_SETPOLICY {
+ Some(Self::setpolicy_callback)
+ } else {
+ None
+ },
+ target: if T::HAS_TARGET {
+ Some(Self::target_callback)
+ } else {
+ None
+ },
+ target_index: if T::HAS_TARGET_INDEX {
+ Some(Self::target_index_callback)
+ } else {
+ None
+ },
+ fast_switch: if T::HAS_FAST_SWITCH {
+ Some(Self::fast_switch_callback)
+ } else {
+ None
+ },
+ adjust_perf: if T::HAS_ADJUST_PERF {
+ Some(Self::adjust_perf_callback)
+ } else {
+ None
+ },
+ get_intermediate: if T::HAS_GET_INTERMEDIATE {
+ Some(Self::get_intermediate_callback)
+ } else {
+ None
+ },
+ target_intermediate: if T::HAS_TARGET_INTERMEDIATE {
+ Some(Self::target_intermediate_callback)
+ } else {
+ None
+ },
+ get: if T::HAS_GET {
+ Some(Self::get_callback)
+ } else {
+ None
+ },
+ update_limits: if T::HAS_UPDATE_LIMITS {
+ Some(Self::update_limits_callback)
+ } else {
+ None
+ },
+ bios_limit: if T::HAS_BIOS_LIMIT {
+ Some(Self::bios_limit_callback)
+ } else {
+ None
+ },
+ online: if T::HAS_ONLINE {
+ Some(Self::online_callback)
+ } else {
+ None
+ },
+ offline: if T::HAS_OFFLINE {
+ Some(Self::offline_callback)
+ } else {
+ None
+ },
+ exit: if T::HAS_EXIT {
+ Some(Self::exit_callback)
+ } else {
+ None
+ },
+ suspend: if T::HAS_SUSPEND {
+ Some(Self::suspend_callback)
+ } else {
+ None
+ },
+ resume: if T::HAS_RESUME {
+ Some(Self::resume_callback)
+ } else {
+ None
+ },
+ ready: if T::HAS_READY {
+ Some(Self::ready_callback)
+ } else {
+ None
+ },
+ set_boost: if T::HAS_SET_BOOST {
+ Some(Self::set_boost_callback)
+ } else {
+ None
+ },
+ register_em: if T::HAS_REGISTER_EM {
+ Some(Self::register_em_callback)
+ } else {
+ None
+ },
+ // SAFETY: All zeros is a valid value for `bindings::cpufreq_driver`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ };
+
+ const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] {
+ let src = name.as_bytes_with_nul();
+ let mut dst = [0; CPUFREQ_NAME_LEN];
+
+ build_assert!(src.len() <= CPUFREQ_NAME_LEN);
+
+ let mut i = 0;
+ while i < src.len() {
+ dst[i] = src[i];
+ i += 1;
+ }
+
+ dst
+ }
+
+ /// Registers a CPU frequency driver with the cpufreq core.
+ pub fn new() -> Result<Self> {
+ // We can't use `&Self::VTABLE` directly because the cpufreq core modifies some fields in
+ // the C `struct cpufreq_driver`, which requires a mutable reference.
+ let mut drv = KBox::new(UnsafeCell::new(Self::VTABLE), GFP_KERNEL)?;
+
+ // SAFETY: `drv` is guaranteed to be valid for the lifetime of `Registration`.
+ to_result(unsafe { bindings::cpufreq_register_driver(drv.get_mut()) })?;
+
+ Ok(Self(drv, PhantomData))
+ }
+
+ /// Same as [`Registration::new`], but does not return a [`Registration`] instance.
+ ///
+ /// Instead the [`Registration`] is owned by [`Devres`] and will be revoked / dropped, once the
+ /// device is detached.
+ pub fn new_foreign_owned(dev: &Device<Bound>) -> Result<()> {
+ Devres::new_foreign_owned(dev, Self::new()?, GFP_KERNEL)
+ }
+}
+
+/// CPU frequency driver callbacks.
+impl<T: Driver> Registration<T> {
+ /// Driver's `init` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ let data = T::init(policy)?;
+ policy.set_data(data)?;
+ Ok(0)
+ })
+ }
+
+ /// Driver's `exit` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ let data = policy.clear_data();
+ let _ = T::exit(policy, data);
+ }
+
+ /// Driver's `online` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::online(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `offline` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::offline(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `suspend` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::suspend(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `resume` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::resume(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `ready` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::ready(policy);
+ }
+
+ /// Driver's `verify` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let data = unsafe { PolicyData::from_raw_mut(ptr) };
+ T::verify(data).map(|()| 0)
+ })
+ }
+
+ /// Driver's `setpolicy` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::setpolicy(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `target` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn target_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ target_freq: u32,
+ relation: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::target(policy, target_freq, Relation::new(relation)?).map(|()| 0)
+ })
+ }
+
+ /// Driver's `target_index` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn target_index_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::target_index(policy, index).map(|()| 0)
+ })
+ }
+
+ /// Driver's `fast_switch` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn fast_switch_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ target_freq: u32,
+ ) -> kernel::ffi::c_uint {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::fast_switch(policy, target_freq)
+ }
+
+ /// Driver's `adjust_perf` callback.
+ extern "C" fn adjust_perf_callback(
+ cpu: u32,
+ min_perf: usize,
+ target_perf: usize,
+ capacity: usize,
+ ) {
+ if let Ok(mut policy) = PolicyCpu::from_cpu(cpu) {
+ T::adjust_perf(&mut policy, min_perf, target_perf, capacity);
+ }
+ }
+
+ /// Driver's `get_intermediate` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn get_intermediate_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_uint {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::get_intermediate(policy, index)
+ }
+
+ /// Driver's `target_intermediate` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn target_intermediate_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::target_intermediate(policy, index).map(|()| 0)
+ })
+ }
+
+ /// Driver's `get` callback.
+ extern "C" fn get_callback(cpu: u32) -> kernel::ffi::c_uint {
+ PolicyCpu::from_cpu(cpu).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f))
+ }
+
+ /// Driver's `update_limit` callback.
+ extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::update_limits(policy);
+ }
+
+ /// Driver's `bios_limit` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn bios_limit_callback(cpu: i32, limit: *mut u32) -> kernel::ffi::c_int {
+ from_result(|| {
+ let mut policy = PolicyCpu::from_cpu(cpu as u32)?;
+
+ // SAFETY: `limit` is guaranteed by the C code to be valid.
+ T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0)
+ })
+ }
+
+ /// Driver's `set_boost` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn set_boost_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ state: i32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::set_boost(policy, state).map(|()| 0)
+ })
+ }
+
+ /// Driver's `register_em` callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::register_em(policy);
+ }
+}
+
+impl<T: Driver> Drop for Registration<T> {
+ /// Unregisters with the cpufreq core.
+ fn drop(&mut self) {
+ // SAFETY: `self.0` is guaranteed to be valid for the lifetime of `Registration`.
+ unsafe { bindings::cpufreq_unregister_driver(self.0.get_mut()) };
+ }
+}
projects / users / hch / misc.git / commitdiff