From 1537ec85ebd7d7aa3ce1a003007cd3588cd58bda Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:06 -0800 Subject: [PATCH 01/16] drm/xe/uapi: Introduce API for EU stall sampling A new hardware feature first introduced in PVC gives capability to periodically sample EU stall state and record counts for different stall reasons, on a per IP basis, aggregate across all EUs in a subslice and record the samples in a buffer in each subslice. Eventually, the aggregated data is written out to a buffer in the memory. This feature is also supported in XE2 and later architecture GPUs. Use an existing IOCTL - DRM_IOCTL_XE_OBSERVATION as the interface into the driver from the user space to do initial setup and obtain a file descriptor for the EU stall data stream. Input parameter to the IOCTL is a struct drm_xe_observation_param in which observation_type should be set to DRM_XE_OBSERVATION_TYPE_EU_STALL, observation_op should be DRM_XE_OBSERVATION_OP_STREAM_OPEN and param should point to a chain of drm_xe_ext_set_property structures in which each structure has a pair of property and value. The EU stall sampling input properties are defined in drm_xe_eu_stall_property_id enum. With the file descriptor obtained from DRM_IOCTL_XE_OBSERVATION, user space can enable and disable EU stall sampling with the IOCTLs: DRM_XE_OBSERVATION_IOCTL_ENABLE and DRM_XE_OBSERVATION_IOCTL_DISABLE. User space can also call poll() to check for availability of data in the buffer. The data can be read with read(). Finally, the file descriptor can be closed with close(). v11: Changed a couple of variables in struct eu_stall_open_properties from unsigned int to int. v10: Use extension number while parsing chain of extensions. Remove function description for static functions. Move code around as per review feedback. v9: Changed some u32 to unsigned int. Moved some code around as per review feedback from v8. v8: Used div_u64 instead of / to fix 32-bit build issue. Changed copyright year in xe_eu_stall.c/h to 2025. v7: Renamed input property DRM_XE_EU_STALL_PROP_EVENT_REPORT_COUNT to DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS to be consistent with OA. Renamed the corresponding internal variables. Fixed some commit messages based on review feedback. v6: Change the input sampling rate to GPU cycles instead of GPU cycles multiplier. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/bb707a27975c33e4a912b9839b023acb7a1f9c90.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_eu_stall.c | 218 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_eu_stall.h | 14 ++ drivers/gpu/drm/xe/xe_observation.c | 14 ++ include/uapi/drm/xe_drm.h | 38 +++++ 5 files changed, 285 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.c create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index ffc836fa8e60..4fb3a4a336fd 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -33,6 +33,7 @@ xe-y += xe_bb.o \ xe_device_sysfs.o \ xe_dma_buf.o \ xe_drm_client.o \ + xe_eu_stall.o \ xe_exec.o \ xe_exec_queue.o \ xe_execlist.o \ diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c new file mode 100644 index 000000000000..62a92aa161e8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include + +#include + +#include "xe_device.h" +#include "xe_eu_stall.h" +#include "xe_gt_printk.h" +#include "xe_gt_topology.h" +#include "xe_macros.h" +#include "xe_observation.h" + +/** + * struct eu_stall_open_properties - EU stall sampling properties received + * from user space at open. + * @sampling_rate_mult: EU stall sampling rate multiplier. + * HW will sample every (sampling_rate_mult x 251) cycles. + * @wait_num_reports: Minimum number of EU stall data reports to unblock poll(). + * @gt: GT on which EU stall data will be captured. + */ +struct eu_stall_open_properties { + int sampling_rate_mult; + int wait_num_reports; + struct xe_gt *gt; +}; + +static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + value = div_u64(value, 251); + if (value == 0 || value > 7) { + drm_dbg(&xe->drm, "Invalid EU stall sampling rate %llu\n", value); + return -EINVAL; + } + props->sampling_rate_mult = value; + return 0; +} + +static int set_prop_eu_stall_wait_num_reports(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + props->wait_num_reports = value; + + return 0; +} + +static int set_prop_eu_stall_gt_id(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + if (value >= xe->info.gt_count) { + drm_dbg(&xe->drm, "Invalid GT ID %llu for EU stall sampling\n", value); + return -EINVAL; + } + props->gt = xe_device_get_gt(xe, value); + return 0; +} + +typedef int (*set_eu_stall_property_fn)(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props); + +static const set_eu_stall_property_fn xe_set_eu_stall_property_funcs[] = { + [DRM_XE_EU_STALL_PROP_SAMPLE_RATE] = set_prop_eu_stall_sampling_rate, + [DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS] = set_prop_eu_stall_wait_num_reports, + [DRM_XE_EU_STALL_PROP_GT_ID] = set_prop_eu_stall_gt_id, +}; + +static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) || + XE_IOCTL_DBG(xe, ext.pad)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs)); + return xe_set_eu_stall_property_funcs[idx](xe, ext.value, props); +} + +typedef int (*xe_eu_stall_user_extension_fn)(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props); +static const xe_eu_stall_user_extension_fn xe_eu_stall_user_extension_funcs[] = { + [DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY] = xe_eu_stall_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 5 +static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, + int ext_number, struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(xe_eu_stall_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_eu_stall_user_extension_funcs)); + err = xe_eu_stall_user_extension_funcs[idx](xe, extension, props); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return xe_eu_stall_user_extensions(xe, ext.next_extension, ++ext_number, props); + + return 0; +} + +/* + * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE + * before calling read(). + */ +static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + ssize_t ret = 0; + + return ret; +} + +static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) +{ + __poll_t ret = 0; + + return ret; +} + +static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + return 0; +} + +static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) +{ + return 0; +} + +static const struct file_operations fops_eu_stall = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .release = xe_eu_stall_stream_close, + .poll = xe_eu_stall_stream_poll, + .read = xe_eu_stall_stream_read, + .unlocked_ioctl = xe_eu_stall_stream_ioctl, + .compat_ioctl = xe_eu_stall_stream_ioctl, +}; + +static inline bool has_eu_stall_sampling_support(struct xe_device *xe) +{ + return false; +} + +/** + * xe_eu_stall_stream_open - Open a xe EU stall data stream fd + * + * @dev: DRM device pointer + * @data: pointer to first struct @drm_xe_ext_set_property in + * the chain of input properties from the user space. + * @file: DRM file pointer + * + * This function opens a EU stall data stream with input properties from + * the user space. + * + * Returns: EU stall data stream fd on success or a negative error code. + */ +int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct eu_stall_open_properties props = {}; + int ret, stream_fd; + + if (!has_eu_stall_sampling_support(xe)) { + drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + return -ENODEV; + } + + if (xe_observation_paranoid && !perfmon_capable()) { + drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n"); + return -EACCES; + } + + ret = xe_eu_stall_user_extensions(xe, data, 0, &props); + if (ret) + return ret; + + if (!props.gt) { + drm_dbg(&xe->drm, "GT ID not provided for EU stall sampling\n"); + return -EINVAL; + } + + stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall, NULL, 0); + if (stream_fd < 0) + xe_gt_dbg(props.gt, "EU stall inode get fd failed : %d\n", stream_fd); + + return stream_fd; +} diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h new file mode 100644 index 000000000000..c1aef8adac6e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef __XE_EU_STALL_H__ +#define __XE_EU_STALL_H__ + +#include "xe_gt_types.h" + +int xe_eu_stall_stream_open(struct drm_device *dev, + u64 data, + struct drm_file *file); +#endif diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c index 8ec1b84cbb9e..5011d0736644 100644 --- a/drivers/gpu/drm/xe/xe_observation.c +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -8,6 +8,7 @@ #include +#include "xe_eu_stall.h" #include "xe_oa.h" #include "xe_observation.h" @@ -29,6 +30,17 @@ static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param * } } +static int xe_eu_stall_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, + struct drm_file *file) +{ + switch (arg->observation_op) { + case DRM_XE_OBSERVATION_OP_STREAM_OPEN: + return xe_eu_stall_stream_open(dev, arg->param, file); + default: + return -EINVAL; + } +} + /** * xe_observation_ioctl - The top level observation layer ioctl * @dev: @drm_device @@ -51,6 +63,8 @@ int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *fi switch (arg->observation_type) { case DRM_XE_OBSERVATION_TYPE_OA: return xe_oa_ioctl(dev, arg, file); + case DRM_XE_OBSERVATION_TYPE_EU_STALL: + return xe_eu_stall_ioctl(dev, arg, file); default: return -EINVAL; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 892f54d3aa09..95cb9e65540b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1496,6 +1496,8 @@ struct drm_xe_wait_user_fence { enum drm_xe_observation_type { /** @DRM_XE_OBSERVATION_TYPE_OA: OA observation stream type */ DRM_XE_OBSERVATION_TYPE_OA, + /** @DRM_XE_OBSERVATION_TYPE_EU_STALL: EU stall sampling observation stream type */ + DRM_XE_OBSERVATION_TYPE_EU_STALL, }; /** @@ -1848,6 +1850,42 @@ enum drm_xe_pxp_session_type { /* ID of the protected content session managed by Xe when PXP is active */ #define DRM_XE_PXP_HWDRM_DEFAULT_SESSION 0xf +/** + * enum drm_xe_eu_stall_property_id - EU stall sampling input property ids. + * + * These properties are passed to the driver at open as a chain of + * @drm_xe_ext_set_property structures with @property set to these + * properties' enums and @value set to the corresponding values of these + * properties. @drm_xe_user_extension base.name should be set to + * @DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY. + * + * With the file descriptor obtained from open, user space must enable + * the EU stall stream fd with @DRM_XE_OBSERVATION_IOCTL_ENABLE before + * calling read(). EIO errno from read() indicates HW dropped data + * due to full buffer. + */ +enum drm_xe_eu_stall_property_id { +#define DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY 0 + /** + * @DRM_XE_EU_STALL_PROP_GT_ID: @gt_id of the GT on which + * EU stall data will be captured. + */ + DRM_XE_EU_STALL_PROP_GT_ID = 1, + + /** + * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate + * in GPU cycles. + */ + DRM_XE_EU_STALL_PROP_SAMPLE_RATE, + + /** + * @DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS: Minimum number of + * EU stall data reports to be present in the kernel buffer + * before unblocking a blocked poll or read. + */ + DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS, +}; + #if defined(__cplusplus) } #endif -- 2.51.0 From 9a0b11d4cf3b4324378c322b7043962e648681ed Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:07 -0800 Subject: [PATCH 02/16] drm/xe/eustall: Add support to init, enable and disable EU stall sampling Implement EU stall sampling APIs introduced in the previous patch for Xe_HPC (PVC). Add register definitions and the code that accesses these registers to the APIs. Add initialization and clean up functions and their implementations, EU stall enable and disable functions. v11: Move stream->xecore_buf alloc to xe_eu_stall_data_buf_alloc(). Register xe_eu_stall_fini() with devm_add_action_or_reset() instead of calling it from xe_gt_fini(). Changed a couple of variables in struct xe_eu_stall_data_stream from unsigned int to int. v10: Fixed error rewinding code Moved code around as per review feedback v9: Moved structure definitions from xe_eu_stall.h to xe_eu_stall.c Moved read and poll implementations to the next patch Used xe_bo_create_pin_map_at_aligned instead of xe_bo_create_pin_map Changed lock names as per review feedback Moved drop data handling into a subsequent patch Moved code around as per review feedback v8: Updated copyright year in xe_eu_stall_regs.h to 2025. Renamed struct drm_xe_eu_stall_data_pvc to struct xe_eu_stall_data_pvc since it is a local structure. v6: Fix buffer wrap around over write bug (Matt Olson) Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/b6aeca593d521828a0b4fbf6cfd2844716c4fc66.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h | 29 ++ drivers/gpu/drm/xe/xe_eu_stall.c | 378 ++++++++++++++++++++- drivers/gpu/drm/xe/xe_eu_stall.h | 1 + drivers/gpu/drm/xe/xe_gt.c | 5 + drivers/gpu/drm/xe/xe_gt_types.h | 3 + 5 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h diff --git a/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h new file mode 100644 index 000000000000..c53f57fdde65 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_EU_STALL_REGS_H_ +#define _XE_EU_STALL_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define XEHPC_EUSTALL_BASE XE_REG_MCR(0xe520) +#define XEHPC_EUSTALL_BASE_BUF_ADDR REG_GENMASK(31, 6) +#define XEHPC_EUSTALL_BASE_XECORE_BUF_SZ REG_GENMASK(5, 3) +#define XEHPC_EUSTALL_BASE_ENABLE_SAMPLING REG_BIT(1) + +#define XEHPC_EUSTALL_BASE_UPPER XE_REG_MCR(0xe524) + +#define XEHPC_EUSTALL_REPORT XE_REG_MCR(0xe528, XE_REG_OPTION_MASKED) +#define XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK REG_GENMASK(15, 2) +#define XEHPC_EUSTALL_REPORT_OVERFLOW_DROP REG_BIT(1) + +#define XEHPC_EUSTALL_REPORT1 XE_REG_MCR(0xe52c, XE_REG_OPTION_MASKED) +#define XEHPC_EUSTALL_REPORT1_READ_PTR_MASK REG_GENMASK(15, 2) + +#define XEHPC_EUSTALL_CTRL XE_REG_MCR(0xe53c, XE_REG_OPTION_MASKED) +#define EUSTALL_MOCS REG_GENMASK(9, 3) +#define EUSTALL_SAMPLE_RATE REG_GENMASK(2, 0) + +#endif diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 62a92aa161e8..2e7d00f8ff40 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -8,14 +8,52 @@ #include #include +#include #include +#include "xe_bo.h" #include "xe_device.h" #include "xe_eu_stall.h" +#include "xe_force_wake.h" +#include "xe_gt_mcr.h" #include "xe_gt_printk.h" #include "xe_gt_topology.h" #include "xe_macros.h" #include "xe_observation.h" +#include "xe_pm.h" + +#include "regs/xe_eu_stall_regs.h" +#include "regs/xe_gt_regs.h" + +static size_t per_xecore_buf_size = SZ_512K; + +struct per_xecore_buf { + /* Buffer vaddr */ + u8 *vaddr; + /* Write pointer */ + u32 write; + /* Read pointer */ + u32 read; +}; + +struct xe_eu_stall_data_stream { + bool enabled; + int wait_num_reports; + int sampling_rate_mult; + size_t data_record_size; + size_t per_xecore_buf_size; + + struct xe_gt *gt; + struct xe_bo *bo; + struct per_xecore_buf *xecore_buf; +}; + +struct xe_eu_stall_gt { + /* Lock to protect stream */ + struct mutex stream_lock; + /* EU stall data stream */ + struct xe_eu_stall_data_stream *stream; +}; /** * struct eu_stall_open_properties - EU stall sampling properties received @@ -31,6 +69,88 @@ struct eu_stall_open_properties { struct xe_gt *gt; }; +/* + * EU stall data format for PVC + */ +struct xe_eu_stall_data_pvc { + __u64 ip_addr:29; /* Bits 0 to 28 */ + __u64 active_count:8; /* Bits 29 to 36 */ + __u64 other_count:8; /* Bits 37 to 44 */ + __u64 control_count:8; /* Bits 45 to 52 */ + __u64 pipestall_count:8; /* Bits 53 to 60 */ + __u64 send_count:8; /* Bits 61 to 68 */ + __u64 dist_acc_count:8; /* Bits 69 to 76 */ + __u64 sbid_count:8; /* Bits 77 to 84 */ + __u64 sync_count:8; /* Bits 85 to 92 */ + __u64 inst_fetch_count:8; /* Bits 93 to 100 */ + __u64 unused_bits:27; + __u64 unused[6]; +} __packed; + +static size_t xe_eu_stall_data_record_size(struct xe_device *xe) +{ + size_t record_size = 0; + + if (xe->info.platform == XE_PVC) + record_size = sizeof(struct xe_eu_stall_data_pvc); + + xe_assert(xe, is_power_of_2(record_size)); + + return record_size; +} + +/** + * num_data_rows - Return the number of EU stall data rows of 64B each + * for a given data size. + * + * @data_size: EU stall data size + */ +static u32 num_data_rows(u32 data_size) +{ + return data_size >> 6; +} + +static void xe_eu_stall_fini(void *arg) +{ + struct xe_gt *gt = arg; + + mutex_destroy(>->eu_stall->stream_lock); + kfree(gt->eu_stall); +} + +/** + * xe_eu_stall_init() - Allocate and initialize GT level EU stall data + * structure xe_eu_stall_gt within struct xe_gt. + * + * @gt: GT being initialized. + * + * Returns: zero on success or a negative error code. + */ +int xe_eu_stall_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int ret; + + gt->eu_stall = kzalloc(sizeof(*gt->eu_stall), GFP_KERNEL); + if (!gt->eu_stall) { + ret = -ENOMEM; + goto exit; + } + + mutex_init(>->eu_stall->stream_lock); + + ret = devm_add_action_or_reset(xe->drm.dev, xe_eu_stall_fini, gt); + if (ret) + goto exit_free; + + return 0; +exit_free: + mutex_destroy(>->eu_stall->stream_lock); + kfree(gt->eu_stall); +exit: + return ret; +} + static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value, struct eu_stall_open_properties *props) { @@ -140,6 +260,135 @@ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, return ret; } +static void xe_eu_stall_stream_free(struct xe_eu_stall_data_stream *stream) +{ + struct xe_gt *gt = stream->gt; + + gt->eu_stall->stream = NULL; + kfree(stream); +} + +static void xe_eu_stall_data_buf_destroy(struct xe_eu_stall_data_stream *stream) +{ + xe_bo_unpin_map_no_vm(stream->bo); + kfree(stream->xecore_buf); +} + +static int xe_eu_stall_data_buf_alloc(struct xe_eu_stall_data_stream *stream, + u16 last_xecore) +{ + struct xe_tile *tile = stream->gt->tile; + struct xe_bo *bo; + u32 size; + + stream->xecore_buf = kcalloc(last_xecore, sizeof(*stream->xecore_buf), GFP_KERNEL); + if (!stream->xecore_buf) + return -ENOMEM; + + size = stream->per_xecore_buf_size * last_xecore; + + bo = xe_bo_create_pin_map_at_aligned(tile->xe, tile, NULL, + size, ~0ull, ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, SZ_64); + if (IS_ERR(bo)) { + kfree(stream->xecore_buf); + return PTR_ERR(bo); + } + + XE_WARN_ON(!IS_ALIGNED(xe_bo_ggtt_addr(bo), SZ_64)); + stream->bo = bo; + + return 0; +} + +static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) +{ + u32 write_ptr_reg, write_ptr, read_ptr_reg, reg_value; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + u16 group, instance; + unsigned int fw_ref; + int xecore; + + /* Take runtime pm ref and forcewake to disable RC6 */ + xe_pm_runtime_get(gt_to_xe(gt)); + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_RENDER); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_RENDER)) { + xe_gt_err(gt, "Failed to get RENDER forcewake\n"); + xe_pm_runtime_put(gt_to_xe(gt)); + return -ETIMEDOUT; + } + + for_each_dss_steering(xecore, gt, group, instance) { + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance); + write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg); + read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, write_ptr); + read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg); + /* Initialize the read pointer to the write pointer */ + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance); + write_ptr <<= 6; + write_ptr &= (stream->per_xecore_buf_size << 1) - 1; + xecore_buf = &stream->xecore_buf[xecore]; + xecore_buf->write = write_ptr; + xecore_buf->read = write_ptr; + } + reg_value = _MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE, + REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) | + REG_FIELD_PREP(EUSTALL_SAMPLE_RATE, + stream->sampling_rate_mult)); + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_CTRL, reg_value); + /* GGTT addresses can never be > 32 bits */ + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE_UPPER, 0); + reg_value = xe_bo_ggtt_addr(stream->bo); + reg_value |= REG_FIELD_PREP(XEHPC_EUSTALL_BASE_XECORE_BUF_SZ, + stream->per_xecore_buf_size / SZ_256K); + reg_value |= XEHPC_EUSTALL_BASE_ENABLE_SAMPLING; + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, reg_value); + + return 0; +} + +static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, + struct eu_stall_open_properties *props) +{ + unsigned int max_wait_num_reports, xecore, last_xecore, num_xecores; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + xe_dss_mask_t all_xecores; + u16 group, instance; + u32 vaddr_offset; + int ret; + + bitmap_or(all_xecores, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS); + num_xecores = bitmap_weight(all_xecores, XE_MAX_DSS_FUSE_BITS); + last_xecore = xe_gt_topology_mask_last_dss(all_xecores) + 1; + + max_wait_num_reports = num_data_rows(per_xecore_buf_size * num_xecores); + if (props->wait_num_reports == 0 || props->wait_num_reports > max_wait_num_reports) { + xe_gt_dbg(gt, "Invalid EU stall event report count %u\n", + props->wait_num_reports); + xe_gt_dbg(gt, "Minimum event report count is 1, maximum is %u\n", + max_wait_num_reports); + return -EINVAL; + } + stream->per_xecore_buf_size = per_xecore_buf_size; + stream->sampling_rate_mult = props->sampling_rate_mult; + stream->wait_num_reports = props->wait_num_reports; + stream->data_record_size = xe_eu_stall_data_record_size(gt_to_xe(gt)); + + ret = xe_eu_stall_data_buf_alloc(stream, last_xecore); + if (ret) + return ret; + + for_each_dss_steering(xecore, gt, group, instance) { + xecore_buf = &stream->xecore_buf[xecore]; + vaddr_offset = xecore * stream->per_xecore_buf_size; + xecore_buf->vaddr = stream->bo->vmap.vaddr + vaddr_offset; + } + return 0; +} + static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) { __poll_t ret = 0; @@ -147,13 +396,75 @@ static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) return ret; } -static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +static int xe_eu_stall_enable_locked(struct xe_eu_stall_data_stream *stream) +{ + int ret = 0; + + if (stream->enabled) + return ret; + + stream->enabled = true; + + ret = xe_eu_stall_stream_enable(stream); + return ret; +} + +static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream) { + struct xe_gt *gt = stream->gt; + + if (!stream->enabled) + return 0; + + stream->enabled = false; + + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, 0); + + xe_force_wake_put(gt_to_fw(gt), XE_FW_RENDER); + xe_pm_runtime_put(gt_to_xe(gt)); + return 0; } +static long xe_eu_stall_stream_ioctl_locked(struct xe_eu_stall_data_stream *stream, + unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case DRM_XE_OBSERVATION_IOCTL_ENABLE: + return xe_eu_stall_enable_locked(stream); + case DRM_XE_OBSERVATION_IOCTL_DISABLE: + return xe_eu_stall_disable_locked(stream); + } + + return -EINVAL; +} + +static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + long ret; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_ioctl_locked(stream, cmd, arg); + mutex_unlock(>->eu_stall->stream_lock); + + return ret; +} + static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) { + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + + drm_dev_put(>->tile->xe->drm); + + mutex_lock(>->eu_stall->stream_lock); + xe_eu_stall_disable_locked(stream); + xe_eu_stall_data_buf_destroy(stream); + xe_eu_stall_stream_free(stream); + mutex_unlock(>->eu_stall->stream_lock); + return 0; } @@ -169,7 +480,56 @@ static const struct file_operations fops_eu_stall = { static inline bool has_eu_stall_sampling_support(struct xe_device *xe) { - return false; + return xe->info.platform == XE_PVC; +} + +static int xe_eu_stall_stream_open_locked(struct drm_device *dev, + struct eu_stall_open_properties *props, + struct drm_file *file) +{ + struct xe_eu_stall_data_stream *stream; + struct xe_gt *gt = props->gt; + unsigned long f_flags = 0; + int ret, stream_fd; + + /* Only one session can be active at any time */ + if (gt->eu_stall->stream) { + xe_gt_dbg(gt, "EU stall sampling session already active\n"); + return -EBUSY; + } + + stream = kzalloc(sizeof(*stream), GFP_KERNEL); + if (!stream) + return -ENOMEM; + + gt->eu_stall->stream = stream; + stream->gt = gt; + + ret = xe_eu_stall_stream_init(stream, props); + if (ret) { + xe_gt_dbg(gt, "EU stall stream init failed : %d\n", ret); + goto err_free; + } + + stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall, stream, f_flags); + if (stream_fd < 0) { + ret = stream_fd; + xe_gt_dbg(gt, "EU stall inode get fd failed : %d\n", ret); + goto err_destroy; + } + + /* Take a reference on the driver that will be kept with stream_fd + * until its release. + */ + drm_dev_get(>->tile->xe->drm); + + return stream_fd; + +err_destroy: + xe_eu_stall_data_buf_destroy(stream); +err_free: + xe_eu_stall_stream_free(stream); + return ret; } /** @@ -189,7 +549,7 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f { struct xe_device *xe = to_xe_device(dev); struct eu_stall_open_properties props = {}; - int ret, stream_fd; + int ret; if (!has_eu_stall_sampling_support(xe)) { drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); @@ -201,6 +561,10 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f return -EACCES; } + /* Initialize and set default values */ + props.wait_num_reports = 1; + props.sampling_rate_mult = 4; + ret = xe_eu_stall_user_extensions(xe, data, 0, &props); if (ret) return ret; @@ -210,9 +574,9 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f return -EINVAL; } - stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall, NULL, 0); - if (stream_fd < 0) - xe_gt_dbg(props.gt, "EU stall inode get fd failed : %d\n", stream_fd); + mutex_lock(&props.gt->eu_stall->stream_lock); + ret = xe_eu_stall_stream_open_locked(dev, &props, file); + mutex_unlock(&props.gt->eu_stall->stream_lock); - return stream_fd; + return ret; } diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h index c1aef8adac6e..24e215b840c0 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.h +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -8,6 +8,7 @@ #include "xe_gt_types.h" +int xe_eu_stall_init(struct xe_gt *gt); int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file); diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 650a0ee56e97..5bd8dfdce300 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -19,6 +19,7 @@ #include "xe_bb.h" #include "xe_bo.h" #include "xe_device.h" +#include "xe_eu_stall.h" #include "xe_exec_queue.h" #include "xe_execlist.h" #include "xe_force_wake.h" @@ -613,6 +614,10 @@ int xe_gt_init(struct xe_gt *gt) xe_gt_record_user_engines(gt); + err = xe_eu_stall_init(gt); + if (err) + return err; + return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index f72b965cc9e6..f67474e06fb3 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -430,6 +430,9 @@ struct xe_gt { /** @oa: oa observation subsystem per gt info */ struct xe_oa_gt oa; + + /** @eu_stall: EU stall counters subsystem per gt info */ + struct xe_eu_stall_gt *eu_stall; }; #endif -- 2.51.0 From 760edec9396851935c20914f5ffdff94dd9d8f8c Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:08 -0800 Subject: [PATCH 03/16] drm/xe/eustall: Add support to read() and poll() EU stall data Implement the EU stall sampling APIs to read() and poll() EU stall data. A work function periodically polls the EU stall data buffer write pointer registers to look for any new data and caches the write pointer. The read function compares the cached read and write pointers and copies any new data to the user space. v11: Used gt->eu_stall->stream_lock instead of stream->buf_lock. Removed read and write offsets from trace and added read size. Moved workqueue from struct xe_eu_stall_data_stream to struct xe_eu_stall_gt. v10: Used cancel_delayed_work_sync() instead of flush_delayed_work() Replaced per xecore lock with a lock for all the xecore buffers Code movement and optimizations as per review feedback v9: New patch split from the previous patch. Used *_delayed_work functions instead of hrtimer Addressed the review feedback in read and poll functions Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/369dee85a3b6bd2c08aeae89ca55e66a9a0242d2.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 267 ++++++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_trace.h | 30 ++++ 2 files changed, 294 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 2e7d00f8ff40..e82a6ebfe979 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -21,10 +21,13 @@ #include "xe_macros.h" #include "xe_observation.h" #include "xe_pm.h" +#include "xe_trace.h" #include "regs/xe_eu_stall_regs.h" #include "regs/xe_gt_regs.h" +#define POLL_PERIOD_MS 5 + static size_t per_xecore_buf_size = SZ_512K; struct per_xecore_buf { @@ -37,15 +40,18 @@ struct per_xecore_buf { }; struct xe_eu_stall_data_stream { + bool pollin; bool enabled; int wait_num_reports; int sampling_rate_mult; + wait_queue_head_t poll_wq; size_t data_record_size; size_t per_xecore_buf_size; struct xe_gt *gt; struct xe_bo *bo; struct per_xecore_buf *xecore_buf; + struct delayed_work buf_poll_work; }; struct xe_eu_stall_gt { @@ -53,6 +59,8 @@ struct xe_eu_stall_gt { struct mutex stream_lock; /* EU stall data stream */ struct xe_eu_stall_data_stream *stream; + /* Workqueue to schedule buffer pointers polling work */ + struct workqueue_struct *buf_ptr_poll_wq; }; /** @@ -114,6 +122,7 @@ static void xe_eu_stall_fini(void *arg) { struct xe_gt *gt = arg; + destroy_workqueue(gt->eu_stall->buf_ptr_poll_wq); mutex_destroy(>->eu_stall->stream_lock); kfree(gt->eu_stall); } @@ -139,11 +148,19 @@ int xe_eu_stall_init(struct xe_gt *gt) mutex_init(>->eu_stall->stream_lock); + gt->eu_stall->buf_ptr_poll_wq = alloc_ordered_workqueue("xe_eu_stall", 0); + if (!gt->eu_stall->buf_ptr_poll_wq) { + ret = -ENOMEM; + goto exit_free; + } + ret = devm_add_action_or_reset(xe->drm.dev, xe_eu_stall_fini, gt); if (ret) - goto exit_free; + goto exit_destroy; return 0; +exit_destroy: + destroy_workqueue(gt->eu_stall->buf_ptr_poll_wq); exit_free: mutex_destroy(>->eu_stall->stream_lock); kfree(gt->eu_stall); @@ -248,6 +265,172 @@ static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, return 0; } +/** + * buf_data_size - Calculate the number of bytes in a circular buffer + * given the read and write pointers and the size of + * the buffer. + * + * @buf_size: Size of the circular buffer + * @read_ptr: Read pointer with an additional overflow bit + * @write_ptr: Write pointer with an additional overflow bit + * + * Since the read and write pointers have an additional overflow bit, + * this function calculates the offsets from the pointers and use the + * offsets to calculate the data size in the buffer. + * + * Returns: number of bytes of data in the buffer + */ +static u32 buf_data_size(size_t buf_size, u32 read_ptr, u32 write_ptr) +{ + u32 read_offset, write_offset, size = 0; + + if (read_ptr == write_ptr) + goto exit; + + read_offset = read_ptr & (buf_size - 1); + write_offset = write_ptr & (buf_size - 1); + + if (write_offset > read_offset) + size = write_offset - read_offset; + else + size = buf_size - read_offset + write_offset; +exit: + return size; +} + +/** + * eu_stall_data_buf_poll - Poll for EU stall data in the buffer. + * + * @stream: xe EU stall data stream instance + * + * Returns: true if the EU stall buffer contains minimum stall data as + * specified by the event report count, else false. + */ +static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) +{ + u32 read_ptr, write_ptr_reg, write_ptr, total_data = 0; + u32 buf_size = stream->per_xecore_buf_size; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + bool min_data_present = false; + u16 group, instance; + unsigned int xecore; + + mutex_lock(>->eu_stall->stream_lock); + for_each_dss_steering(xecore, gt, group, instance) { + xecore_buf = &stream->xecore_buf[xecore]; + read_ptr = xecore_buf->read; + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, + group, instance); + write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg); + write_ptr <<= 6; + write_ptr &= ((buf_size << 1) - 1); + if (!min_data_present) { + total_data += buf_data_size(buf_size, read_ptr, write_ptr); + if (num_data_rows(total_data) >= stream->wait_num_reports) + min_data_present = true; + } + xecore_buf->write = write_ptr; + } + mutex_unlock(>->eu_stall->stream_lock); + + return min_data_present; +} + +static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream, + char __user *buf, size_t count, + size_t *total_data_size, struct xe_gt *gt, + u16 group, u16 instance, unsigned int xecore) +{ + size_t read_data_size, copy_size, buf_size; + u32 read_ptr_reg, read_ptr, write_ptr; + u8 *xecore_start_vaddr, *read_vaddr; + struct per_xecore_buf *xecore_buf; + u32 read_offset, write_offset; + + /* Hardware increments the read and write pointers such that they can + * overflow into one additional bit. For example, a 256KB size buffer + * offset pointer needs 18 bits. But HW uses 19 bits for the read and + * write pointers. This technique avoids wasting a slot in the buffer. + * Read and write offsets are calculated from the pointers in order to + * check if the write pointer has wrapped around the array. + */ + xecore_buf = &stream->xecore_buf[xecore]; + xecore_start_vaddr = xecore_buf->vaddr; + read_ptr = xecore_buf->read; + write_ptr = xecore_buf->write; + buf_size = stream->per_xecore_buf_size; + + read_data_size = buf_data_size(buf_size, read_ptr, write_ptr); + /* Read only the data that the user space buffer can accommodate */ + read_data_size = min_t(size_t, count - *total_data_size, read_data_size); + if (read_data_size == 0) + return 0; + + read_offset = read_ptr & (buf_size - 1); + write_offset = write_ptr & (buf_size - 1); + read_vaddr = xecore_start_vaddr + read_offset; + + if (write_offset > read_offset) { + if (copy_to_user(buf + *total_data_size, read_vaddr, read_data_size)) + return -EFAULT; + } else { + if (read_data_size >= buf_size - read_offset) + copy_size = buf_size - read_offset; + else + copy_size = read_data_size; + if (copy_to_user(buf + *total_data_size, read_vaddr, copy_size)) + return -EFAULT; + if (copy_to_user(buf + *total_data_size + copy_size, + xecore_start_vaddr, read_data_size - copy_size)) + return -EFAULT; + } + + *total_data_size += read_data_size; + read_ptr += read_data_size; + + /* Read pointer can overflow into one additional bit */ + read_ptr &= (buf_size << 1) - 1; + read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, (read_ptr >> 6)); + read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg); + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance); + xecore_buf->read = read_ptr; + trace_xe_eu_stall_data_read(group, instance, read_ptr, write_ptr, + read_data_size, *total_data_size); + return 0; +} + +/** + * xe_eu_stall_stream_read_locked - copy EU stall counters data from the + * per xecore buffers to the userspace buffer + * @stream: A stream opened for EU stall count metrics + * @file: An xe EU stall data stream file + * @buf: destination buffer given by userspace + * @count: the number of bytes userspace wants to read + * + * Returns: Number of bytes copied or a negative error code + * If we've successfully copied any data then reporting that takes + * precedence over any internal error status, so the data isn't lost. + */ +static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *stream, + struct file *file, char __user *buf, + size_t count) +{ + struct xe_gt *gt = stream->gt; + size_t total_size = 0; + u16 group, instance; + unsigned int xecore; + int ret = 0; + + for_each_dss_steering(xecore, gt, group, instance) { + ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, + gt, group, instance, xecore); + if (ret || count == total_size) + break; + } + return total_size ?: (ret ?: -EAGAIN); +} + /* * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE * before calling read(). @@ -255,7 +438,41 @@ static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - ssize_t ret = 0; + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + ssize_t ret, aligned_count; + + aligned_count = ALIGN_DOWN(count, stream->data_record_size); + if (aligned_count == 0) + return -EINVAL; + + if (!stream->enabled) { + xe_gt_dbg(gt, "EU stall data stream not enabled to read\n"); + return -EINVAL; + } + + if (!(file->f_flags & O_NONBLOCK)) { + do { + ret = wait_event_interruptible(stream->poll_wq, stream->pollin); + if (ret) + return -EINTR; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_read_locked(stream, file, buf, aligned_count); + mutex_unlock(>->eu_stall->stream_lock); + } while (ret == -EAGAIN); + } else { + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_read_locked(stream, file, buf, aligned_count); + mutex_unlock(>->eu_stall->stream_lock); + } + + /* + * This may not work correctly if the user buffer is very small. + * We don't want to block the next read() when there is data in the buffer + * now, but couldn't be accommodated in the small user buffer. + */ + stream->pollin = false; return ret; } @@ -348,6 +565,21 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) return 0; } +static void eu_stall_data_buf_poll_work_fn(struct work_struct *work) +{ + struct xe_eu_stall_data_stream *stream = + container_of(work, typeof(*stream), buf_poll_work.work); + struct xe_gt *gt = stream->gt; + + if (eu_stall_data_buf_poll(stream)) { + stream->pollin = true; + wake_up(&stream->poll_wq); + } + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); +} + static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, struct eu_stall_open_properties *props) { @@ -372,6 +604,9 @@ static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, max_wait_num_reports); return -EINVAL; } + + init_waitqueue_head(&stream->poll_wq); + INIT_DELAYED_WORK(&stream->buf_poll_work, eu_stall_data_buf_poll_work_fn); stream->per_xecore_buf_size = per_xecore_buf_size; stream->sampling_rate_mult = props->sampling_rate_mult; stream->wait_num_reports = props->wait_num_reports; @@ -389,15 +624,35 @@ static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, return 0; } +static __poll_t xe_eu_stall_stream_poll_locked(struct xe_eu_stall_data_stream *stream, + struct file *file, poll_table *wait) +{ + __poll_t events = 0; + + poll_wait(file, &stream->poll_wq, wait); + + if (stream->pollin) + events |= EPOLLIN; + + return events; +} + static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) { - __poll_t ret = 0; + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + __poll_t ret; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_poll_locked(stream, file, wait); + mutex_unlock(>->eu_stall->stream_lock); return ret; } static int xe_eu_stall_enable_locked(struct xe_eu_stall_data_stream *stream) { + struct xe_gt *gt = stream->gt; int ret = 0; if (stream->enabled) @@ -406,6 +661,10 @@ static int xe_eu_stall_enable_locked(struct xe_eu_stall_data_stream *stream) stream->enabled = true; ret = xe_eu_stall_stream_enable(stream); + + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); return ret; } @@ -420,6 +679,8 @@ static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream) xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, 0); + cancel_delayed_work_sync(&stream->buf_poll_work); + xe_force_wake_put(gt_to_fw(gt), XE_FW_RENDER); xe_pm_runtime_put(gt_to_xe(gt)); diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index d5281de04d54..b4a3577df70c 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -427,6 +427,36 @@ DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_get_ioctl, TP_ARGS(xe, caller) ); +TRACE_EVENT(xe_eu_stall_data_read, + TP_PROTO(u8 slice, u8 subslice, + u32 read_ptr, u32 write_ptr, + size_t read_size, size_t total_size), + TP_ARGS(slice, subslice, + read_ptr, write_ptr, + read_size, total_size), + + TP_STRUCT__entry(__field(u8, slice) + __field(u8, subslice) + __field(u32, read_ptr) + __field(u32, write_ptr) + __field(size_t, read_size) + __field(size_t, total_size) + ), + + TP_fast_assign(__entry->slice = slice; + __entry->subslice = subslice; + __entry->read_ptr = read_ptr; + __entry->write_ptr = write_ptr; + __entry->read_size = read_size; + __entry->total_size = total_size; + ), + + TP_printk("slice: %u subslice: %u read ptr: 0x%x write ptr: 0x%x read size: %zu total read size: %zu", + __entry->slice, __entry->subslice, + __entry->read_ptr, __entry->write_ptr, + __entry->read_size, __entry->total_size) +); + #endif /* This part must be outside protection */ -- 2.51.0 From 9e0590eedede7bef999f9a4388243612456ffffd Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:09 -0800 Subject: [PATCH 04/16] drm/xe/eustall: Add support to handle dropped EU stall data If the user space doesn't read the EU stall data fast enough, it is possible that the EU stall data buffer can get filled, and if the hardware wants to write more data, it simply drops data due to unavailable buffer space. In that case, hardware sets a bit in a register. If the driver detects data drop, the driver read() returns -EIO error to let the user space know that HW has dropped data. The -EIO error is returned even if there is EU stall data in the buffer. A subsequent read by the user space returns the remaining EU stall data. v12: Move 'goto exit_drop;' to the next 'if (read_data_size == 0)' statement. v11: Clear drop bit even for empty data buffer as the data was read from the buffer in the previous read. v10: Reverted the changes back to v8: Clear the drop bits only after reading the data. v9: Move all data drop handling code to this patch Clear all drop data bits before returning -EIO. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/6fbfd7cfa42cb3ef5515b6412573d74c7cd3d27a.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 46 +++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index e82a6ebfe979..ebda54ea8871 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -51,6 +51,10 @@ struct xe_eu_stall_data_stream { struct xe_gt *gt; struct xe_bo *bo; struct per_xecore_buf *xecore_buf; + struct { + bool reported_to_user; + xe_dss_mask_t mask; + } data_drop; struct delayed_work buf_poll_work; }; @@ -330,6 +334,8 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) if (num_data_rows(total_data) >= stream->wait_num_reports) min_data_present = true; } + if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP) + set_bit(xecore, stream->data_drop.mask); xecore_buf->write = write_ptr; } mutex_unlock(>->eu_stall->stream_lock); @@ -337,6 +343,16 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) return min_data_present; } +static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) +{ + u32 write_ptr_reg; + + /* On PVC, the overflow bit has to be cleared by writing 1 to it. */ + write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); + + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, write_ptr_reg, group, instance); +} + static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream, char __user *buf, size_t count, size_t *total_data_size, struct xe_gt *gt, @@ -365,7 +381,7 @@ static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream, /* Read only the data that the user space buffer can accommodate */ read_data_size = min_t(size_t, count - *total_data_size, read_data_size); if (read_data_size == 0) - return 0; + goto exit_drop; read_offset = read_ptr & (buf_size - 1); write_offset = write_ptr & (buf_size - 1); @@ -397,6 +413,15 @@ static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream, xecore_buf->read = read_ptr; trace_xe_eu_stall_data_read(group, instance, read_ptr, write_ptr, read_data_size, *total_data_size); +exit_drop: + /* Clear drop bit (if set) after any data was read or if the buffer was empty. + * Drop bit can be set even if the buffer is empty as the buffer may have been emptied + * in the previous read() and the data drop bit was set during the previous read(). + */ + if (test_bit(xecore, stream->data_drop.mask)) { + clear_dropped_eviction_line_bit(gt, group, instance); + clear_bit(xecore, stream->data_drop.mask); + } return 0; } @@ -422,6 +447,16 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st unsigned int xecore; int ret = 0; + if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) { + if (!stream->data_drop.reported_to_user) { + stream->data_drop.reported_to_user = true; + xe_gt_dbg(gt, "EU stall data dropped in XeCores: %*pb\n", + XE_MAX_DSS_FUSE_BITS, stream->data_drop.mask); + return -EIO; + } + stream->data_drop.reported_to_user = false; + } + for_each_dss_steering(xecore, gt, group, instance) { ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, gt, group, instance, xecore); @@ -434,6 +469,9 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st /* * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE * before calling read(). + * + * Returns: The number of bytes copied or a negative error code on failure. + * -EIO if HW drops any EU stall data when the buffer is full. */ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) @@ -538,6 +576,9 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) for_each_dss_steering(xecore, gt, group, instance) { write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance); + /* Clear any drop bits set and not cleared in the previous session. */ + if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP) + clear_dropped_eviction_line_bit(gt, group, instance); write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg); read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, write_ptr); read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg); @@ -549,6 +590,9 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) xecore_buf->write = write_ptr; xecore_buf->read = write_ptr; } + stream->data_drop.reported_to_user = false; + bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS); + reg_value = _MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE, REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) | REG_FIELD_PREP(EUSTALL_SAMPLE_RATE, -- 2.51.0 From e827cf32eab449299f6062a8bca05f981c09ac50 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:10 -0800 Subject: [PATCH 05/16] drm/xe/eustall: Add EU stall sampling support for Xe2 Add EU stall sampling support for Xe2 architecture GPUs - LNL and BMG. EU stall data format for LNL and BMG is different from that of PVC. v10: Update comments as per review feedback v9: Use GRAPHICS_VER() check instead of platform v8: Renamed struct drm_xe_eu_stall_data_xe2 to struct xe_eu_stall_data_xe2 since it is a local structure. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/d85093e9ab1204d14d2cc783f304a4bc8688951c.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 35 +++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index ebda54ea8871..87f978373bd0 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -99,12 +99,35 @@ struct xe_eu_stall_data_pvc { __u64 unused[6]; } __packed; +/* + * EU stall data format for Xe2 arch GPUs (LNL, BMG). + */ +struct xe_eu_stall_data_xe2 { + __u64 ip_addr:29; /* Bits 0 to 28 */ + __u64 tdr_count:8; /* Bits 29 to 36 */ + __u64 other_count:8; /* Bits 37 to 44 */ + __u64 control_count:8; /* Bits 45 to 52 */ + __u64 pipestall_count:8; /* Bits 53 to 60 */ + __u64 send_count:8; /* Bits 61 to 68 */ + __u64 dist_acc_count:8; /* Bits 69 to 76 */ + __u64 sbid_count:8; /* Bits 77 to 84 */ + __u64 sync_count:8; /* Bits 85 to 92 */ + __u64 inst_fetch_count:8; /* Bits 93 to 100 */ + __u64 active_count:8; /* Bits 101 to 108 */ + __u64 ex_id:3; /* Bits 109 to 111 */ + __u64 end_flag:1; /* Bit 112 */ + __u64 unused_bits:15; + __u64 unused[6]; +} __packed; + static size_t xe_eu_stall_data_record_size(struct xe_device *xe) { size_t record_size = 0; if (xe->info.platform == XE_PVC) record_size = sizeof(struct xe_eu_stall_data_pvc); + else if (GRAPHICS_VER(xe) >= 20) + record_size = sizeof(struct xe_eu_stall_data_xe2); xe_assert(xe, is_power_of_2(record_size)); @@ -345,10 +368,16 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) { + struct xe_device *xe = gt_to_xe(gt); u32 write_ptr_reg; - /* On PVC, the overflow bit has to be cleared by writing 1 to it. */ - write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); + /* On PVC, the overflow bit has to be cleared by writing 1 to it. + * On Xe2 and later GPUs, the bit has to be cleared by writing 0 to it. + */ + if (GRAPHICS_VER(xe) >= 20) + write_ptr_reg = _MASKED_BIT_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); + else + write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, write_ptr_reg, group, instance); } @@ -785,7 +814,7 @@ static const struct file_operations fops_eu_stall = { static inline bool has_eu_stall_sampling_support(struct xe_device *xe) { - return xe->info.platform == XE_PVC; + return xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20; } static int xe_eu_stall_stream_open_locked(struct drm_device *dev, -- 2.51.0 From cd5bbb2532f276626d1416b237676772bb61f11e Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:11 -0800 Subject: [PATCH 06/16] drm/xe/uapi: Add a device query to get EU stall sampling information User space can get the EU stall data record size, EU stall capabilities, EU stall sampling rates, and per XeCore buffer size with query IOCTL DRM_IOCTL_XE_DEVICE_QUERY with .query set to DRM_XE_DEVICE_QUERY_EU_STALL. A struct drm_xe_query_eu_stall will be returned to the user space along with an array of supported sampling rates sorted in the fastest sampling rate first order. sampling_rates in struct drm_xe_query_eu_stall will point to the array of sampling rates. Any capabilities in EU stall sampling as of this patch are considered as base capabilities. New capability bits will be added for any new functionality added later. v12: Rename has_eu_stall_sampling_support() to xe_eu_stall_supported_on_platform() and move it to header file. v11: Check if EU stall sampling is supported on the platform. v10: Change comments and variable names as per feedback v9: Move reserved fields above num_sampling_rates in struct drm_xe_query_eu_stall. v7: Change sampling_rates from a pointer to flexible array. v6: Include EU stall sampling rates information and per XeCore buffer size in the query information. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/67ba42796a5a99d648239c315694cd222812a49b.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 48 +++++++++++++++++++++++++++----- drivers/gpu/drm/xe/xe_eu_stall.h | 9 ++++++ drivers/gpu/drm/xe/xe_query.c | 43 ++++++++++++++++++++++++++++ include/uapi/drm/xe_drm.h | 40 ++++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 87f978373bd0..54a0d1f6a491 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -120,7 +120,46 @@ struct xe_eu_stall_data_xe2 { __u64 unused[6]; } __packed; -static size_t xe_eu_stall_data_record_size(struct xe_device *xe) +const u64 eu_stall_sampling_rates[] = {251, 251 * 2, 251 * 3, 251 * 4, 251 * 5, 251 * 6, 251 * 7}; + +/** + * xe_eu_stall_get_sampling_rates - get EU stall sampling rates information. + * + * @num_rates: Pointer to a u32 to return the number of sampling rates. + * @rates: double u64 pointer to point to an array of sampling rates. + * + * Stores the number of sampling rates and pointer to the array of + * sampling rates in the input pointers. + * + * Returns: Size of the EU stall sampling rates array. + */ +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates) +{ + *num_rates = ARRAY_SIZE(eu_stall_sampling_rates); + *rates = eu_stall_sampling_rates; + + return sizeof(eu_stall_sampling_rates); +} + +/** + * xe_eu_stall_get_per_xecore_buf_size - get per XeCore buffer size. + * + * Returns: The per XeCore buffer size used to allocate the per GT + * EU stall data buffer. + */ +size_t xe_eu_stall_get_per_xecore_buf_size(void) +{ + return per_xecore_buf_size; +} + +/** + * xe_eu_stall_data_record_size - get EU stall data record size. + * + * @xe: Pointer to a Xe device. + * + * Returns: EU stall data record size. + */ +size_t xe_eu_stall_data_record_size(struct xe_device *xe) { size_t record_size = 0; @@ -812,11 +851,6 @@ static const struct file_operations fops_eu_stall = { .compat_ioctl = xe_eu_stall_stream_ioctl, }; -static inline bool has_eu_stall_sampling_support(struct xe_device *xe) -{ - return xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20; -} - static int xe_eu_stall_stream_open_locked(struct drm_device *dev, struct eu_stall_open_properties *props, struct drm_file *file) @@ -885,7 +919,7 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f struct eu_stall_open_properties props = {}; int ret; - if (!has_eu_stall_sampling_support(xe)) { + if (!xe_eu_stall_supported_on_platform(xe)) { drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); return -ENODEV; } diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h index 24e215b840c0..ed9d0f233566 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.h +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -8,8 +8,17 @@ #include "xe_gt_types.h" +size_t xe_eu_stall_get_per_xecore_buf_size(void); +size_t xe_eu_stall_data_record_size(struct xe_device *xe); +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates); + int xe_eu_stall_init(struct xe_gt *gt); int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file); + +static inline bool xe_eu_stall_supported_on_platform(struct xe_device *xe) +{ + return xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20; +} #endif diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index ebfae746f861..781dd21682e5 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -16,6 +16,7 @@ #include "regs/xe_gt_regs.h" #include "xe_bo.h" #include "xe_device.h" +#include "xe_eu_stall.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" #include "xe_ggtt.h" @@ -729,6 +730,47 @@ static int query_pxp_status(struct xe_device *xe, struct drm_xe_device_query *qu return 0; } +static int query_eu_stall(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + struct drm_xe_query_eu_stall *info; + size_t size, array_size; + const u64 *rates; + u32 num_rates; + int ret; + + if (!xe_eu_stall_supported_on_platform(xe)) { + drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + return -ENODEV; + } + + array_size = xe_eu_stall_get_sampling_rates(&num_rates, &rates); + size = sizeof(struct drm_xe_query_eu_stall) + array_size; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + info = kzalloc(size, GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->num_sampling_rates = num_rates; + info->capabilities = DRM_XE_EU_STALL_CAPS_BASE; + info->record_size = xe_eu_stall_data_record_size(xe); + info->per_xecore_buf_size = xe_eu_stall_get_per_xecore_buf_size(); + memcpy(info->sampling_rates, rates, array_size); + + ret = copy_to_user(query_ptr, info, size); + kfree(info); + + return ret ? -EFAULT : 0; +} + static int (* const xe_query_funcs[])(struct xe_device *xe, struct drm_xe_device_query *query) = { query_engines, @@ -741,6 +783,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe, query_uc_fw_version, query_oa_units, query_pxp_status, + query_eu_stall, }; int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 95cb9e65540b..76a462fae05f 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -735,6 +735,7 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_UC_FW_VERSION 7 #define DRM_XE_DEVICE_QUERY_OA_UNITS 8 #define DRM_XE_DEVICE_QUERY_PXP_STATUS 9 +#define DRM_XE_DEVICE_QUERY_EU_STALL 10 /** @query: The type of data to query */ __u32 query; @@ -1873,8 +1874,8 @@ enum drm_xe_eu_stall_property_id { DRM_XE_EU_STALL_PROP_GT_ID = 1, /** - * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate - * in GPU cycles. + * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate in + * GPU cycles from @sampling_rates in struct @drm_xe_query_eu_stall */ DRM_XE_EU_STALL_PROP_SAMPLE_RATE, @@ -1886,6 +1887,41 @@ enum drm_xe_eu_stall_property_id { DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS, }; +/** + * struct drm_xe_query_eu_stall - Information about EU stall sampling. + * + * If a query is made with a struct @drm_xe_device_query where .query + * is equal to @DRM_XE_DEVICE_QUERY_EU_STALL, then the reply uses + * struct @drm_xe_query_eu_stall in .data. + */ +struct drm_xe_query_eu_stall { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @capabilities: EU stall capabilities bit-mask */ + __u64 capabilities; +#define DRM_XE_EU_STALL_CAPS_BASE (1 << 0) + + /** @record_size: size of each EU stall data record */ + __u64 record_size; + + /** @per_xecore_buf_size: internal per XeCore buffer size */ + __u64 per_xecore_buf_size; + + /** @reserved: Reserved */ + __u64 reserved[5]; + + /** @num_sampling_rates: Number of sampling rates in @sampling_rates array */ + __u64 num_sampling_rates; + + /** + * @sampling_rates: Flexible array of sampling rates + * sorted in the fastest to slowest order. + * Sampling rates are specified in GPU clock cycles. + */ + __u64 sampling_rates[]; +}; + #if defined(__cplusplus) } #endif -- 2.51.0 From e67a35bc953a873881a2a8c1ea27ede5870f5f42 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:12 -0800 Subject: [PATCH 07/16] drm/xe/eustall: Add workaround 22016596838 which applies to PVC. Add PVC workaround 22016596838 that disables EU DOP gating during EU stall sampling. Signed-off-by: Harish Chegondi Reviewed-by: Ashutosh Dixit Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/062a12ed9e110fea420cd47cb70fb10136ee9132.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 10 ++++++++++ drivers/gpu/drm/xe/xe_wa_oob.rules | 1 + 2 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 54a0d1f6a491..88a92baf5c95 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -9,6 +9,7 @@ #include #include +#include #include #include "xe_bo.h" @@ -22,6 +23,7 @@ #include "xe_observation.h" #include "xe_pm.h" #include "xe_trace.h" +#include "xe_wa.h" #include "regs/xe_eu_stall_regs.h" #include "regs/xe_gt_regs.h" @@ -642,6 +644,10 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) return -ETIMEDOUT; } + if (XE_WA(gt, 22016596838)) + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DISABLE_DOP_GATING)); + for_each_dss_steering(xecore, gt, group, instance) { write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance); /* Clear any drop bits set and not cleared in the previous session. */ @@ -793,6 +799,10 @@ static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream) cancel_delayed_work_sync(&stream->buf_poll_work); + if (XE_WA(gt, 22016596838)) + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, + _MASKED_BIT_DISABLE(DISABLE_DOP_GATING)); + xe_force_wake_put(gt_to_fw(gt), XE_FW_RENDER); xe_pm_runtime_put(gt_to_xe(gt)); diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 1dd02a231926..e0c5fa460487 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -5,6 +5,7 @@ 22011391025 PLATFORM(DG2) 22012727170 SUBPLATFORM(DG2, G11) 22012727685 SUBPLATFORM(DG2, G11) +22016596838 PLATFORM(PVC) 18020744125 PLATFORM(PVC) 1509372804 PLATFORM(PVC), GRAPHICS_STEP(A0, C0) 1409600907 GRAPHICS_VERSION_RANGE(1200, 1250) -- 2.51.0 From a33c9699e73456d08182ad7b87a4af52ac24f779 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Tue, 25 Feb 2025 15:53:28 -0800 Subject: [PATCH 08/16] drm/xe/pxp: Don't kill queues while holding PXP locks xe_exec_queue_kill can sleep, so we can't call it from under a spinlock. We can instead move the queues to a separate list and then kill them all after we release the spinlock. Furthermore, xe_exec_queue_kill can take the VM lock so we can't call it while holding the PXP mutex because the mutex is taken under VM lock at queue creation time. Note that while it is safe to call the kill without holding the mutex, we must call it after the PXP state has been updated, otherwise an app might be able to create a queue between the invalidation and the state update, which would break the state machine. Since being in the list is used to track whether RPM cleanup is needed, we can no longer defer that to queue_destroy, so we perform it immediately instead. v2: also avoid calling kill() under pxp->mutex. Reported-by: Dan Carpenter Closes: https://lore.kernel.org/intel-xe/34aaced9-4a9d-4e8c-900a-b8f73452e35c@stanley.mountain/ Fixes: f8caa80154c4 ("drm/xe/pxp: Add PXP queue tracking and session start") Signed-off-by: Daniele Ceraolo Spurio Cc: John Harrison Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://patchwork.freedesktop.org/patch/msgid/20250225235328.2895877-1-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/xe/xe_pxp.c | 78 +++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 3cd3f83e86b0..47499ca02693 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -132,14 +132,6 @@ static int pxp_wait_for_session_state(struct xe_pxp *pxp, u32 id, bool in_play) static void pxp_invalidate_queues(struct xe_pxp *pxp); -static void pxp_invalidate_state(struct xe_pxp *pxp) -{ - pxp_invalidate_queues(pxp); - - if (pxp->status == XE_PXP_ACTIVE) - pxp->key_instance++; -} - static int pxp_terminate_hw(struct xe_pxp *pxp) { struct xe_gt *gt = pxp->gt; @@ -193,7 +185,8 @@ static void pxp_terminate(struct xe_pxp *pxp) mutex_lock(&pxp->mutex); - pxp_invalidate_state(pxp); + if (pxp->status == XE_PXP_ACTIVE) + pxp->key_instance++; /* * we'll mark the status as needing termination on resume, so no need to @@ -220,6 +213,8 @@ static void pxp_terminate(struct xe_pxp *pxp) mutex_unlock(&pxp->mutex); + pxp_invalidate_queues(pxp); + ret = pxp_terminate_hw(pxp); if (ret) { drm_err(&xe->drm, "PXP termination failed: %pe\n", ERR_PTR(ret)); @@ -665,23 +660,15 @@ out: return ret; } -/** - * xe_pxp_exec_queue_remove - remove a queue from the PXP list - * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) - * @q: the queue to remove from the list - * - * If PXP is enabled and the exec_queue is in the list, the queue will be - * removed from the list and its PM reference will be released. It is safe to - * call this function multiple times for the same queue. - */ -void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) +static void __pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q, bool lock) { bool need_pm_put = false; if (!xe_pxp_is_enabled(pxp)) return; - spin_lock_irq(&pxp->queues.lock); + if (lock) + spin_lock_irq(&pxp->queues.lock); if (!list_empty(&q->pxp.link)) { list_del_init(&q->pxp.link); @@ -690,36 +677,54 @@ void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) q->pxp.type = DRM_XE_PXP_TYPE_NONE; - spin_unlock_irq(&pxp->queues.lock); + if (lock) + spin_unlock_irq(&pxp->queues.lock); if (need_pm_put) xe_pm_runtime_put(pxp->xe); } +/** + * xe_pxp_exec_queue_remove - remove a queue from the PXP list + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @q: the queue to remove from the list + * + * If PXP is enabled and the exec_queue is in the list, the queue will be + * removed from the list and its PM reference will be released. It is safe to + * call this function multiple times for the same queue. + */ +void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) +{ + __pxp_exec_queue_remove(pxp, q, true); +} + static void pxp_invalidate_queues(struct xe_pxp *pxp) { struct xe_exec_queue *tmp, *q; + LIST_HEAD(to_clean); spin_lock_irq(&pxp->queues.lock); - /* - * Removing a queue from the PXP list requires a put of the RPM ref that - * the queue holds to keep the PXP session alive, which can't be done - * under spinlock. Since it is safe to kill a queue multiple times, we - * can leave the invalid queue in the list for now and postpone the - * removal and associated RPM put to when the queue is destroyed. - */ - list_for_each_entry(tmp, &pxp->queues.list, pxp.link) { - q = xe_exec_queue_get_unless_zero(tmp); - + list_for_each_entry_safe(q, tmp, &pxp->queues.list, pxp.link) { + q = xe_exec_queue_get_unless_zero(q); if (!q) continue; + list_move_tail(&q->pxp.link, &to_clean); + } + spin_unlock_irq(&pxp->queues.lock); + + list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) { xe_exec_queue_kill(q); + + /* + * We hold a ref to the queue so there is no risk of racing with + * the calls to exec_queue_remove coming from exec_queue_destroy. + */ + __pxp_exec_queue_remove(pxp, q, false); + xe_exec_queue_put(q); } - - spin_unlock_irq(&pxp->queues.lock); } /** @@ -816,6 +821,7 @@ int xe_pxp_obj_key_check(struct xe_pxp *pxp, struct drm_gem_object *obj) */ int xe_pxp_pm_suspend(struct xe_pxp *pxp) { + bool needs_queue_inval = false; int ret = 0; if (!xe_pxp_is_enabled(pxp)) @@ -848,7 +854,8 @@ wait_for_activation: break; fallthrough; case XE_PXP_ACTIVE: - pxp_invalidate_state(pxp); + pxp->key_instance++; + needs_queue_inval = true; break; default: drm_err(&pxp->xe->drm, "unexpected state during PXP suspend: %u", @@ -865,6 +872,9 @@ wait_for_activation: mutex_unlock(&pxp->mutex); + if (needs_queue_inval) + pxp_invalidate_queues(pxp); + /* * if there is a termination in progress, wait for it. * We need to wait outside the lock because the completion is done from -- 2.51.0 From 18778b5fdd018baa6eb492166d04605b39030e1d Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 25 Feb 2025 14:49:09 -0800 Subject: [PATCH 09/16] drm/xe: Eliminate usage of TIMESTAMP_OVERRIDE Recent discussions with the hardware architects have revealed that the TIMESTAMP_OVERRIDE register is never expected to hold a valid/useful value on production hardware. That register would only get used by hardware workarounds (although there are none that use it today) or during early internal hardware testing. Due to lack of documentation it's not clear exactly what the driver should be doing if CTC_MODE[0] is set (or even whether that's a setting that would ever be encountered on real hardware), but it's definitely not what Xe and i915 have been doing. So drop the incorrect code trying to use TIMESTAMP_REGISTER. If the driver does encounter CTC_MODE[0] in the wild, we'll print a warning and just continue trying to use the crystal clock frequency since that's probably less incorrect than what we're doing today. Cc: Lucas De Marchi Cc: Balasubramani Vivekanandan Reviewed-by: Lucas De Marchi Link: https://patchwork.freedesktop.org/patch/msgid/20250225224908.1671554-2-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/regs/xe_regs.h | 4 -- drivers/gpu/drm/xe/xe_gt_clock.c | 57 +++++++++------------ drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c | 5 -- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_regs.h b/drivers/gpu/drm/xe/regs/xe_regs.h index 6cf282618836..3abb17d2ca33 100644 --- a/drivers/gpu/drm/xe/regs/xe_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_regs.h @@ -7,10 +7,6 @@ #include "regs/xe_reg_defs.h" -#define TIMESTAMP_OVERRIDE XE_REG(0x44074) -#define TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK REG_GENMASK(15, 12) -#define TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK REG_GENMASK(9, 0) - #define GU_CNTL_PROTECTED XE_REG(0x10100C) #define DRIVERINT_FLR_DIS REG_BIT(31) diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c index cc2ae159298e..2a958c92d8ea 100644 --- a/drivers/gpu/drm/xe/xe_gt_clock.c +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -12,25 +12,10 @@ #include "xe_assert.h" #include "xe_device.h" #include "xe_gt.h" +#include "xe_gt_printk.h" #include "xe_macros.h" #include "xe_mmio.h" -static u32 read_reference_ts_freq(struct xe_gt *gt) -{ - u32 ts_override = xe_mmio_read32(>->mmio, TIMESTAMP_OVERRIDE); - u32 base_freq, frac_freq; - - base_freq = REG_FIELD_GET(TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK, - ts_override) + 1; - base_freq *= 1000000; - - frac_freq = REG_FIELD_GET(TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK, - ts_override); - frac_freq = 1000000 / (frac_freq + 1); - - return base_freq + frac_freq; -} - static u32 get_crystal_clock_freq(u32 rpm_config_reg) { const u32 f19_2_mhz = 19200000; @@ -57,26 +42,30 @@ static u32 get_crystal_clock_freq(u32 rpm_config_reg) int xe_gt_clock_init(struct xe_gt *gt) { - u32 ctc_reg = xe_mmio_read32(>->mmio, CTC_MODE); + u32 c0 = xe_mmio_read32(>->mmio, RPM_CONFIG0); u32 freq = 0; - /* Assuming gen11+ so assert this assumption is correct */ - xe_gt_assert(gt, GRAPHICS_VER(gt_to_xe(gt)) >= 11); - - if (ctc_reg & CTC_SOURCE_DIVIDE_LOGIC) { - freq = read_reference_ts_freq(gt); - } else { - u32 c0 = xe_mmio_read32(>->mmio, RPM_CONFIG0); - - freq = get_crystal_clock_freq(c0); - - /* - * Now figure out how the command stream's timestamp - * register increments from this frequency (it might - * increment only every few clock cycle). - */ - freq >>= 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, c0); - } + /* + * CTC_MODE[0] = 1 is definitely not supported for Xe2 and later + * platforms. In theory it could be a valid setting for pre-Xe2 + * platforms, but there's no documentation on how to properly handle + * this case. Reading TIMESTAMP_OVERRIDE, as the driver attempted in + * the past has been confirmed as incorrect by the hardware architects. + * + * For now just warn if we ever encounter hardware in the wild that + * has this setting and move on as if it hadn't been set. + */ + if (xe_mmio_read32(>->mmio, CTC_MODE) & CTC_SOURCE_DIVIDE_LOGIC) + xe_gt_warn(gt, "CTC_MODE[0] is set; this is unexpected and undocumented\n"); + + freq = get_crystal_clock_freq(c0); + + /* + * Now figure out how the command stream's timestamp + * register increments from this frequency (it might + * increment only every few clock cycle). + */ + freq >>= 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, c0); gt->info.reference_clock = freq; return 0; diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c index 6b5f849a0722..4efde5f46b43 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c @@ -114,7 +114,6 @@ static const struct xe_reg tgl_runtime_regs[] = { GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ CTC_MODE, /* _MMIO(0xa26c) */ HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ - TIMESTAMP_OVERRIDE, /* _MMIO(0x44074) */ }; static const struct xe_reg ats_m_runtime_regs[] = { @@ -127,7 +126,6 @@ static const struct xe_reg ats_m_runtime_regs[] = { XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ CTC_MODE, /* _MMIO(0xa26c) */ HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ - TIMESTAMP_OVERRIDE, /* _MMIO(0x44074) */ }; static const struct xe_reg pvc_runtime_regs[] = { @@ -140,7 +138,6 @@ static const struct xe_reg pvc_runtime_regs[] = { XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ CTC_MODE, /* _MMIO(0xA26C) */ HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ - TIMESTAMP_OVERRIDE, /* _MMIO(0x44074) */ }; static const struct xe_reg ver_1270_runtime_regs[] = { @@ -155,7 +152,6 @@ static const struct xe_reg ver_1270_runtime_regs[] = { XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ CTC_MODE, /* _MMIO(0xa26c) */ HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ - TIMESTAMP_OVERRIDE, /* _MMIO(0x44074) */ }; static const struct xe_reg ver_2000_runtime_regs[] = { @@ -173,7 +169,6 @@ static const struct xe_reg ver_2000_runtime_regs[] = { XE2_GT_GEOMETRY_DSS_2, /* _MMIO(0x9154) */ CTC_MODE, /* _MMIO(0xa26c) */ HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ - TIMESTAMP_OVERRIDE, /* _MMIO(0x44074) */ }; static const struct xe_reg ver_3000_runtime_regs[] = { -- 2.51.0 From dd8c01e42f4c5c1eaf02f003d7d588ba6706aa71 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 26 Feb 2025 17:47:49 +0000 Subject: [PATCH 10/16] drm/xe/userptr: properly setup pfn_flags_mask MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Currently we just leave it uninitialised, which at first looks harmless, however we also don't zero out the pfn array, and with pfn_flags_mask the idea is to be able set individual flags for a given range of pfn or completely ignore them, outside of default_flags. So here we end up with pfn[i] & pfn_flags_mask, and if both are uninitialised we might get back an unexpected flags value, like asking for read only with default_flags, but getting back write on top, leading to potentially bogus behaviour. To fix this ensure we zero the pfn_flags_mask, such that hmm only considers the default_flags and not also the initial pfn[i] value. v2 (Thomas): - Prefer proper initializer. Fixes: 81e058a3e7fd ("drm/xe: Introduce helper to populate userptr") Signed-off-by: Matthew Auld Cc: Matthew Brost Cc: Thomas Hellström Cc: # v6.10+ Reviewed-by: Thomas Hellström Reviewed-by: Tejas Upadhyay Link: https://patchwork.freedesktop.org/patch/msgid/20250226174748.294285-2-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_hmm.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hmm.c b/drivers/gpu/drm/xe/xe_hmm.c index 089834467880..2e4ae61567d8 100644 --- a/drivers/gpu/drm/xe/xe_hmm.c +++ b/drivers/gpu/drm/xe/xe_hmm.c @@ -166,13 +166,20 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, { unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - unsigned long *pfns, flags = HMM_PFN_REQ_FAULT; + unsigned long *pfns; struct xe_userptr *userptr; struct xe_vma *vma = &uvma->vma; u64 userptr_start = xe_vma_userptr(vma); u64 userptr_end = userptr_start + xe_vma_size(vma); struct xe_vm *vm = xe_vma_vm(vma); - struct hmm_range hmm_range; + struct hmm_range hmm_range = { + .pfn_flags_mask = 0, /* ignore pfns */ + .default_flags = HMM_PFN_REQ_FAULT, + .start = userptr_start, + .end = userptr_end, + .notifier = &uvma->userptr.notifier, + .dev_private_owner = vm->xe, + }; bool write = !xe_vma_read_only(vma); unsigned long notifier_seq; u64 npages; @@ -199,19 +206,14 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, return -ENOMEM; if (write) - flags |= HMM_PFN_REQ_WRITE; + hmm_range.default_flags |= HMM_PFN_REQ_WRITE; if (!mmget_not_zero(userptr->notifier.mm)) { ret = -EFAULT; goto free_pfns; } - hmm_range.default_flags = flags; hmm_range.hmm_pfns = pfns; - hmm_range.notifier = &userptr->notifier; - hmm_range.start = userptr_start; - hmm_range.end = userptr_end; - hmm_range.dev_private_owner = vm->xe; while (true) { hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier); -- 2.51.0 From 89eb42b5539f6ae6a0cabcb39e5b6fcc83c106a1 Mon Sep 17 00:00:00 2001 From: Daniel Gomez Date: Mon, 24 Feb 2025 07:23:13 +0100 Subject: [PATCH 11/16] drm/xe: xe_gen_wa_oob: replace program_invocation_short_name program_invocation_short_name may not be available in other systems. Instead, replace it with the argv[0] to pass the executable name. Fixes build error when program_invocation_short_name is not available: drivers/gpu/drm/xe/xe_gen_wa_oob.c:34:3: error: use of undeclared identifier 'program_invocation_short_name' 34 | program_invocation_short_name); | ^ 1 error generated. Suggested-by: Masahiro Yamada Signed-off-by: Daniel Gomez Reviewed-by: Lucas De Marchi Link: https://patchwork.freedesktop.org/patch/msgid/20250224-macos-build-support-xe-v3-1-d2c9ed3a27cc@samsung.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_gen_wa_oob.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gen_wa_oob.c b/drivers/gpu/drm/xe/xe_gen_wa_oob.c index 904cf47925aa..ed9183599e31 100644 --- a/drivers/gpu/drm/xe/xe_gen_wa_oob.c +++ b/drivers/gpu/drm/xe/xe_gen_wa_oob.c @@ -28,10 +28,10 @@ "\n" \ "#endif\n" -static void print_usage(FILE *f) +static void print_usage(FILE *f, const char *progname) { fprintf(f, "usage: %s \n", - program_invocation_short_name); + progname); } static void print_parse_error(const char *err_msg, const char *line, @@ -144,7 +144,7 @@ int main(int argc, const char *argv[]) if (argc < 3) { fprintf(stderr, "ERROR: wrong arguments\n"); - print_usage(stderr); + print_usage(stderr, argv[0]); return 1; } -- 2.51.0 From 2399bcc07c01189737858e0a88ac4ffdd1d4b03d Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Fri, 21 Feb 2025 16:52:00 +0530 Subject: [PATCH 12/16] drm/xe/xe3lpg: Add Wa_13012615864 Wa_13012615864 applies to xe3lpg Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20250221112200.388612-1-tejas.upadhyay@intel.com Signed-off-by: Tejas Upadhyay --- drivers/gpu/drm/xe/regs/xe_gt_regs.h | 1 + drivers/gpu/drm/xe/xe_wa.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 124cc398798e..d08dd437172f 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -480,6 +480,7 @@ #define TDL_TSL_CHICKEN XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED) #define STK_ID_RESTRICT REG_BIT(12) #define SLM_WMTP_RESTORE REG_BIT(11) +#define RES_CHK_SPR_DIS REG_BIT(6) #define ROW_CHICKEN XE_REG_MCR(0xe4f0, XE_REG_OPTION_MASKED) #define UGM_BACKUP_MODE REG_BIT(13) diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index d4982799383c..b797c863bbe5 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -619,6 +619,11 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(TDL_CHICKEN, QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE)) }, + { XE_RTP_NAME("13012615864"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS)) + }, {} }; -- 2.51.0 From 8c0aff7d92e2be25717669eb65a81a89740a24f2 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 24 Feb 2025 15:58:06 +0530 Subject: [PATCH 13/16] drm/xe/pf: Create a link between PF and VF devices MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When both PF and VF devices are enabled on the host, they resume simultaneously during system resume. However, the PF must finish provisioning the VF before any VFs can successfully resume. Establish a parent-child device link between the PF and VF devices to ensure the correct order of resumption. V4 -> V5: - Added missing break in the error condition. V3 -> V4: - Made xe_pci_pf_get_vf_dev() as a static function and updated input parameter types. - Updated xe_sriov_warn() to xe_sriov_abort() when VF device cannot be found. V2 -> V3: - Added function documentation for xe_pci_pf_get_vf_dev(). - Added assertion if not called from PF. V1 -> V2: - Added a helper function to get VF pci_dev. - Updated xe_sriov_notice() to xe_sriov_warn() if vf pci_dev is not found. Signed-off-by: Satyanarayana K V P Cc: Michał Wajdeczko Cc: Michał Winiarski Cc: Piotr Piórkowski Reviewed-by: Piotr Piorkowski Signed-off-by: Michal Wajdeczko Link: https://patchwork.freedesktop.org/patch/msgid/20250224102807.11065-2-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_pci_sriov.c | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index aaceee748287..09ee8a06fe2e 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -62,6 +62,55 @@ static void pf_reset_vfs(struct xe_device *xe, unsigned int num_vfs) xe_gt_sriov_pf_control_trigger_flr(gt, n); } +static struct pci_dev *xe_pci_pf_get_vf_dev(struct xe_device *xe, unsigned int vf_id) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + xe_assert(xe, IS_SRIOV_PF(xe)); + + /* caller must use pci_dev_put() */ + return pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), + pdev->bus->number, + pci_iov_virtfn_devfn(pdev, vf_id)); +} + +static void pf_link_vfs(struct xe_device *xe, int num_vfs) +{ + struct pci_dev *pdev_pf = to_pci_dev(xe->drm.dev); + struct device_link *link; + struct pci_dev *pdev_vf; + unsigned int n; + + /* + * When both PF and VF devices are enabled on the host, during system + * resume they are resuming in parallel. + * + * But PF has to complete the provision of VF first to allow any VFs to + * successfully resume. + * + * Create a parent-child device link between PF and VF devices that will + * enforce correct resume order. + */ + for (n = 1; n <= num_vfs; n++) { + pdev_vf = xe_pci_pf_get_vf_dev(xe, n - 1); + + /* unlikely, something weird is happening, abort */ + if (!pdev_vf) { + xe_sriov_err(xe, "Cannot find VF%u device, aborting link%s creation!\n", + n, str_plural(num_vfs)); + break; + } + + link = device_link_add(&pdev_vf->dev, &pdev_pf->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + /* unlikely and harmless, continue with other VFs */ + if (!link) + xe_sriov_notice(xe, "Failed linking VF%u\n", n); + + pci_dev_put(pdev_vf); + } +} + static int pf_enable_vfs(struct xe_device *xe, int num_vfs) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); @@ -92,6 +141,8 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) if (err < 0) goto failed; + pf_link_vfs(xe, num_vfs); + xe_sriov_info(xe, "Enabled %u of %u VF%s\n", num_vfs, total_vfs, str_plural(total_vfs)); return num_vfs; -- 2.51.0 From ba757a65d2a28d46a8ccf50538f4f05036983f1b Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 24 Feb 2025 15:58:07 +0530 Subject: [PATCH 14/16] drm/xe/vf: Retry sending MMIO request to GUC on timeout error MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Add support to allow retrying the sending of MMIO requests from the VF to the GUC in the event of an error. During the suspend/resume process, VFs begin resuming only after the PF has resumed. Although the PF resumes, the GUC reset and provisioning occur later in a separate worker process. When there are a large number of VFs, some may attempt to resume before the PF has completed its provisioning. Therefore, if a MMIO request from a VF fails during this period, we will retry sending the request up to GUC_RESET_VF_STATE_RETRY_MAX times, which is set to a maximum of 10 attempts. Signed-off-by: Satyanarayana K V P Cc: Michał Wajdeczko Cc: Michał Winiarski Cc: Piotr Piórkowski Reviewed-by: Piotr Piorkowski Signed-off-by: Michal Wajdeczko Link: https://patchwork.freedesktop.org/patch/msgid/20250224102807.11065-3-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 4831549da319..a439261bf4d7 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -47,12 +47,19 @@ static int guc_action_vf_reset(struct xe_guc *guc) return ret > 0 ? -EPROTO : ret; } +#define GUC_RESET_VF_STATE_RETRY_MAX 10 static int vf_reset_guc_state(struct xe_gt *gt) { + unsigned int retry = GUC_RESET_VF_STATE_RETRY_MAX; struct xe_guc *guc = >->uc.guc; int err; - err = guc_action_vf_reset(guc); + do { + err = guc_action_vf_reset(guc); + if (!err || err != -ETIMEDOUT) + break; + } while (--retry); + if (unlikely(err)) xe_gt_sriov_err(gt, "Failed to reset GuC state (%pe)\n", ERR_PTR(err)); return err; -- 2.51.0 From 25d434cef791e03cf40680f5441b576c639bfa84 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Thu, 27 Feb 2025 10:13:00 +0000 Subject: [PATCH 15/16] drm/xe: Fix GT "for each engine" workarounds Any rules using engine matching are currently broken due RTP processing happening too in early init, before the list of hardware engines has been initialised. Fix this by moving workaround processing to later in the driver probe sequence, to just before the processed list is used for the first time. Looking at the debugfs gt0/workarounds on ADL-P we notice 14011060649 should be present while we see, before: GT Workarounds 14011059788 14015795083 And with the patch: GT Workarounds 14011060649 14011059788 14015795083 Signed-off-by: Tvrtko Ursulin Cc: Lucas De Marchi Cc: Matt Roper Cc: stable@vger.kernel.org # v6.11+ Reviewed-by: Lucas De Marchi Link: https://patchwork.freedesktop.org/patch/msgid/20250227101304.46660-2-tvrtko.ursulin@igalia.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_gt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 5bd8dfdce300..90ff51ad76a6 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -362,9 +362,7 @@ int xe_gt_init_early(struct xe_gt *gt) if (err) return err; - xe_wa_process_gt(gt); xe_wa_process_oob(gt); - xe_tuning_process_gt(gt); xe_force_wake_init_gt(gt, gt_to_fw(gt)); spin_lock_init(>->global_invl_lock); @@ -451,6 +449,8 @@ static int all_fw_domain_init(struct xe_gt *gt) } xe_gt_mcr_set_implicit_defaults(gt); + xe_wa_process_gt(gt); + xe_tuning_process_gt(gt); xe_reg_sr_apply_mmio(>->reg_sr, gt); err = xe_gt_clock_init(gt); -- 2.51.0 From d9b5d83c5a4d720af6ddbefe2825c78f0325a3fd Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Thu, 27 Feb 2025 10:13:01 +0000 Subject: [PATCH 16/16] drm/xe/xelp: Move Wa_16011163337 from tunings to workarounds Workaround database specifies 16011163337 as a workaround so lets move it there. Signed-off-by: Tvrtko Ursulin Cc: Lucas De Marchi Cc: Matt Roper Cc: Gustavo Sousa Reviewed-by: Lucas De Marchi Link: https://patchwork.freedesktop.org/patch/msgid/20250227101304.46660-3-tvrtko.ursulin@igalia.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_tuning.c | 8 -------- drivers/gpu/drm/xe/xe_wa.c | 7 +++++++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c index d449de0fb6ec..3c78f3d71559 100644 --- a/drivers/gpu/drm/xe/xe_tuning.c +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -97,14 +97,6 @@ static const struct xe_rtp_entry_sr engine_tunings[] = { }; static const struct xe_rtp_entry_sr lrc_tunings[] = { - { XE_RTP_NAME("Tuning: ganged timer, also known as 16011163337"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), - /* read verification is ignored due to 1608008084. */ - XE_RTP_ACTIONS(FIELD_SET_NO_READ_MASK(FF_MODE2, - FF_MODE2_GS_TIMER_MASK, - FF_MODE2_GS_TIMER_224)) - }, - /* DG2 */ { XE_RTP_NAME("Tuning: L3 cache"), diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index b797c863bbe5..3e32fd4353a0 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -629,6 +629,13 @@ static const struct xe_rtp_entry_sr engine_was[] = { }; static const struct xe_rtp_entry_sr lrc_was[] = { + { XE_RTP_NAME("16011163337"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + /* read verification is ignored due to 1608008084. */ + XE_RTP_ACTIONS(FIELD_SET_NO_READ_MASK(FF_MODE2, + FF_MODE2_GS_TIMER_MASK, + FF_MODE2_GS_TIMER_224)) + }, { XE_RTP_NAME("1409342910, 14010698770, 14010443199, 1408979724, 1409178076, 1409207793, 1409217633, 1409252684, 1409347922, 1409142259"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN3, -- 2.51.0