drm/nouveau/kms/nv04-nv40: prevent undisplayable framebuffers from creation

Pre-nv50 YUV overlays have stringent requirements for working with the
internal machinery. Instead of rejecting these at update_plane time, we
should instead prevent the framebuffers from being created in the first
place.

Signed-off-by: Ilia Mirkin <imirkin@alum.mit.edu>
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index d666400479137076353d27e093e4106a75cc7fc3..2e7785f49e6d54c1c0941c69de2ff0a869cfd3f0 100644 (file)
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -231,9 +231,30 @@ nouveau_framebuffer_new(struct drm_device *dev,
                        struct nouveau_bo *nvbo,
                        struct nouveau_framebuffer **pfb)
 {
+       struct nouveau_drm *drm = nouveau_drm(dev);
        struct nouveau_framebuffer *fb;
        int ret;
 
+        /* YUV overlays have special requirements pre-NV50 */
+       if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA &&
+
+           (mode_cmd->pixel_format == DRM_FORMAT_YUYV ||
+            mode_cmd->pixel_format == DRM_FORMAT_UYVY ||
+            mode_cmd->pixel_format == DRM_FORMAT_NV12 ||
+            mode_cmd->pixel_format == DRM_FORMAT_NV21) &&
+           (mode_cmd->pitches[0] & 0x3f || /* align 64 */
+            mode_cmd->pitches[0] >= 0x10000 || /* at most 64k pitch */
+            (mode_cmd->pitches[1] && /* pitches for planes must match */
+             mode_cmd->pitches[0] != mode_cmd->pitches[1]))) {
+               struct drm_format_name_buf format_name;
+               DRM_DEBUG_KMS("Unsuitable framebuffer: format: %s; pitches: 0x%x\n 0x%x\n",
+                             drm_get_format_name(mode_cmd->pixel_format,
+                                                 &format_name),
+                             mode_cmd->pitches[0],
+                             mode_cmd->pitches[1]);
+               return -EINVAL;
+       }
+
        if (!(fb = *pfb = kzalloc(sizeof(*fb), GFP_KERNEL)))
                return -ENOMEM;