container_of(cb, struct vc4_async_flip_state, cb);
        struct drm_crtc *crtc = flip_state->crtc;
        struct drm_device *dev = crtc->dev;
-       struct vc4_dev *vc4 = to_vc4_dev(dev);
        struct drm_plane *plane = crtc->primary;
 
        vc4_plane_async_set_fb(plane, flip_state->fb);
        }
 
        kfree(flip_state);
-
-       up(&vc4->async_modeset);
 }
 
 /* Implements async (non-vblank-synced) page flips.
                               uint32_t flags)
 {
        struct drm_device *dev = crtc->dev;
-       struct vc4_dev *vc4 = to_vc4_dev(dev);
        struct drm_plane *plane = crtc->primary;
        int ret = 0;
        struct vc4_async_flip_state *flip_state;
        flip_state->crtc = crtc;
        flip_state->event = event;
 
-       /* Make sure all other async modesetes have landed. */
-       ret = down_interruptible(&vc4->async_modeset);
-       if (ret) {
-               drm_framebuffer_put(fb);
-               vc4_bo_dec_usecnt(bo);
-               kfree(flip_state);
-               return ret;
-       }
-
        /* Save the current FB before it's replaced by the new one in
         * drm_atomic_set_fb_for_plane(). We'll need the old FB in
         * vc4_async_page_flip_complete() to decrement the BO usecnt and keep
 
                clk_set_min_rate(hvs->core_clk, 0);
 
        drm_atomic_state_put(state);
-
-       up(&vc4->async_modeset);
 }
 
 static void commit_work(struct work_struct *work)
                             struct drm_atomic_state *state,
                             bool nonblock)
 {
-       struct vc4_dev *vc4 = to_vc4_dev(dev);
        int ret;
 
        if (state->async_update) {
-               ret = down_interruptible(&vc4->async_modeset);
-               if (ret)
-                       return ret;
-
                ret = drm_atomic_helper_prepare_planes(dev, state);
-               if (ret) {
-                       up(&vc4->async_modeset);
+               if (ret)
                        return ret;
-               }
 
                drm_atomic_helper_async_commit(dev, state);
 
                drm_atomic_helper_cleanup_planes(dev, state);
 
-               up(&vc4->async_modeset);
-
                return 0;
        }
 
 
        INIT_WORK(&state->commit_work, commit_work);
 
-       ret = down_interruptible(&vc4->async_modeset);
-       if (ret)
-               return ret;
-
        ret = drm_atomic_helper_prepare_planes(dev, state);
-       if (ret) {
-               up(&vc4->async_modeset);
+       if (ret)
                return ret;
-       }
 
        if (!nonblock) {
                ret = drm_atomic_helper_wait_for_fences(dev, state, true);
                if (ret) {
                        drm_atomic_helper_cleanup_planes(dev, state);
-                       up(&vc4->async_modeset);
                        return ret;
                }
        }
                vc4->load_tracker_enabled = true;
        }
 
-       sema_init(&vc4->async_modeset, 1);
-
        /* Set support for vblank irq fast disable, before drm_vblank_init() */
        dev->vblank_disable_immediate = true;