drm/i915: Reduce duplicated forcewake logic

Introduce a structure to track the individual forcewake domains and use
that to eliminate duplicate logic.

v2: - Rebase on latest dinq (Mika)
    - for_each_fw_domain macro (Mika)
    - Handle reset atomically, keeping the timer running (Mika)
    - for_each_fw_domain parameter ordering (Chris)
    - defer timer on new register access (Mika)

v3: - Fix forcewake_reset/get race by waiting pending timers

v4: - cond_resched and verbose warning on timer deletion (Chris)
    - need to run pending timers manually on reset

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v1)
Signed-off-by: Mika Kuoppala <mika.kuoppala@intel.com>
Acked-by: Deepak S <deepak.s@linux.intel.com> (v2)
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index ac517eb07c00ca13bc678d87f6c3e869f9e5b630..4d156e65837f4ffe76c075b5c892d78335414609 100644 (file)
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1288,14 +1288,36 @@ static int ironlake_drpc_info(struct seq_file *m)
        return 0;
 }
 
-static int vlv_drpc_info(struct seq_file *m)
+static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
 {
+       struct drm_info_node *node = m->private;
+       struct drm_device *dev = node->minor->dev;
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct intel_uncore_forcewake_domain *fw_domain;
+       const char *domain_names[] = {
+               "render",
+               "blitter",
+               "media",
+       };
+       int i;
+
+       spin_lock_irq(&dev_priv->uncore.lock);
+       for_each_fw_domain(fw_domain, dev_priv, i) {
+               seq_printf(m, "%s.wake_count = %u\n",
+                          domain_names[i],
+                          fw_domain->wake_count);
+       }
+       spin_unlock_irq(&dev_priv->uncore.lock);
 
+       return 0;
+}
+
+static int vlv_drpc_info(struct seq_file *m)
+{
        struct drm_info_node *node = m->private;
        struct drm_device *dev = node->minor->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        u32 rpmodectl1, rcctl1, pw_status;
-       unsigned fw_rendercount = 0, fw_mediacount = 0;
 
        intel_runtime_pm_get(dev_priv);
 
@@ -1327,22 +1349,11 @@ static int vlv_drpc_info(struct seq_file *m)
        seq_printf(m, "Media RC6 residency since boot: %u\n",
                   I915_READ(VLV_GT_MEDIA_RC6));
 
-       spin_lock_irq(&dev_priv->uncore.lock);
-       fw_rendercount = dev_priv->uncore.fw_rendercount;
-       fw_mediacount = dev_priv->uncore.fw_mediacount;
-       spin_unlock_irq(&dev_priv->uncore.lock);
-
-       seq_printf(m, "Forcewake Render Count = %u\n", fw_rendercount);
-       seq_printf(m, "Forcewake Media Count = %u\n", fw_mediacount);
-
-
-       return 0;
+       return i915_gen6_forcewake_count_info(m, NULL);
 }
 
-
 static int gen6_drpc_info(struct seq_file *m)
 {
-
        struct drm_info_node *node = m->private;
        struct drm_device *dev = node->minor->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1356,7 +1367,7 @@ static int gen6_drpc_info(struct seq_file *m)
        intel_runtime_pm_get(dev_priv);
 
        spin_lock_irq(&dev_priv->uncore.lock);
-       forcewake_count = dev_priv->uncore.forcewake_count;
+       forcewake_count = dev_priv->uncore.fw_domain[FW_DOMAIN_ID_RENDER].wake_count;
        spin_unlock_irq(&dev_priv->uncore.lock);
 
        if (forcewake_count) {
@@ -1984,30 +1995,6 @@ static int i915_execlists(struct seq_file *m, void *data)
        return 0;
 }
 
-static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
-{
-       struct drm_info_node *node = m->private;
-       struct drm_device *dev = node->minor->dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0;
-
-       spin_lock_irq(&dev_priv->uncore.lock);
-       if (IS_VALLEYVIEW(dev)) {
-               fw_rendercount = dev_priv->uncore.fw_rendercount;
-               fw_mediacount = dev_priv->uncore.fw_mediacount;
-       } else
-               forcewake_count = dev_priv->uncore.forcewake_count;
-       spin_unlock_irq(&dev_priv->uncore.lock);
-
-       if (IS_VALLEYVIEW(dev)) {
-               seq_printf(m, "fw_rendercount = %u\n", fw_rendercount);
-               seq_printf(m, "fw_mediacount = %u\n", fw_mediacount);
-       } else
-               seq_printf(m, "forcewake count = %u\n", forcewake_count);
-
-       return 0;
-}
-
 static const char *swizzle_string(unsigned swizzle)
 {
        switch (swizzle) {

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3a32e0d81a9f25def5a9d56f0d07f1ab82a996b8..ac2082b022f3f5d09ff70a51dc35b82415abf19c 100644 (file)
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -596,20 +596,45 @@ struct intel_uncore_funcs {
                                uint64_t val, bool trace);
 };
 
+enum {
+       FW_DOMAIN_ID_RENDER = 0,
+       FW_DOMAIN_ID_BLITTER,
+       FW_DOMAIN_ID_MEDIA,
+
+       FW_DOMAIN_ID_COUNT
+};
+
 struct intel_uncore {
        spinlock_t lock; /** lock is also taken in irq contexts. */
 
        struct intel_uncore_funcs funcs;
 
        unsigned fifo_count;
-       unsigned forcewake_count;
-
-       unsigned fw_rendercount;
-       unsigned fw_mediacount;
-       unsigned fw_blittercount;
-
-       struct timer_list force_wake_timer;
-};
+       unsigned fw_domains;
+
+       struct intel_uncore_forcewake_domain {
+               struct drm_i915_private *i915;
+               int id;
+               unsigned wake_count;
+               struct timer_list timer;
+       } fw_domain[FW_DOMAIN_ID_COUNT];
+#define FORCEWAKE_RENDER       (1 << FW_DOMAIN_ID_RENDER)
+#define FORCEWAKE_BLITTER      (1 << FW_DOMAIN_ID_BLITTER)
+#define FORCEWAKE_MEDIA                (1 << FW_DOMAIN_ID_MEDIA)
+#define FORCEWAKE_ALL          (FORCEWAKE_RENDER | \
+                                FORCEWAKE_BLITTER | \
+                                FORCEWAKE_MEDIA)
+};
+
+/* Iterate over initialised fw domains */
+#define for_each_fw_domain_mask(domain__, mask__, dev_priv__, i__) \
+       for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \
+            (i__) < FW_DOMAIN_ID_COUNT; \
+            (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \
+               if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))
+
+#define for_each_fw_domain(domain__, dev_priv__, i__) \
+       for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__)
 
 #define DEV_INFO_FOR_EACH_FLAG(func, sep) \
        func(is_mobile) sep \
@@ -3167,8 +3192,10 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
  * must be set to prevent GT core from power down and stale values being
  * returned.
  */
-void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine);
-void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine);
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
+                           unsigned fw_domains);
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
+                           unsigned fw_domains);
 void assert_force_wake_inactive(struct drm_i915_private *dev_priv);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
@@ -3200,13 +3227,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
 int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val);
 
-#define FORCEWAKE_RENDER       (1 << 0)
-#define FORCEWAKE_MEDIA                (1 << 1)
-#define FORCEWAKE_BLITTER      (1 << 2)
-#define FORCEWAKE_ALL          (FORCEWAKE_RENDER | FORCEWAKE_MEDIA | \
-                                       FORCEWAKE_BLITTER)
-
-
 #define I915_READ8(reg)                dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)
 #define I915_WRITE8(reg, val)  dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)
 

diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 3d1ffacb7612d52fa2725ad8ae808ac87758f568..b35f3a9e6869a6cc50c172b96fe10a154117b67f 100644 (file)
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -67,7 +67,7 @@ static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
-                                                       int fw_engine)
+                                    int fw_engine)
 {
        if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
                            FORCEWAKE_ACK_TIMEOUT_MS))
@@ -93,7 +93,7 @@ static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
-                                                       int fw_engine)
+                                       int fw_engine)
 {
        u32 forcewake_ack;
 
@@ -129,7 +129,7 @@ static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 }
 
 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
-                                                       int fw_engine)
+                                    int fw_engine)
 {
        __raw_i915_write32(dev_priv, FORCEWAKE, 0);
        /* something from same cacheline, but !FORCEWAKE */
@@ -138,7 +138,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 }
 
 static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
-                                                       int fw_engine)
+                                       int fw_engine)
 {
        __raw_i915_write32(dev_priv, FORCEWAKE_MT,
                           _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
@@ -187,7 +187,7 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 }
 
 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
-                                               int fw_engine)
+                                int fw_engine)
 {
        /* Check for Render Engine */
        if (FORCEWAKE_RENDER & fw_engine) {
@@ -227,9 +227,8 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 }
 
 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
-                                       int fw_engine)
+                                int fw_engine)
 {
-
        /* Check for Render Engine */
        if (FORCEWAKE_RENDER & fw_engine)
                __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
@@ -247,37 +246,6 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
                gen6_gt_check_fifodbg(dev_priv);
 }
 
-static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
-{
-       if (fw_engine & FORCEWAKE_RENDER &&
-           dev_priv->uncore.fw_rendercount++ != 0)
-               fw_engine &= ~FORCEWAKE_RENDER;
-       if (fw_engine & FORCEWAKE_MEDIA &&
-           dev_priv->uncore.fw_mediacount++ != 0)
-               fw_engine &= ~FORCEWAKE_MEDIA;
-
-       if (fw_engine)
-               dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
-}
-
-static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
-{
-       if (fw_engine & FORCEWAKE_RENDER) {
-               WARN_ON(!dev_priv->uncore.fw_rendercount);
-               if (--dev_priv->uncore.fw_rendercount != 0)
-                       fw_engine &= ~FORCEWAKE_RENDER;
-       }
-
-       if (fw_engine & FORCEWAKE_MEDIA) {
-               WARN_ON(!dev_priv->uncore.fw_mediacount);
-               if (--dev_priv->uncore.fw_mediacount != 0)
-                       fw_engine &= ~FORCEWAKE_MEDIA;
-       }
-
-       if (fw_engine)
-               dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
-}
-
 static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 {
        __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
@@ -367,80 +335,72 @@ __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
                                _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 }
 
-static void
-gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
+static void gen6_force_wake_timer(unsigned long arg)
 {
-       if (FORCEWAKE_RENDER & fw_engine) {
-               if (dev_priv->uncore.fw_rendercount++ == 0)
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                       FORCEWAKE_RENDER);
-       }
+       struct intel_uncore_forcewake_domain *domain = (void *)arg;
+       unsigned long irqflags;
 
-       if (FORCEWAKE_MEDIA & fw_engine) {
-               if (dev_priv->uncore.fw_mediacount++ == 0)
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                       FORCEWAKE_MEDIA);
-       }
+       assert_device_not_suspended(domain->i915);
 
-       if (FORCEWAKE_BLITTER & fw_engine) {
-               if (dev_priv->uncore.fw_blittercount++ == 0)
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                       FORCEWAKE_BLITTER);
-       }
+       spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
+       if (WARN_ON(domain->wake_count == 0))
+               domain->wake_count++;
+
+       if (--domain->wake_count == 0)
+               domain->i915->uncore.funcs.force_wake_put(domain->i915,
+                                                         1 << domain->id);
+
+       spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
 }
 
-static void
-gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
+void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 {
-       if (FORCEWAKE_RENDER & fw_engine) {
-               WARN_ON(dev_priv->uncore.fw_rendercount == 0);
-               if (--dev_priv->uncore.fw_rendercount == 0)
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                       FORCEWAKE_RENDER);
-       }
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       unsigned long irqflags, fw = 0;
+       struct intel_uncore_forcewake_domain *domain;
+       int id, active_domains, retry_count = 100;
 
-       if (FORCEWAKE_MEDIA & fw_engine) {
-               WARN_ON(dev_priv->uncore.fw_mediacount == 0);
-               if (--dev_priv->uncore.fw_mediacount == 0)
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                       FORCEWAKE_MEDIA);
-       }
+       /* Hold uncore.lock across reset to prevent any register access
+        * with forcewake not set correctly. Wait until all pending
+        * timers are run before holding.
+        */
+       while (1) {
+               active_domains = 0;
 
-       if (FORCEWAKE_BLITTER & fw_engine) {
-               WARN_ON(dev_priv->uncore.fw_blittercount == 0);
-               if (--dev_priv->uncore.fw_blittercount == 0)
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                       FORCEWAKE_BLITTER);
-       }
-}
+               for_each_fw_domain(domain, dev_priv, id) {
+                       if (del_timer_sync(&domain->timer) == 0)
+                               continue;
 
-static void gen6_force_wake_timer(unsigned long arg)
-{
-       struct drm_i915_private *dev_priv = (void *)arg;
-       unsigned long irqflags;
+                       gen6_force_wake_timer((unsigned long)domain);
+               }
 
-       assert_device_not_suspended(dev_priv);
+               spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-       spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-       WARN_ON(!dev_priv->uncore.forcewake_count);
+               for_each_fw_domain(domain, dev_priv, id) {
+                       if (timer_pending(&domain->timer))
+                               active_domains |= (1 << id);
+               }
 
-       if (--dev_priv->uncore.forcewake_count == 0)
-               dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
-       spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
-}
+               if (active_domains == 0)
+                       break;
 
-void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
-{
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       unsigned long irqflags;
+               if (--retry_count == 0) {
+                       DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
+                       break;
+               }
 
-       if (del_timer_sync(&dev_priv->uncore.force_wake_timer))
-               gen6_force_wake_timer((unsigned long)dev_priv);
+               spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+               cond_resched();
+       }
 
-       /* Hold uncore.lock across reset to prevent any register access
-        * with forcewake not set correctly
-        */
-       spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+       WARN_ON(active_domains);
+
+       for_each_fw_domain(domain, dev_priv, id)
+               if (domain->wake_count)
+                       fw |= 1 << id;
+
+       if (fw)
+               dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 
        if (IS_VALLEYVIEW(dev))
                vlv_force_wake_reset(dev_priv);
@@ -454,28 +414,6 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
                __gen9_gt_force_wake_mt_reset(dev_priv);
 
        if (restore) { /* If reset with a user forcewake, try to restore */
-               unsigned fw = 0;
-
-               if (IS_VALLEYVIEW(dev)) {
-                       if (dev_priv->uncore.fw_rendercount)
-                               fw |= FORCEWAKE_RENDER;
-
-                       if (dev_priv->uncore.fw_mediacount)
-                               fw |= FORCEWAKE_MEDIA;
-               } else if (IS_GEN9(dev)) {
-                       if (dev_priv->uncore.fw_rendercount)
-                               fw |= FORCEWAKE_RENDER;
-
-                       if (dev_priv->uncore.fw_mediacount)
-                               fw |= FORCEWAKE_MEDIA;
-
-                       if (dev_priv->uncore.fw_blittercount)
-                               fw |= FORCEWAKE_BLITTER;
-               } else {
-                       if (dev_priv->uncore.forcewake_count)
-                               fw = FORCEWAKE_ALL;
-               }
-
                if (fw)
                        dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 
@@ -485,6 +423,9 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
                                GT_FIFO_FREE_ENTRIES_MASK;
        }
 
+       if (!restore)
+               assert_force_wake_inactive(dev_priv);
+
        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
@@ -533,53 +474,59 @@ void intel_uncore_sanitize(struct drm_device *dev)
  * be called at the beginning of the sequence followed by a call to
  * gen6_gt_force_wake_put() at the end of the sequence.
  */
-void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
+void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
+                           unsigned fw_domains)
 {
        unsigned long irqflags;
+       struct intel_uncore_forcewake_domain *domain;
+       int id;
 
        if (!dev_priv->uncore.funcs.force_wake_get)
                return;
 
        WARN_ON(dev_priv->pm.suspended);
 
+       fw_domains &= dev_priv->uncore.fw_domains;
+
        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-       if (IS_GEN9(dev_priv->dev)) {
-               gen9_force_wake_get(dev_priv, fw_engine);
-       } else if (IS_VALLEYVIEW(dev_priv->dev)) {
-               vlv_force_wake_get(dev_priv, fw_engine);
-       } else {
-               if (dev_priv->uncore.forcewake_count++ == 0)
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                             FORCEWAKE_ALL);
+       for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
+               if (domain->wake_count++)
+                       fw_domains &= ~(1 << id);
        }
 
+       if (fw_domains)
+               dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+
        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
 /*
  * see gen6_gt_force_wake_get()
  */
-void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
+void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
+                           unsigned fw_domains)
 {
        unsigned long irqflags;
+       struct intel_uncore_forcewake_domain *domain;
+       int id;
 
        if (!dev_priv->uncore.funcs.force_wake_put)
                return;
 
+       fw_domains &= dev_priv->uncore.fw_domains;
+
        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-       if (IS_GEN9(dev_priv->dev)) {
-               gen9_force_wake_put(dev_priv, fw_engine);
-       } else if (IS_VALLEYVIEW(dev_priv->dev)) {
-               vlv_force_wake_put(dev_priv, fw_engine);
-       } else {
-               WARN_ON(!dev_priv->uncore.forcewake_count);
-               if (--dev_priv->uncore.forcewake_count == 0) {
-                       dev_priv->uncore.forcewake_count++;
-                       mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
-                                        jiffies + 1);
-               }
+       for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
+               if (WARN_ON(domain->wake_count == 0))
+                       continue;
+
+               if (--domain->wake_count)
+                       continue;
+
+               domain->wake_count++;
+               mod_timer_pinned(&domain->timer, jiffies + 1);
        }
 
        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
@@ -587,10 +534,14 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 
 void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
 {
+       struct intel_uncore_forcewake_domain *domain;
+       int i;
+
        if (!dev_priv->uncore.funcs.force_wake_get)
                return;
 
-       WARN_ON(dev_priv->uncore.forcewake_count > 0);
+       for_each_fw_domain(domain, dev_priv, i)
+               WARN_ON(domain->wake_count);
 }
 
 /* We give fast paths for the really cool registers */
@@ -753,19 +704,37 @@ __gen2_read(64)
        trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
        return val
 
+static inline void __force_wake_get(struct drm_i915_private *dev_priv,
+                                   unsigned fw_domains)
+{
+       struct intel_uncore_forcewake_domain *domain;
+       int i;
+
+       if (WARN_ON(!fw_domains))
+               return;
+
+       /* Ideally GCC would be constant-fold and eliminate this loop */
+       for_each_fw_domain_mask(domain, fw_domains, dev_priv, i) {
+               if (domain->wake_count) {
+                       fw_domains &= ~(1 << i);
+                       continue;
+               }
+
+               domain->wake_count++;
+               mod_timer_pinned(&domain->timer, jiffies + 1);
+       }
+
+       if (fw_domains)
+               dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+}
+
 #define __gen6_read(x) \
 static u##x \
 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
        GEN6_READ_HEADER(x); \
        hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
-       if (dev_priv->uncore.forcewake_count == 0 && \
-           NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
-               dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                     FORCEWAKE_ALL); \
-               dev_priv->uncore.forcewake_count++; \
-               mod_timer_pinned(&dev_priv->uncore.force_wake_timer, \
-                                jiffies + 1); \
-       } \
+       if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \
+               __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
        val = __raw_i915_read##x(dev_priv, reg); \
        hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
        GEN6_READ_FOOTER; \
@@ -774,45 +743,27 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 #define __vlv_read(x) \
 static u##x \
 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
-       unsigned fwengine = 0; \
        GEN6_READ_HEADER(x); \
-       if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
-               if (dev_priv->uncore.fw_rendercount == 0) \
-                       fwengine = FORCEWAKE_RENDER; \
-       } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
-               if (dev_priv->uncore.fw_mediacount == 0) \
-                       fwengine = FORCEWAKE_MEDIA; \
-       }  \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
+       if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \
+               __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+       else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \
+               __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
        val = __raw_i915_read##x(dev_priv, reg); \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
        GEN6_READ_FOOTER; \
 }
 
 #define __chv_read(x) \
 static u##x \
 chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
-       unsigned fwengine = 0; \
        GEN6_READ_HEADER(x); \
-       if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
-               if (dev_priv->uncore.fw_rendercount == 0) \
-                       fwengine = FORCEWAKE_RENDER; \
-       } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
-               if (dev_priv->uncore.fw_mediacount == 0) \
-                       fwengine = FORCEWAKE_MEDIA; \
-       } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
-               if (dev_priv->uncore.fw_rendercount == 0) \
-                       fwengine |= FORCEWAKE_RENDER; \
-               if (dev_priv->uncore.fw_mediacount == 0) \
-                       fwengine |= FORCEWAKE_MEDIA; \
-       } \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
+       if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
+               __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+       else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
+               __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
+       else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
+               __force_wake_get(dev_priv, \
+                                FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
        val = __raw_i915_read##x(dev_priv, reg); \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
        GEN6_READ_FOOTER; \
 }
 
@@ -822,32 +773,21 @@ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 #define __gen9_read(x) \
 static u##x \
 gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
+       unsigned fw_engine; \
        GEN6_READ_HEADER(x); \
-       if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
-               val = __raw_i915_read##x(dev_priv, reg); \
-       } else { \
-               unsigned fwengine = 0; \
-               if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine = FORCEWAKE_RENDER; \
-               } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine = FORCEWAKE_MEDIA; \
-               } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine |= FORCEWAKE_RENDER; \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine |= FORCEWAKE_MEDIA; \
-               } else { \
-                       if (dev_priv->uncore.fw_blittercount == 0) \
-                               fwengine = FORCEWAKE_BLITTER; \
-               } \
-               if (fwengine) \
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
-               val = __raw_i915_read##x(dev_priv, reg); \
-               if (fwengine) \
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
-       } \
+       if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)))   \
+               fw_engine = 0; \
+       else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg))       \
+               fw_engine = FORCEWAKE_RENDER; \
+       else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+               fw_engine = FORCEWAKE_MEDIA; \
+       else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+               fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+       else \
+               fw_engine = FORCEWAKE_BLITTER; \
+       if (fw_engine) \
+               __force_wake_get(dev_priv, fw_engine); \
+       val = __raw_i915_read##x(dev_priv, reg); \
        GEN6_READ_FOOTER; \
 }
 
@@ -981,17 +921,9 @@ static void \
 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
        GEN6_WRITE_HEADER; \
        hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
-       if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
-               if (dev_priv->uncore.forcewake_count == 0) \
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                             FORCEWAKE_ALL); \
-               __raw_i915_write##x(dev_priv, reg, val); \
-               if (dev_priv->uncore.forcewake_count == 0) \
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                                             FORCEWAKE_ALL); \
-       } else { \
-               __raw_i915_write##x(dev_priv, reg, val); \
-       } \
+       if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \
+               __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+       __raw_i915_write##x(dev_priv, reg, val); \
        hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
        hsw_unclaimed_reg_detect(dev_priv); \
        GEN6_WRITE_FOOTER; \
@@ -1000,28 +932,17 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
 #define __chv_write(x) \
 static void \
 chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
-       unsigned fwengine = 0; \
        bool shadowed = is_gen8_shadowed(dev_priv, reg); \
        GEN6_WRITE_HEADER; \
        if (!shadowed) { \
-               if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine = FORCEWAKE_RENDER; \
-               } else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine = FORCEWAKE_MEDIA; \
-               } else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine |= FORCEWAKE_RENDER; \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine |= FORCEWAKE_MEDIA; \
-               } \
+               if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
+                       __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
+               else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
+                       __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
+               else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
+                       __force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
        } \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
        __raw_i915_write##x(dev_priv, reg, val); \
-       if (fwengine) \
-               dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
        GEN6_WRITE_FOOTER; \
 }
 
@@ -1052,35 +973,22 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 static void \
 gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \
                bool trace) { \
+       unsigned fw_engine; \
        GEN6_WRITE_HEADER; \
-       if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
-                       is_gen9_shadowed(dev_priv, reg)) { \
-               __raw_i915_write##x(dev_priv, reg, val); \
-       } else { \
-               unsigned fwengine = 0; \
-               if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine = FORCEWAKE_RENDER; \
-               } else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine = FORCEWAKE_MEDIA; \
-               } else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \
-                       if (dev_priv->uncore.fw_rendercount == 0) \
-                               fwengine |= FORCEWAKE_RENDER; \
-                       if (dev_priv->uncore.fw_mediacount == 0) \
-                               fwengine |= FORCEWAKE_MEDIA; \
-               } else { \
-                       if (dev_priv->uncore.fw_blittercount == 0) \
-                               fwengine = FORCEWAKE_BLITTER; \
-               } \
-               if (fwengine) \
-                       dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                       fwengine); \
-               __raw_i915_write##x(dev_priv, reg, val); \
-               if (fwengine) \
-                       dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                       fwengine); \
-       } \
+       if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
+           is_gen9_shadowed(dev_priv, reg)) \
+               fw_engine = 0; \
+       else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
+               fw_engine = FORCEWAKE_RENDER; \
+       else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
+               fw_engine = FORCEWAKE_MEDIA; \
+       else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
+               fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+       else \
+               fw_engine = FORCEWAKE_BLITTER; \
+       if (fw_engine) \
+               __force_wake_get(dev_priv, fw_engine); \
+       __raw_i915_write##x(dev_priv, reg, val); \
        GEN6_WRITE_FOOTER; \
 }
 
@@ -1132,21 +1040,24 @@ do { \
 void intel_uncore_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
-
-       setup_timer(&dev_priv->uncore.force_wake_timer,
-                   gen6_force_wake_timer, (unsigned long)dev_priv);
+       struct intel_uncore_forcewake_domain *domain;
+       int i;
 
        __intel_uncore_early_sanitize(dev, false);
 
        if (IS_GEN9(dev)) {
                dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get;
                dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put;
+               dev_priv->uncore.fw_domains = FORCEWAKE_RENDER |
+                       FORCEWAKE_BLITTER | FORCEWAKE_MEDIA;
        } else if (IS_VALLEYVIEW(dev)) {
                dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
                dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
+               dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | FORCEWAKE_MEDIA;
        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
                dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
+               dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
        } else if (IS_IVYBRIDGE(dev)) {
                u32 ecobus;
 
@@ -1178,11 +1089,21 @@ void intel_uncore_init(struct drm_device *dev)
                        dev_priv->uncore.funcs.force_wake_put =
                                __gen6_gt_force_wake_put;
                }
+               dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
        } else if (IS_GEN6(dev)) {
                dev_priv->uncore.funcs.force_wake_get =
                        __gen6_gt_force_wake_get;
                dev_priv->uncore.funcs.force_wake_put =
                        __gen6_gt_force_wake_put;
+               dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
+       }
+
+       for_each_fw_domain(domain, dev_priv, i) {
+               domain->i915 = dev_priv;
+               domain->id = i;
+
+               setup_timer(&domain->timer, gen6_force_wake_timer,
+                           (unsigned long)domain);
        }
 
        switch (INTEL_INFO(dev)->gen) {