#include "i915_trace.h"
 #include "i915_vgpu.h"
 #include "intel_csr.h"
+#include "intel_memory_region.h"
 #include "intel_pm.h"
 
 static struct drm_driver driver;
        if (ret)
                goto err_ggtt;
 
+       ret = intel_memory_regions_hw_probe(dev_priv);
+       if (ret)
+               goto err_ggtt;
+
        intel_gt_init_hw_early(dev_priv);
 
        ret = i915_ggtt_enable_hw(dev_priv);
        if (ret) {
                DRM_ERROR("failed to enable GGTT\n");
-               goto err_ggtt;
+               goto err_mem_regions;
        }
 
        pci_set_master(pdev);
                if (ret) {
                        DRM_ERROR("failed to set DMA mask\n");
 
-                       goto err_ggtt;
+                       goto err_mem_regions;
                }
        }
 
                if (ret) {
                        DRM_ERROR("failed to set DMA mask\n");
 
-                       goto err_ggtt;
+                       goto err_mem_regions;
                }
        }
 
        if (pdev->msi_enabled)
                pci_disable_msi(pdev);
        pm_qos_remove_request(&dev_priv->pm_qos);
+err_mem_regions:
+       intel_memory_regions_driver_release(dev_priv);
 err_ggtt:
        i915_ggtt_driver_release(dev_priv);
 err_perf:
 
 out_cleanup_hw:
        i915_driver_hw_remove(dev_priv);
+       intel_memory_regions_driver_release(dev_priv);
        i915_ggtt_driver_release(dev_priv);
 out_cleanup_mmio:
        i915_driver_mmio_release(dev_priv);
 
        i915_gem_driver_release(dev_priv);
 
+       intel_memory_regions_driver_release(dev_priv);
        i915_ggtt_driver_release(dev_priv);
 
        i915_driver_mmio_release(dev_priv);
 
                                         unsigned int flags);
 int i915_gem_evict_vm(struct i915_address_space *vm);
 
-void i915_gem_cleanup_memory_regions(struct drm_i915_private *i915);
-int i915_gem_init_memory_regions(struct drm_i915_private *i915);
-
 /* i915_gem_internal.c */
 struct drm_i915_gem_object *
 i915_gem_object_create_internal(struct drm_i915_private *dev_priv,
 
        return 0;
 }
 
-void i915_gem_cleanup_memory_regions(struct drm_i915_private *i915)
-{
-       int i;
-
-       for (i = 0; i < INTEL_REGION_UNKNOWN; i++) {
-               struct intel_memory_region *region = i915->mm.regions[i];
-
-               if (region)
-                       intel_memory_region_put(region);
-       }
-}
-
-int i915_gem_init_memory_regions(struct drm_i915_private *i915)
-{
-       int err, i;
-
-       for (i = 0; i < INTEL_REGION_UNKNOWN; i++) {
-               struct intel_memory_region *mem = ERR_PTR(-ENODEV);
-               u32 type;
-
-               if (!HAS_REGION(i915, BIT(i)))
-                       continue;
-
-               type = MEMORY_TYPE_FROM_REGION(intel_region_map[i]);
-               switch (type) {
-               case INTEL_MEMORY_SYSTEM:
-                       mem = i915_gem_shmem_setup(i915);
-                       break;
-               case INTEL_MEMORY_STOLEN:
-                       mem = i915_gem_stolen_setup(i915);
-                       break;
-               }
-
-               if (IS_ERR(mem)) {
-                       err = PTR_ERR(mem);
-                       DRM_ERROR("Failed to setup region(%d) type=%d\n", err, type);
-                       goto out_cleanup;
-               }
-
-               mem->id = intel_region_map[i];
-               mem->type = type;
-               mem->instance = MEMORY_INSTANCE_FROM_REGION(intel_region_map[i]);
-
-               i915->mm.regions[i] = mem;
-       }
-
-       return 0;
-
-out_cleanup:
-       i915_gem_cleanup_memory_regions(i915);
-       return err;
-}
-
 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
 {
        struct i915_vma *vma, *vn;
 {
        struct pagevec *pvec;
 
-       i915_gem_cleanup_memory_regions(i915);
-
        fini_aliasing_ppgtt(&i915->ggtt);
 
        ggtt_cleanup_hw(&i915->ggtt);
        if (ret)
                return ret;
 
-       ret = i915_gem_init_memory_regions(dev_priv);
-       if (ret)
-               goto out_gtt_cleanup;
-
        return 0;
-
-out_gtt_cleanup:
-       dev_priv->ggtt.vm.cleanup(&dev_priv->ggtt.vm);
-       return ret;
 }
 
 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
 
        kref_put(&mem->kref, __intel_memory_region_destroy);
 }
 
+/* Global memory region registration -- only slight layer inversions! */
+
+int intel_memory_regions_hw_probe(struct drm_i915_private *i915)
+{
+       int err, i;
+
+       for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
+               struct intel_memory_region *mem = ERR_PTR(-ENODEV);
+               u32 type;
+
+               if (!HAS_REGION(i915, BIT(i)))
+                       continue;
+
+               type = MEMORY_TYPE_FROM_REGION(intel_region_map[i]);
+               switch (type) {
+               case INTEL_MEMORY_SYSTEM:
+                       mem = i915_gem_shmem_setup(i915);
+                       break;
+               case INTEL_MEMORY_STOLEN:
+                       mem = i915_gem_stolen_setup(i915);
+                       break;
+               }
+
+               if (IS_ERR(mem)) {
+                       err = PTR_ERR(mem);
+                       DRM_ERROR("Failed to setup region(%d) type=%d\n", err, type);
+                       goto out_cleanup;
+               }
+
+               mem->id = intel_region_map[i];
+               mem->type = type;
+               mem->instance = MEMORY_INSTANCE_FROM_REGION(intel_region_map[i]);
+
+               i915->mm.regions[i] = mem;
+       }
+
+       return 0;
+
+out_cleanup:
+       intel_memory_regions_driver_release(i915);
+       return err;
+}
+
+void intel_memory_regions_driver_release(struct drm_i915_private *i915)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
+               struct intel_memory_region *region =
+                       fetch_and_zero(&i915->mm.regions[i]);
+
+               if (region)
+                       intel_memory_region_put(region);
+       }
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/intel_memory_region.c"
 #include "selftests/mock_region.c"
 
 intel_memory_region_get(struct intel_memory_region *mem);
 void intel_memory_region_put(struct intel_memory_region *mem);
 
+int intel_memory_regions_hw_probe(struct drm_i915_private *i915);
+void intel_memory_regions_driver_release(struct drm_i915_private *i915);
+
 #endif
 
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_requests.h"
 #include "gt/mock_engine.h"
+#include "intel_memory_region.h"
 
 #include "mock_request.h"
 #include "mock_gem_device.h"
        mock_fini_ggtt(&i915->ggtt);
        destroy_workqueue(i915->wq);
 
-       i915_gem_cleanup_memory_regions(i915);
+       intel_memory_regions_driver_release(i915);
 
        drm_mode_config_cleanup(&i915->drm);
 
                I915_GTT_PAGE_SIZE_2M;
 
        mkwrite_device_info(i915)->memory_regions = REGION_SMEM;
+       intel_memory_regions_hw_probe(i915);
 
        mock_uncore_init(&i915->uncore, i915);
 
 
        intel_engines_driver_register(i915);
 
-       err = i915_gem_init_memory_regions(i915);
-       if (err)
-               goto err_context;
-
        return i915;
 
 err_context:
        intel_timelines_fini(i915);
        destroy_workqueue(i915->wq);
 err_drv:
+       intel_memory_regions_driver_release(i915);
        drm_mode_config_cleanup(&i915->drm);
        drm_dev_fini(&i915->drm);
 put_device: