return 0;
        else if (ret)
                return ret;
+       pfdevfreq->opp_of_table_added = true;
 
        spin_lock_init(&pfdevfreq->lock);
 
        cur_freq = clk_get_rate(pfdev->clock);
 
        opp = devfreq_recommended_opp(dev, &cur_freq, 0);
-       if (IS_ERR(opp))
-               return PTR_ERR(opp);
+       if (IS_ERR(opp)) {
+               ret = PTR_ERR(opp);
+               goto err_fini;
+       }
 
        panfrost_devfreq_profile.initial_freq = cur_freq;
        dev_pm_opp_put(opp);
                                          DEVFREQ_GOV_SIMPLE_ONDEMAND, NULL);
        if (IS_ERR(devfreq)) {
                DRM_DEV_ERROR(dev, "Couldn't initialize GPU devfreq\n");
-               dev_pm_opp_of_remove_table(dev);
-               return PTR_ERR(devfreq);
+               ret = PTR_ERR(devfreq);
+               goto err_fini;
        }
        pfdevfreq->devfreq = devfreq;
 
                pfdevfreq->cooling = cooling;
 
        return 0;
+
+err_fini:
+       panfrost_devfreq_fini(pfdev);
+       return ret;
 }
 
 void panfrost_devfreq_fini(struct panfrost_device *pfdev)
 {
        struct panfrost_devfreq *pfdevfreq = &pfdev->pfdevfreq;
 
-       if (pfdevfreq->cooling)
+       if (pfdevfreq->cooling) {
                devfreq_cooling_unregister(pfdevfreq->cooling);
-       dev_pm_opp_of_remove_table(&pfdev->pdev->dev);
+               pfdevfreq->cooling = NULL;
+       }
+
+       if (pfdevfreq->opp_of_table_added) {
+               dev_pm_opp_of_remove_table(&pfdev->pdev->dev);
+               pfdevfreq->opp_of_table_added = false;
+       }
 }
 
 void panfrost_devfreq_resume(struct panfrost_device *pfdev)