} else {
                                atomic_inc(&pf_loading);
                                err = pci_enable_sriov(pdev, total_vfs);
-                               atomic_dec(&pf_loading);
                                if (err) {
                                        mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d).\n",
                                                 err);
+                                       atomic_dec(&pf_loading);
                                        err = 0;
                                } else {
                                        mlx4_warn(dev, "Running in master mode\n");
 
        priv->removed = 0;
 
+       if (mlx4_is_master(dev) && dev->num_vfs)
+               atomic_dec(&pf_loading);
+
        return 0;
 
 err_port:
        if (!mlx4_is_slave(dev))
                mlx4_free_ownership(dev);
 
+       if (mlx4_is_master(dev) && dev->num_vfs)
+               atomic_dec(&pf_loading);
+
        kfree(priv->dev.dev_vfs);
 
 err_free_dev:
        if (dev->flags & MLX4_FLAG_SRIOV) {
                mlx4_warn(dev, "Disabling SR-IOV\n");
                pci_disable_sriov(pdev);
+               dev->num_vfs = 0;
        }
 
        if (!mlx4_is_slave(dev))