if (ret)
                goto error_unregister_notifier;
 
-       if (musb->port_mode == MUSB_PORT_MODE_HOST) {
-               ret = phy_power_on(glue->phy);
-               if (ret)
-                       goto error_phy_exit;
-               set_bit(SUNXI_MUSB_FL_PHY_ON, &glue->flags);
-               /* Stop musb work from turning vbus off again */
-               set_bit(SUNXI_MUSB_FL_VBUS_ON, &glue->flags);
-       }
-
        musb->isr = sunxi_musb_interrupt;
 
        /* Stop the musb-core from doing runtime pm (not supported on sunxi) */
 
        return 0;
 
-error_phy_exit:
-       phy_exit(glue->phy);
 error_unregister_notifier:
        if (musb->port_mode == MUSB_PORT_MODE_DUAL_ROLE)
                extcon_unregister_notifier(glue->extcon, EXTCON_USB_HOST,
        return 0;
 }
 
+static int sunxi_set_mode(struct musb *musb, u8 mode)
+{
+       struct sunxi_glue *glue = dev_get_drvdata(musb->controller->parent);
+       int ret;
+
+       if (mode == MUSB_HOST) {
+               ret = phy_power_on(glue->phy);
+               if (ret)
+                       return ret;
+
+               set_bit(SUNXI_MUSB_FL_PHY_ON, &glue->flags);
+               /* Stop musb work from turning vbus off again */
+               set_bit(SUNXI_MUSB_FL_VBUS_ON, &glue->flags);
+       }
+
+       return 0;
+}
+
 static void sunxi_musb_enable(struct musb *musb)
 {
        struct sunxi_glue *glue = dev_get_drvdata(musb->controller->parent);
        .exit           = sunxi_musb_exit,
        .enable         = sunxi_musb_enable,
        .disable        = sunxi_musb_disable,
+       .set_mode       = sunxi_set_mode,
        .fifo_offset    = sunxi_musb_fifo_offset,
        .ep_offset      = sunxi_musb_ep_offset,
        .busctl_offset  = sunxi_musb_busctl_offset,