switch (pcm512x->fmt & SND_SOC_DAIFMT_MASTER_MASK) {
        case SND_SOC_DAIFMT_CBS_CFS:
-               ret = regmap_update_bits(pcm512x->regmap,
-                                        PCM512x_BCLK_LRCLK_CFG,
-                                        PCM512x_BCKP
-                                        | PCM512x_BCKO | PCM512x_LRKO,
-                                        0);
-               if (ret != 0) {
-                       dev_err(component->dev,
-                               "Failed to enable slave mode: %d\n", ret);
-                       return ret;
-               }
+               clock_output = 0;
+               master_mode = 0;
 
                ret = regmap_update_bits(pcm512x->regmap, PCM512x_ERROR_DETECT,
                                         PCM512x_DCAS, 0);
                                ret);
                        return ret;
                }
-               return 0;
+               goto skip_pll;
        case SND_SOC_DAIFMT_CBM_CFM:
                clock_output = PCM512x_BCKO | PCM512x_LRKO;
                master_mode = PCM512x_RLRK | PCM512x_RBCK;
                        dev_err(component->dev, "Failed to enable pll: %d\n", ret);
                        return ret;
                }
-       }
-
-       ret = regmap_update_bits(pcm512x->regmap, PCM512x_BCLK_LRCLK_CFG,
-                                PCM512x_BCKP | PCM512x_BCKO | PCM512x_LRKO,
-                                clock_output);
-       if (ret != 0) {
-               dev_err(component->dev, "Failed to enable clock output: %d\n", ret);
-               return ret;
-       }
 
-       ret = regmap_update_bits(pcm512x->regmap, PCM512x_MASTER_MODE,
-                                PCM512x_RLRK | PCM512x_RBCK,
-                                master_mode);
-       if (ret != 0) {
-               dev_err(component->dev, "Failed to enable master mode: %d\n", ret);
-               return ret;
-       }
-
-       if (pcm512x->pll_out) {
                gpio = PCM512x_G1OE << (pcm512x->pll_out - 1);
                ret = regmap_update_bits(pcm512x->regmap, PCM512x_GPIO_EN,
                                         gpio, gpio);
                return ret;
        }
 
+skip_pll:
+       ret = regmap_update_bits(pcm512x->regmap, PCM512x_BCLK_LRCLK_CFG,
+                                PCM512x_BCKP | PCM512x_BCKO | PCM512x_LRKO,
+                                clock_output);
+       if (ret != 0) {
+               dev_err(component->dev, "Failed to enable clock output: %d\n", ret);
+               return ret;
+       }
+
+       ret = regmap_update_bits(pcm512x->regmap, PCM512x_MASTER_MODE,
+                                PCM512x_RLRK | PCM512x_RBCK,
+                                master_mode);
+       if (ret != 0) {
+               dev_err(component->dev, "Failed to enable master mode: %d\n", ret);
+               return ret;
+       }
+
        return 0;
 }