{
        struct drm_crtc_state *crtc_state;
        struct drm_crtc *crtc;
-       unsigned char dsp2_mux = 0;
-       unsigned char dsp3_mux = 3;
-       unsigned char dsp4_mux = 3;
-       unsigned char dsp5_mux = 3;
+       unsigned char mux;
        unsigned int i;
        u32 reg;
 
                struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
                struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 
-               if (!crtc_state->active)
+               if (!vc4_state->update_muxing)
                        continue;
 
                switch (vc4_crtc->data->hvs_output) {
                case 2:
-                       dsp2_mux = (vc4_state->assigned_channel == 2) ? 0 : 1;
+                       mux = (vc4_state->assigned_channel == 2) ? 0 : 1;
+                       reg = HVS_READ(SCALER_DISPECTRL);
+                       HVS_WRITE(SCALER_DISPECTRL,
+                                 (reg & ~SCALER_DISPECTRL_DSP2_MUX_MASK) |
+                                 VC4_SET_FIELD(mux, SCALER_DISPECTRL_DSP2_MUX));
                        break;
 
                case 3:
-                       dsp3_mux = vc4_state->assigned_channel;
+                       if (vc4_state->assigned_channel == VC4_HVS_CHANNEL_DISABLED)
+                               mux = 3;
+                       else
+                               mux = vc4_state->assigned_channel;
+
+                       reg = HVS_READ(SCALER_DISPCTRL);
+                       HVS_WRITE(SCALER_DISPCTRL,
+                                 (reg & ~SCALER_DISPCTRL_DSP3_MUX_MASK) |
+                                 VC4_SET_FIELD(mux, SCALER_DISPCTRL_DSP3_MUX));
                        break;
 
                case 4:
-                       dsp4_mux = vc4_state->assigned_channel;
+                       if (vc4_state->assigned_channel == VC4_HVS_CHANNEL_DISABLED)
+                               mux = 3;
+                       else
+                               mux = vc4_state->assigned_channel;
+
+                       reg = HVS_READ(SCALER_DISPEOLN);
+                       HVS_WRITE(SCALER_DISPEOLN,
+                                 (reg & ~SCALER_DISPEOLN_DSP4_MUX_MASK) |
+                                 VC4_SET_FIELD(mux, SCALER_DISPEOLN_DSP4_MUX));
+
                        break;
 
                case 5:
-                       dsp5_mux = vc4_state->assigned_channel;
+                       if (vc4_state->assigned_channel == VC4_HVS_CHANNEL_DISABLED)
+                               mux = 3;
+                       else
+                               mux = vc4_state->assigned_channel;
+
+                       reg = HVS_READ(SCALER_DISPDITHER);
+                       HVS_WRITE(SCALER_DISPDITHER,
+                                 (reg & ~SCALER_DISPDITHER_DSP5_MUX_MASK) |
+                                 VC4_SET_FIELD(mux, SCALER_DISPDITHER_DSP5_MUX));
                        break;
 
                default:
                        break;
                }
        }
-
-       reg = HVS_READ(SCALER_DISPECTRL);
-       HVS_WRITE(SCALER_DISPECTRL,
-                 (reg & ~SCALER_DISPECTRL_DSP2_MUX_MASK) |
-                 VC4_SET_FIELD(dsp2_mux, SCALER_DISPECTRL_DSP2_MUX));
-
-       reg = HVS_READ(SCALER_DISPCTRL);
-       HVS_WRITE(SCALER_DISPCTRL,
-                 (reg & ~SCALER_DISPCTRL_DSP3_MUX_MASK) |
-                 VC4_SET_FIELD(dsp3_mux, SCALER_DISPCTRL_DSP3_MUX));
-
-       reg = HVS_READ(SCALER_DISPEOLN);
-       HVS_WRITE(SCALER_DISPEOLN,
-                 (reg & ~SCALER_DISPEOLN_DSP4_MUX_MASK) |
-                 VC4_SET_FIELD(dsp4_mux, SCALER_DISPEOLN_DSP4_MUX));
-
-       reg = HVS_READ(SCALER_DISPDITHER);
-       HVS_WRITE(SCALER_DISPDITHER,
-                 (reg & ~SCALER_DISPDITHER_DSP5_MUX_MASK) |
-                 VC4_SET_FIELD(dsp5_mux, SCALER_DISPDITHER_DSP5_MUX));
 }
 
 static void
                struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
                unsigned int matching_channels;
 
-               if (old_crtc_state->enable && !new_crtc_state->enable) {
-                       hvs_new_state->unassigned_channels |= BIT(old_vc4_crtc_state->assigned_channel);
-                       new_vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
-               }
-
-               if (!new_crtc_state->enable)
+               /* Nothing to do here, let's skip it */
+               if (old_crtc_state->enable == new_crtc_state->enable)
                        continue;
 
-               if (new_vc4_crtc_state->assigned_channel != VC4_HVS_CHANNEL_DISABLED)
+               /* Muxing will need to be modified, mark it as such */
+               new_vc4_crtc_state->update_muxing = true;
+
+               /* If we're disabling our CRTC, we put back our channel */
+               if (!new_crtc_state->enable) {
+                       hvs_new_state->unassigned_channels |= BIT(old_vc4_crtc_state->assigned_channel);
+                       new_vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
                        continue;
+               }
 
                /*
                 * The problem we have to solve here is that we have