#define RTL8366RB_PAACR_RX_PAUSE       BIT(6)
 #define RTL8366RB_PAACR_AN             BIT(7)
 
-#define RTL8366RB_PAACR_CPU_PORT       (RTL8366RB_PAACR_SPEED_1000M | \
-                                        RTL8366RB_PAACR_FULL_DUPLEX | \
-                                        RTL8366RB_PAACR_LINK_UP | \
-                                        RTL8366RB_PAACR_TX_PAUSE | \
-                                        RTL8366RB_PAACR_RX_PAUSE)
-
 /* bits 0..7 = port 0, bits 8..15 = port 1 */
 #define RTL8366RB_PSTAT0               0x0014
 /* bits 0..7 = port 2, bits 8..15 = port 3 */
                      int speed, int duplex, bool tx_pause, bool rx_pause)
 {
        struct realtek_priv *priv = ds->priv;
+       unsigned int val;
        int ret;
 
+       /* Allow forcing the mode on the fixed CPU port, no autonegotiation.
+        * We assume autonegotiation works on the PHY-facing ports.
+        */
        if (port != priv->cpu_port)
                return;
 
        dev_dbg(priv->dev, "MAC link up on CPU port (%d)\n", port);
 
-       /* Force the fixed CPU port into 1Gbit mode, no autonegotiation */
        ret = regmap_update_bits(priv->map, RTL8366RB_MAC_FORCE_CTRL_REG,
                                 BIT(port), BIT(port));
        if (ret) {
-               dev_err(priv->dev, "failed to force 1Gbit on CPU port\n");
+               dev_err(priv->dev, "failed to force CPU port\n");
                return;
        }
 
+       /* Conjure port config */
+       switch (speed) {
+       case SPEED_10:
+               val = RTL8366RB_PAACR_SPEED_10M;
+               break;
+       case SPEED_100:
+               val = RTL8366RB_PAACR_SPEED_100M;
+               break;
+       case SPEED_1000:
+               val = RTL8366RB_PAACR_SPEED_1000M;
+               break;
+       default:
+               val = RTL8366RB_PAACR_SPEED_1000M;
+               break;
+       }
+
+       if (duplex == DUPLEX_FULL)
+               val |= RTL8366RB_PAACR_FULL_DUPLEX;
+
+       if (tx_pause)
+               val |=  RTL8366RB_PAACR_TX_PAUSE;
+
+       if (rx_pause)
+               val |= RTL8366RB_PAACR_RX_PAUSE;
+
+       val |= RTL8366RB_PAACR_LINK_UP;
+
        ret = regmap_update_bits(priv->map, RTL8366RB_PAACR2,
                                 0xFF00U,
-                                RTL8366RB_PAACR_CPU_PORT << 8);
+                                val << 8);
        if (ret) {
                dev_err(priv->dev, "failed to set PAACR on CPU port\n");
                return;
        }
 
+       dev_dbg(priv->dev, "set PAACR to %04x\n", val);
+
        /* Enable the CPU port */
        ret = regmap_update_bits(priv->map, RTL8366RB_PECR, BIT(port),
                                 0);