improved MTU calculation for GlobalProtect ESP

On Wed, Aug 16, 2017 at 8:03 PM, Daniel Lenski wrote:

> I believe the correct [ESP padding] algorithm is actually as follows:
>
> * From payload MTU, add 2 footer bytes, round *up* to a multiple of
>   the blocksize. Add the size of the MAC, IV, and other headers. That's
>   the size of the packet on the wire.
> * From wire packet MTU, subtract headers and MAC and IV, round *down*
>   to a multiple of blocksize, subtract TWO footer bytes, and that's the
>   largest payload you can carry.

Signed-off-by: Daniel Lenski <dlenski@gmail.com>
Signed-off-by: David Woodhouse <dwmw2@infradead.org>

diff --git a/gpst.c b/gpst.c
index 77d0d7b0b7005a690d4b961012b3a2fbf3269736..87cfbc99282fc7dada44a8acaa9140cdeb087c39 100644 (file)
--- a/gpst.c
+++ b/gpst.c
@@ -267,14 +267,17 @@ out:
 }
 
 
-/* XXX: Look at set_esp_algo() and tell me again what the biggest supported IV is? */
-#define ESP_OVERHEAD (4 /* SPI */ + 4 /* sequence number */ + \
-                      1 /* pad length */ + 1 /* next header */ + \
-                      16 /* max padding */ )
+#define ESP_HEADER_SIZE (4 /* SPI */ + 4 /* sequence number */)
+#define ESP_FOOTER_SIZE (1 /* pad length */ + 1 /* next header */)
 #define UDP_HEADER_SIZE 8
 #define IPV4_HEADER_SIZE 20
 #define IPV6_HEADER_SIZE 40
 
+/* Based on cstp.c's calculate_mtu().
+ *
+ * With HTTPS tunnel, there are 21 bytes of overhead beyond the
+ * TCP MSS: 5 bytes for TLS and 16 for GPST.
+ */
 static int calculate_mtu(struct openconnect_info *vpninfo)
 {
        int mtu = vpninfo->reqmtu, base_mtu = vpninfo->basemtu;
@@ -296,9 +299,9 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 
                        if (!base_mtu) {
                                if (ti.tcpi_rcv_mss < ti.tcpi_snd_mss)
-                                       base_mtu = ti.tcpi_rcv_mss - 13;
+                                       base_mtu = ti.tcpi_rcv_mss - 21;
                                else
-                                       base_mtu = ti.tcpi_snd_mss - 13;
+                                       base_mtu = ti.tcpi_snd_mss - 21;
                        }
                }
        }
@@ -310,7 +313,7 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
                if (!getsockopt(vpninfo->ssl_fd, IPPROTO_TCP, TCP_MAXSEG,
                                &mss, &mss_size)) {
                        vpn_progress(vpninfo, PRG_DEBUG, _("TCP_MAXSEG %d\n"), mss);
-                       base_mtu = mss - 13;
+                       base_mtu = mss - 21;
                }
        }
 #endif
@@ -323,14 +326,18 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
                base_mtu = 1280;
 
        if (!mtu) {
-               /* remove IP/UDP and ESP overhead from base MTU to calculate tunnel MTU */
-               mtu = ( base_mtu - UDP_HEADER_SIZE - ESP_OVERHEAD
-                       - (vpninfo->hmac_key_len ? : 20) /* biggest supported MAC (SHA1) */
+               /* remove ESP, UDP, IP headers from base (wire) MTU */
+               mtu = ( base_mtu - UDP_HEADER_SIZE - ESP_HEADER_SIZE
+                       - 12 /* both supported algos (SHA1 and MD5) have 96-bit MAC lengths (RFC2403 and RFC2404) */
                        - (vpninfo->enc_key_len ? : 32) /* biggest supported IV (AES-256) */ );
                if (vpninfo->peer_addr->sa_family == AF_INET6)
                        mtu -= IPV6_HEADER_SIZE;
                else
                        mtu -= IPV4_HEADER_SIZE;
+               /* round down to a multiple of blocksize */
+               mtu -= mtu % (vpninfo->enc_key_len ? : 32);
+               /* subtract ESP footer, which is included in the payload before padding to the blocksize */
+               mtu -= ESP_FOOTER_SIZE;
        }
        return mtu;
 }