atmel_port->desc_rx = NULL;
        atmel_port->chan_rx = NULL;
        atmel_port->cookie_rx = -EINVAL;
-
-       if (!atmel_port->is_usart)
-               del_timer_sync(&atmel_port->uart_timer);
 }
 
 static void atmel_rx_from_dma(struct uart_port *port)
                                 DMA_FROM_DEVICE);
                kfree(pdc->buf);
        }
-
-       if (!atmel_port->is_usart)
-               del_timer_sync(&atmel_port->uart_timer);
 }
 
 static void atmel_rx_from_pdc(struct uart_port *port)
        /* enable xmit & rcvr */
        UART_PUT_CR(port, ATMEL_US_TXEN | ATMEL_US_RXEN);
 
+       setup_timer(&atmel_port->uart_timer,
+                       atmel_uart_timer_callback,
+                       (unsigned long)port);
+
        if (atmel_use_pdc_rx(port)) {
                /* set UART timeout */
                if (!atmel_port->is_usart) {
-                       setup_timer(&atmel_port->uart_timer,
-                                       atmel_uart_timer_callback,
-                                       (unsigned long)port);
                        mod_timer(&atmel_port->uart_timer,
                                        jiffies + uart_poll_timeout(port));
                /* set USART timeout */
        } else if (atmel_use_dma_rx(port)) {
                /* set UART timeout */
                if (!atmel_port->is_usart) {
-                       setup_timer(&atmel_port->uart_timer,
-                                       atmel_uart_timer_callback,
-                                       (unsigned long)port);
                        mod_timer(&atmel_port->uart_timer,
                                        jiffies + uart_poll_timeout(port));
                /* set USART timeout */
 {
        struct atmel_uart_port *atmel_port = to_atmel_uart_port(port);
 
+       /*
+        * Prevent any tasklets being scheduled during
+        * cleanup
+        */
+       del_timer_sync(&atmel_port->uart_timer);
+
        /*
         * Clear out any scheduled tasklets before
         * we destroy the buffers