*/
static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
{
+ struct rxrpc_ackpacket *ack = NULL;
struct rxrpc_txbuf *txb;
+ struct sk_buff *ack_skb = NULL;
unsigned long resend_at;
rxrpc_seq_t transmitted = READ_ONCE(call->tx_transmitted);
ktime_t now, max_age, oldest, ack_ts;
bool unacked = false;
+ unsigned int i;
LIST_HEAD(retrans_queue);
_enter("{%d,%d}", call->acks_hard_ack, call->tx_top);
now = ktime_get_real();
max_age = ktime_sub_us(now, jiffies_to_usecs(call->peer->rto_j));
+ oldest = now;
+
+ /* See if there's an ACK saved with a soft-ACK table in it. */
+ if (call->acks_soft_tbl) {
+ spin_lock_bh(&call->acks_ack_lock);
+ ack_skb = call->acks_soft_tbl;
+ if (ack_skb) {
+ rxrpc_get_skb(ack_skb, rxrpc_skb_ack);
+ ack = (void *)ack_skb->data + sizeof(struct rxrpc_wire_header);
+ }
+ spin_unlock_bh(&call->acks_ack_lock);
+ }
+
+ if (list_empty(&call->tx_buffer))
+ goto no_resend;
spin_lock(&call->tx_lock);
- /* Scan the packet list without dropping the lock and decide which of
- * the packets in the Tx buffer we're going to resend and what the new
- * resend timeout will be.
- */
+ if (list_empty(&call->tx_buffer))
+ goto no_further_resend;
+
trace_rxrpc_resend(call);
- oldest = now;
- list_for_each_entry(txb, &call->tx_buffer, call_link) {
- if (test_bit(RXRPC_TXBUF_ACKED, &txb->flags))
- continue;
- if (after(txb->seq, transmitted))
- break;
+ txb = list_first_entry(&call->tx_buffer, struct rxrpc_txbuf, call_link);
- rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked);
+ /* Scan the soft ACK table without dropping the lock and resend any
+ * explicitly NAK'd packets.
+ */
+ if (ack) {
+ for (i = 0; i < ack->nAcks; i++) {
+ rxrpc_seq_t seq;
- if (test_bit(RXRPC_TXBUF_RESENT, &txb->flags)) {
- if (ktime_after(txb->last_sent, max_age)) {
- if (ktime_before(txb->last_sent, oldest))
- oldest = txb->last_sent;
+ if (ack->acks[i] & 1)
continue;
+ seq = ntohl(ack->firstPacket) + i;
+ if (after(txb->seq, transmitted))
+ break;
+ if (after(txb->seq, seq))
+ continue; /* A new hard ACK probably came in */
+ list_for_each_entry_from(txb, &call->tx_buffer, call_link) {
+ if (txb->seq == seq)
+ goto found_txb;
+ }
+ goto no_further_resend;
+
+ found_txb:
+ if (after(ntohl(txb->wire.serial), call->acks_highest_serial))
+ continue; /* Ack point not yet reached */
+
+ rxrpc_see_txbuf(txb, rxrpc_txbuf_see_unacked);
+
+ if (list_empty(&txb->tx_link)) {
+ rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
+ rxrpc_get_call(call, rxrpc_call_got_tx);
+ list_add_tail(&txb->tx_link, &retrans_queue);
+ set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
}
- unacked = true;
+
+ trace_rxrpc_retransmit(call, txb->seq,
+ ktime_to_ns(ktime_sub(txb->last_sent,
+ max_age)));
+
+ if (list_is_last(&txb->call_link, &call->tx_buffer))
+ goto no_further_resend;
+ txb = list_next_entry(txb, call_link);
}
+ }
- rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
- list_move_tail(&txb->tx_link, &retrans_queue);
+ /* Fast-forward through the Tx queue to the point the peer says it has
+ * seen. Anything between the soft-ACK table and that point will get
+ * ACK'd or NACK'd in due course, so don't worry about it here; here we
+ * need to consider retransmitting anything beyond that point.
+ *
+ * Note that ACK for a packet can beat the update of tx_transmitted.
+ */
+ if (after_eq(READ_ONCE(call->acks_prev_seq), READ_ONCE(call->tx_transmitted)))
+ goto no_further_resend;
+
+ list_for_each_entry_from(txb, &call->tx_buffer, call_link) {
+ if (before_eq(txb->seq, READ_ONCE(call->acks_prev_seq)))
+ continue;
+ if (after(txb->seq, READ_ONCE(call->tx_transmitted)))
+ break; /* Not transmitted yet */
+
+ if (ack && ack->reason == RXRPC_ACK_PING_RESPONSE &&
+ before(ntohl(txb->wire.serial), ntohl(ack->serial)))
+ goto do_resend; /* Wasn't accounted for by a more recent ping. */
+
+ if (ktime_after(txb->last_sent, max_age)) {
+ if (ktime_before(txb->last_sent, oldest))
+ oldest = txb->last_sent;
+ continue;
+ }
+
+ do_resend:
+ unacked = true;
+ if (list_empty(&txb->tx_link)) {
+ rxrpc_get_txbuf(txb, rxrpc_txbuf_get_retrans);
+ list_add_tail(&txb->tx_link, &retrans_queue);
+ set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
+ rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
+ }
}
+no_further_resend:
spin_unlock(&call->tx_lock);
+no_resend:
+ rxrpc_free_skb(ack_skb, rxrpc_skb_freed);
resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest)));
resend_at += jiffies + rxrpc_get_rto_backoff(call->peer,
while ((txb = list_first_entry_or_null(&retrans_queue,
struct rxrpc_txbuf, tx_link))) {
list_del_init(&txb->tx_link);
- set_bit(RXRPC_TXBUF_RESENT, &txb->flags);
- rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
rxrpc_send_data_packet(call, txb);
rxrpc_put_txbuf(txb, rxrpc_txbuf_put_trans);
switch (call->cong_mode) {
case RXRPC_CALL_SLOW_START:
- if (summary->nr_nacks > 0)
+ if (summary->saw_nacks)
goto packet_loss_detected;
if (summary->cumulative_acks > 0)
cwnd += 1;
goto out;
case RXRPC_CALL_CONGEST_AVOIDANCE:
- if (summary->nr_nacks > 0)
+ if (summary->saw_nacks)
goto packet_loss_detected;
/* We analyse the number of packets that get ACK'd per RTT
goto out;
case RXRPC_CALL_PACKET_LOSS:
- if (summary->nr_nacks == 0)
+ if (!summary->saw_nacks)
goto resume_normality;
if (summary->new_low_nack) {
} else {
change = rxrpc_cong_progress;
cwnd = call->cong_ssthresh;
- if (summary->nr_nacks == 0)
+ if (!summary->saw_nacks)
goto resume_normality;
}
goto out;
list_for_each_entry_rcu(txb, &call->tx_buffer, call_link, false) {
if (before_eq(txb->seq, call->acks_hard_ack))
continue;
- if (!test_bit(RXRPC_TXBUF_ACKED, &txb->flags))
- summary->nr_rot_new_acks++;
+ summary->nr_rot_new_acks++;
if (test_bit(RXRPC_TXBUF_LAST, &txb->flags)) {
set_bit(RXRPC_CALL_TX_LAST, &call->flags);
rot_last = true;
*/
static void rxrpc_input_check_for_lost_ack(struct rxrpc_call *call)
{
- struct rxrpc_txbuf *txb;
- rxrpc_seq_t top, bottom;
- bool resend = false;
-
- bottom = READ_ONCE(call->acks_hard_ack) + 1;
- top = READ_ONCE(call->acks_lost_top);
- if (before(bottom, top)) {
- list_for_each_entry_rcu(txb, &call->tx_buffer, call_link, false) {
- if (test_bit(RXRPC_TXBUF_ACKED, &txb->flags))
- continue;
- set_bit(RXRPC_TXBUF_RETRANS, &txb->flags);
- resend = true;
- }
- }
-
- if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
+ if (after(call->acks_lost_top, call->acks_prev_seq) &&
+ !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
rxrpc_queue_call(call);
}
rxrpc_seq_t seq, int nr_acks,
struct rxrpc_ack_summary *summary)
{
- struct rxrpc_txbuf *txb;
+ unsigned int i;
- list_for_each_entry_rcu(txb, &call->tx_buffer, call_link, false) {
- if (before(txb->seq, seq))
- continue;
- if (after_eq(txb->seq, seq + nr_acks))
- break;
- switch (acks[txb->seq - seq]) {
- case RXRPC_ACK_TYPE_ACK:
+ for (i = 0; i < nr_acks; i++) {
+ if (acks[i] == RXRPC_ACK_TYPE_ACK) {
summary->nr_acks++;
- if (test_bit(RXRPC_TXBUF_ACKED, &txb->flags))
- continue;
- /* A lot of the time the packet is going to
- * have been ACK.'d already.
- */
- clear_bit(RXRPC_TXBUF_NACKED, &txb->flags);
- set_bit(RXRPC_TXBUF_ACKED, &txb->flags);
summary->nr_new_acks++;
- break;
- case RXRPC_ACK_TYPE_NACK:
- if (!summary->nr_nacks &&
- call->acks_lowest_nak != seq) {
- call->acks_lowest_nak = seq;
+ } else {
+ if (!summary->saw_nacks &&
+ call->acks_lowest_nak != seq + i) {
+ call->acks_lowest_nak = seq + i;
summary->new_low_nack = true;
}
- summary->nr_nacks++;
- if (test_bit(RXRPC_TXBUF_NACKED, &txb->flags))
- continue;
- summary->nr_new_nacks++;
- clear_bit(RXRPC_TXBUF_ACKED, &txb->flags);
- set_bit(RXRPC_TXBUF_NACKED, &txb->flags);
- set_bit(RXRPC_TXBUF_RETRANS, &txb->flags);
- break;
- default:
- return rxrpc_proto_abort("SFT", call, 0);
+ summary->saw_nacks = true;
}
}
}
static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
{
struct rxrpc_ack_summary summary = { 0 };
+ struct rxrpc_ackpacket ack;
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
- union {
- struct rxrpc_ackpacket ack;
- struct rxrpc_ackinfo info;
- u8 acks[RXRPC_MAXACKS];
- } buf;
+ struct rxrpc_ackinfo info;
+ struct sk_buff *skb_old = NULL, *skb_put = skb;
rxrpc_serial_t ack_serial, acked_serial;
rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt;
int nr_acks, offset, ioffset;
_enter("");
offset = sizeof(struct rxrpc_wire_header);
- if (skb_copy_bits(skb, offset, &buf.ack, sizeof(buf.ack)) < 0) {
- _debug("extraction failure");
- return rxrpc_proto_abort("XAK", call, 0);
+ if (skb_copy_bits(skb, offset, &ack, sizeof(ack)) < 0) {
+ rxrpc_proto_abort("XAK", call, 0);
+ goto out_not_locked;
}
- offset += sizeof(buf.ack);
+ offset += sizeof(ack);
ack_serial = sp->hdr.serial;
- acked_serial = ntohl(buf.ack.serial);
- first_soft_ack = ntohl(buf.ack.firstPacket);
- prev_pkt = ntohl(buf.ack.previousPacket);
+ acked_serial = ntohl(ack.serial);
+ first_soft_ack = ntohl(ack.firstPacket);
+ prev_pkt = ntohl(ack.previousPacket);
hard_ack = first_soft_ack - 1;
- nr_acks = buf.ack.nAcks;
- summary.ack_reason = (buf.ack.reason < RXRPC_ACK__INVALID ?
- buf.ack.reason : RXRPC_ACK__INVALID);
+ nr_acks = ack.nAcks;
+ summary.ack_reason = (ack.reason < RXRPC_ACK__INVALID ?
+ ack.reason : RXRPC_ACK__INVALID);
trace_rxrpc_rx_ack(call, ack_serial, acked_serial,
first_soft_ack, prev_pkt,
summary.ack_reason, nr_acks);
- rxrpc_inc_stat(call->rxnet, stat_rx_acks[buf.ack.reason]);
+ rxrpc_inc_stat(call->rxnet, stat_rx_acks[ack.reason]);
- switch (buf.ack.reason) {
+ switch (ack.reason) {
case RXRPC_ACK_PING_RESPONSE:
- rxrpc_input_ping_response(call, skb->tstamp, acked_serial,
- ack_serial);
rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
rxrpc_rtt_rx_ping_response);
break;
break;
}
- if (buf.ack.reason == RXRPC_ACK_PING) {
+ if (ack.reason == RXRPC_ACK_PING) {
_proto("Rx ACK %%%u PING Request", ack_serial);
rxrpc_send_ACK(call, RXRPC_ACK_PING_RESPONSE, ack_serial,
rxrpc_propose_ack_respond_to_ping);
* indicates that the client address changed due to NAT. The server
* lost the call because it switched to a different peer.
*/
- if (unlikely(buf.ack.reason == RXRPC_ACK_EXCEEDS_WINDOW) &&
+ if (unlikely(ack.reason == RXRPC_ACK_EXCEEDS_WINDOW) &&
first_soft_ack == 1 &&
prev_pkt == 0 &&
rxrpc_is_client_call(call)) {
* indicate a change of address. However, we can retransmit the call
* if we still have it buffered to the beginning.
*/
- if (unlikely(buf.ack.reason == RXRPC_ACK_OUT_OF_SEQUENCE) &&
+ if (unlikely(ack.reason == RXRPC_ACK_OUT_OF_SEQUENCE) &&
first_soft_ack == 1 &&
prev_pkt == 0 &&
call->acks_hard_ack == 0 &&
trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
first_soft_ack, call->acks_first_seq,
prev_pkt, call->acks_prev_seq);
- return;
+ goto out_not_locked;
}
- buf.info.rxMTU = 0;
+ info.rxMTU = 0;
ioffset = offset + nr_acks + 3;
- if (skb->len >= ioffset + sizeof(buf.info) &&
- skb_copy_bits(skb, ioffset, &buf.info, sizeof(buf.info)) < 0)
- return rxrpc_proto_abort("XAI", call, 0);
+ if (skb->len >= ioffset + sizeof(info) &&
+ skb_copy_bits(skb, ioffset, &info, sizeof(info)) < 0) {
+ rxrpc_proto_abort("XAI", call, 0);
+ goto out_not_locked;
+ }
+
+ if (nr_acks > 0)
+ skb_condense(skb);
spin_lock(&call->input_lock);
call->acks_first_seq = first_soft_ack;
call->acks_prev_seq = prev_pkt;
- if (buf.ack.reason != RXRPC_ACK_PING &&
- after(acked_serial, call->acks_highest_serial))
- call->acks_highest_serial = acked_serial;
+ switch (ack.reason) {
+ case RXRPC_ACK_PING:
+ break;
+ case RXRPC_ACK_PING_RESPONSE:
+ rxrpc_input_ping_response(call, skb->tstamp, acked_serial,
+ ack_serial);
+ fallthrough;
+ default:
+ if (after(acked_serial, call->acks_highest_serial))
+ call->acks_highest_serial = acked_serial;
+ break;
+ }
/* Parse rwind and mtu sizes if provided. */
- if (buf.info.rxMTU)
- rxrpc_input_ackinfo(call, skb, &buf.info);
+ if (info.rxMTU)
+ rxrpc_input_ackinfo(call, skb, &info);
if (first_soft_ack == 0) {
rxrpc_proto_abort("AK0", call, 0);
}
if (nr_acks > 0) {
- if (skb_copy_bits(skb, offset, buf.acks, nr_acks) < 0) {
+ if (offset > (int)skb->len - nr_acks) {
rxrpc_proto_abort("XSA", call, 0);
goto out;
}
- rxrpc_input_soft_acks(call, buf.acks, first_soft_ack, nr_acks,
- &summary);
+
+ spin_lock(&call->acks_ack_lock);
+ skb_old = call->acks_soft_tbl;
+ call->acks_soft_tbl = skb;
+ spin_unlock(&call->acks_ack_lock);
+
+ rxrpc_input_soft_acks(call, skb->data + offset, first_soft_ack,
+ nr_acks, &summary);
+ skb_put = NULL;
+ } else if (call->acks_soft_tbl) {
+ spin_lock(&call->acks_ack_lock);
+ skb_old = call->acks_soft_tbl;
+ call->acks_soft_tbl = NULL;
+ spin_unlock(&call->acks_ack_lock);
}
if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) &&
rxrpc_congestion_management(call, skb, &summary, acked_serial);
out:
spin_unlock(&call->input_lock);
+out_not_locked:
+ rxrpc_free_skb(skb_put, rxrpc_skb_freed);
+ rxrpc_free_skb(skb_old, rxrpc_skb_freed);
}
/*
case RXRPC_PACKET_TYPE_ACK:
rxrpc_input_ack(call, skb);
- break;
+ goto no_free;
case RXRPC_PACKET_TYPE_BUSY:
_proto("Rx BUSY %%%u", sp->hdr.serial);
projects / nvme.git / commitdiff