diff --git a/README.md b/README.md index 109c797..da402d3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A peer-reviewed [paper on TCP ROCCET](https://opendl.ifip-tc6.org/db/conf/wons/w ### Versions -- latest on master branch +- latest on master branch (requires kernel version 6.15 or higher) - Improved min RTT probing - Packet pacing - Code refactor diff --git a/tcp_roccet.c b/tcp_roccet.c index 83ed77c..83c4b1a 100644 --- a/tcp_roccet.c +++ b/tcp_roccet.c @@ -45,14 +45,13 @@ * this behaves the same as the original Reno. */ -#include "tcp_roccet.h" -#include -#include +#include "linux/limits.h" #include #include #include #include #include +#include "tcp_roccet.h" /* Scale factor beta calculation (max_cwnd = snd_cwnd * beta) */ #define BICTCP_BETA_SCALE 1024 @@ -109,14 +108,16 @@ static __always_inline void roccettcp_reset(struct roccettcp *ca) ca->ece_received = false; ca->roccet_last_event_time_us = 0; - ca->ack_rate.last_rate = 0; - /* Initialize to current time to avoid overflow in ack rate calculation */ - ca->ack_rate.last_rate_time = jiffies_to_usecs(tcp_jiffies32); - ca->ack_rate.curr_rate = 0; - ca->ack_rate.cnt = 0; + ca->ack_rate_last_rate = 0; + /* Initialize to current time to avoid an + * overflow in the ack rate calculation + */ + ca->ack_rate_last_rate_time = jiffies_to_usecs(tcp_jiffies32); + ca->ack_rate_curr_rate = 0; + ca->ack_rate_cnt = 0; - /* Start state is LAUNCH */ - ca->state = LAUNCH; + /* Start state is LAUNCH */ + ca->state = LAUNCH; } /* Return true if ROCCET is in min RTT probing. @@ -138,8 +139,9 @@ static __always_inline void update_min_rtt(struct sock *sk) /* Probe for the min RTT in ROCCET_NEXT_MIN_RTT_PROBE seconds * if no other update occurs. */ - ca->next_min_rtt_probe = jiffies_to_usecs(tcp_jiffies32) + - ROCCET_NEXT_MIN_RTT_PROBE * USEC_PER_MSEC; + ca->next_min_rtt_probe = + jiffies_to_usecs(tcp_jiffies32) + + ROCCET_NEXT_MIN_RTT_PROBE * USEC_PER_MSEC; } } @@ -147,9 +149,9 @@ static __always_inline void update_min_rtt(struct sock *sk) */ static __always_inline s32 get_ack_rate_diff(struct roccettcp *ca) { - if (ca->ack_rate.curr_rate < ca->ack_rate.last_rate) + if (ca->ack_rate_curr_rate < ca->ack_rate_last_rate) return 0; - return (s32)(ca->ack_rate.curr_rate - ca->ack_rate.last_rate); + return (s32)(ca->ack_rate_curr_rate - ca->ack_rate_last_rate); } /* Update ack rate sampled by 100ms. @@ -159,26 +161,34 @@ static __always_inline void update_ack_rate(struct sock *sk, u32 acked, u32 now) struct roccettcp *ca = inet_csk_ca(sk); s32 interval = USEC_PER_MSEC * 100; - s32 time_delta = (s32)(ca->ack_rate.last_rate_time - now); + s32 time_delta = (s32)(ca->ack_rate_last_rate_time - now); const s32 idle_threshold = USEC_PER_SEC * 2; // Check if the time has arrived in the new interval if (time_delta < -interval) { - // Check if the connection was idle for X seconds (e.g. no ACK for X seconds) + /* Check if the connection was idle for X seconds + * (e.g. no ACK for X seconds) + */ if (time_delta < -idle_threshold) { - // Reset ack counting as if a new connection was created - ca->ack_rate.last_rate = 0; - ca->ack_rate.last_rate_time = jiffies_to_usecs(tcp_jiffies32); - ca->ack_rate.curr_rate = 0; - ca->ack_rate.cnt = 0; + /* Reset ack counting as if a new + * connection was created + */ + ca->ack_rate_last_rate = 0; + ca->ack_rate_last_rate_time = + jiffies_to_usecs(tcp_jiffies32); + ca->ack_rate_curr_rate = 0; + ca->ack_rate_cnt = 0; } else { - ca->ack_rate.last_rate_time = now; - ca->ack_rate.last_rate = ca->ack_rate.curr_rate; - ca->ack_rate.curr_rate = ca->ack_rate.cnt; - ca->ack_rate.cnt = acked; // start counting for the new interval + ca->ack_rate_last_rate_time = now; + ca->ack_rate_last_rate = ca->ack_rate_curr_rate; + ca->ack_rate_curr_rate = ca->ack_rate_cnt; + ca->ack_rate_cnt = + acked; // start counting for the new interval } } else { - ca->ack_rate.cnt += acked; + // Cap the ack count to avoid overflow + ca->ack_rate_cnt = min_t(u32, ca->ack_rate_cnt + acked, + U16_MAX); } } @@ -213,11 +223,12 @@ static __always_inline void update_srrtt(struct sock *sk) * 0 is a valid value for rrtt. */ u32 rrtt = div_u64(100 * (u64)(ca->curr_rtt - ca->curr_min_rtt), - ca->curr_min_rtt); + ca->curr_min_rtt); // (1 - alpha) * srRTT + alpha * rRTT ca->curr_srrtt = ((100 - ROCCET_ALPHA_TIMES_100) * ca->curr_srrtt + - ROCCET_ALPHA_TIMES_100 * rrtt) / 100; + ROCCET_ALPHA_TIMES_100 * rrtt) / + 100; } /* Do a ROCCET congestion event. @@ -230,14 +241,14 @@ static __always_inline void roccet_congestion_event(struct sock *sk, u32 now) ca->epoch_start = 0; ca->roccet_last_event_time_us = now; ca->cnt = 100 * tcp_snd_cwnd(tp); - /*Set W_max only if the current cwnd is larger */ - if (tcp_snd_cwnd(tp) > ca->last_max_cwnd) - ca->last_max_cwnd = tcp_snd_cwnd(tp); - tcp_snd_cwnd_set(tp, min(tp->snd_cwnd_clamp, - max((tcp_snd_cwnd(tp) * beta) / - BICTCP_BETA_SCALE, 2U))); - tp->snd_ssthresh = tcp_snd_cwnd(tp); - + /*Set W_max only if the current cwnd is larger */ + if (tcp_snd_cwnd(tp) > ca->last_max_cwnd) + ca->last_max_cwnd = tcp_snd_cwnd(tp); + tcp_snd_cwnd_set(tp, + min(tp->snd_cwnd_clamp, + max((tcp_snd_cwnd(tp) * beta) + / BICTCP_BETA_SCALE, 2U))); + tp->snd_ssthresh = tcp_snd_cwnd(tp); } /* Do minimum RTT probing. @@ -246,61 +257,61 @@ static __always_inline void roccet_min_rtt_probe(struct sock *sk, u32 now) { struct tcp_sock *tp = tcp_sk(sk); struct roccettcp *ca = inet_csk_ca(sk); - u32 interval, probe_cwnd; + u32 interval, probe_cwnd; - /* Do nothing if we are probing */ - if (before(now, ca->probe_min_rtt_until) && ca->probe_min_rtt_until > 0) + /* Do nothing if we are probing */ + if (before(now, ca->probe_min_rtt_until) && ca->probe_min_rtt_until > 0) return; - /* Start of min RTT probing*/ - if(ca->probe_min_rtt_until == 0){ - /* Probe 1*RTT or at least 200ms */ - interval = max(200 * USEC_PER_MSEC, ca->curr_rtt); + /* Start of min RTT probing*/ + if (ca->probe_min_rtt_until == 0) { + /* Probe 1*RTT or at least 200ms */ + interval = max(200 * USEC_PER_MSEC, ca->curr_rtt); - /* This is to handle deep shared buffers with loss-based - * congestion control like CUBIC. If the the cwnd is not limited - * by the application but falsely detected (see ROCCET paper), - * we have to empty the pipe more. - * If the limit detection is correct this will cause no harm - * to the tcp flow because the cwnd is not fully utilized and - * we set the cwnd to its previous value after probing. - */ - probe_cwnd = max(tcp_snd_cwnd(tp) / 2, TCP_INIT_CWND); - if (!tcp_is_cwnd_limited(sk)) - probe_cwnd = max(tcp_snd_cwnd(tp) / 3, TCP_INIT_CWND); + /* This is to handle deep shared buffers with loss-based + * congestion control like CUBIC. If the cwnd is not limited + * by the application but falsely detected (see ROCCET paper), + * we have to empty the pipe more. + * If the limit detection is correct this will cause no harm + * to the tcp flow because the cwnd is not fully utilized and + * we set the cwnd to its previous value after probing. + */ + probe_cwnd = max(tcp_snd_cwnd(tp) / 2, TCP_INIT_CWND); + if (!tcp_is_cwnd_limited(sk)) + probe_cwnd = max(tcp_snd_cwnd(tp) / 3, TCP_INIT_CWND); - ca->probe_min_rtt_until = now + interval; - ca->cwnd_before_min_rtt_probe = tcp_snd_cwnd(tp); + ca->probe_min_rtt_until = now + interval; + ca->cwnd_before_min_rtt_probe = tcp_snd_cwnd(tp); - /* Half the cwnd to drain the buffer for probing. - * Set the ssthresh to the probing cwnd otherwise - * the TCP state machine is in slow start. - */ - tcp_snd_cwnd_set(tp, probe_cwnd); - tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp); + /* Half the cwnd to drain the buffer for probing. + * Set the ssthresh to the probing cwnd otherwise + * the TCP state machine is in slow start. + */ + tcp_snd_cwnd_set(tp, probe_cwnd); + tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp); - /* Reset current min RTT to allow probing for - * a new lower and higher minimum RTT. - */ - ca->curr_min_rtt = ~0U; + /* Reset current min RTT to allow probing for + * a new lower and higher minimum RTT. + */ + ca->curr_min_rtt = ~0U; - /* Refill the pipe after probing. - * To this end we need the previous cwnd over the - * the probing interval. - */ - ca->refill_until = ca->probe_min_rtt_until + interval; - }else if(before(now, ca->refill_until)){ - /* Reset cwnd and refill the pipe. */ - if(ca->state != RTT_PROBE_REFILL){ - tcp_snd_cwnd_set(tp, ca->cwnd_before_min_rtt_probe); - tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp); - ca->state = RTT_PROBE_REFILL; - } - }else{ - /* End min RTT probing phase. */ - ca->probe_min_rtt_until = 0; - ca->state = ORBITER; - } + /* Refill the pipe after probing. + * To this end we need the previous cwnd over + * the probing interval. + */ + ca->refill_until = ca->probe_min_rtt_until + interval; + } else if (before(now, ca->refill_until)) { + /* Reset cwnd and refill the pipe. */ + if (ca->state != RTT_PROBE_REFILL) { + tcp_snd_cwnd_set(tp, ca->cwnd_before_min_rtt_probe); + tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp); + ca->state = RTT_PROBE_REFILL; + } + } else { + /* End min RTT probing phase. */ + ca->probe_min_rtt_until = 0; + ca->state = ORBITER; + } } static void roccettcp_init(struct sock *sk) @@ -312,7 +323,7 @@ static void roccettcp_init(struct sock *sk) if (initial_ssthresh) tcp_sk(sk)->snd_ssthresh = initial_ssthresh; - cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); //WRITE_ONCE(sk->sk_pacing_rate, 0); } @@ -492,177 +503,187 @@ tcp_friendliness: ca->cnt = max(ca->cnt, 2U); } -static void roccettcp_cong_avoid(struct sock *sk, u32 ack, - u32 acked) +static void roccettcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) { struct tcp_sock *tp = tcp_sk(sk); struct roccettcp *ca = inet_csk_ca(sk); u32 now = jiffies_to_usecs(tcp_jiffies32); bool evaluate_srrtt = false; - bool send_more_than_acked = false; + bool send_more_than_acked = false; u32 roccet_xj; u32 jitter; - u32 send, received; + u32 send, received; - if(ca->state == LAUNCH){ - /* LAUNCH: Detect an exit point for tcp slow start - * in networks with large buffers of multiple BDP - * Like in cellular networks (5G, ...). - * Or exit LAUNCH if cwnd is too large for application layer - * data rate (tcp cwnd validation). - */ - if ((ca->curr_srrtt > sr_rtt_upper_bound && - get_ack_rate_diff(ca) <= ack_rate_diff_ss) || - !tcp_is_cwnd_limited(sk)) { - ca->epoch_start = 0; + if (ca->state == LAUNCH) { + /* LAUNCH: Detect an exit point for tcp slow start + * in networks with large buffers of multiple BDP + * Like in cellular networks (5G, ...). + * Or exit LAUNCH if cwnd is too large for application layer + * data rate (tcp cwnd validation). + */ + if ((ca->curr_srrtt > sr_rtt_upper_bound && + get_ack_rate_diff(ca) <= ack_rate_diff_ss) || + !tcp_is_cwnd_limited(sk)) { + ca->epoch_start = 0; - /* Handle initial slow start. Here occur most bufferbloat */ - if (tp->snd_ssthresh == TCP_INFINITE_SSTHRESH) { - tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp) / 2; - /* since this is the initial slow start, - * the min cwnd won't be 1, so the window - * can't be set to 0 by accident. - * Halfing the cwnd will undo the previous step - * of slow start. Which is fine since the pipe - * is already full. - */ - tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp) / 2, - TCP_INIT_CWND)); - } else { - tcp_sk(sk)->snd_ssthresh = - tcp_snd_cwnd(tp) - (tcp_snd_cwnd(tp) / 3); - tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - - (tcp_snd_cwnd(tp) / 3)); - } - ca->roccet_last_event_time_us = now; - return; - } + /* Handle initial slow start. + * Most bufferbloat occurs here + */ + if (tp->snd_ssthresh == TCP_INFINITE_SSTHRESH) { + tcp_sk(sk)->snd_ssthresh = tcp_snd_cwnd(tp) + / 2; + /* since this is the initial slow start, + * the min cwnd won't be 1, so the window + * can't be set to 0 by accident. + * Halfing the cwnd will undo the previous step + * of slow start. Which is fine since the pipe + * is already full. + */ + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp) / 2, + TCP_INIT_CWND)); + } else { + tcp_sk(sk)->snd_ssthresh = + tcp_snd_cwnd(tp) - + (tcp_snd_cwnd(tp) / 3); + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - + (tcp_snd_cwnd(tp) / 3)); + } + ca->roccet_last_event_time_us = now; + return; + } - acked = tcp_slow_start(tp, acked); - if (!acked) - return; + acked = tcp_slow_start(tp, acked); + if (!acked) + return; - }else if(ca->state == ORBITER){ - /* ORBITER: Increase the cwnd by using the CUBIC - * cwnd growth function, if no roccet congestion - * event is detechted. - */ + } else if (ca->state == ORBITER) { + /* ORBITER: Increase the cwnd by using the CUBIC + * cwnd growth function, if no roccet congestion + * event is detechted. + */ - /* Calculate jitter */ - if ((s32)(ca->curr_rtt - ca->last_rtt) < 0) - jitter = ca->last_rtt - ca->curr_rtt; - else - jitter = ca->curr_rtt - ca->last_rtt; + /* Calculate jitter */ + if ((s32)(ca->curr_rtt - ca->last_rtt) < 0) + jitter = ca->last_rtt - ca->curr_rtt; + else + jitter = ca->curr_rtt - ca->last_rtt; - if (ca->next_srrtt_check == 0) - ca->next_srrtt_check = now + 5 * ca->curr_rtt; + if (ca->next_srrtt_check == 0) + ca->next_srrtt_check = now + 5 * ca->curr_rtt; - /* Calculate if more bytes was send than reveived - * in the time interval. - */ - if (tp->snd_nxt < ca->interval_snd_seq_start){ - /* We had a wrap around in seq no counter */ - send = (~0U - ca->interval_snd_seq_start + tp->snd_nxt); - }else{ - send = (tp->snd_nxt - ca->interval_snd_seq_start); - } - if (tp->snd_una < ca->interval_una_seq_start){ - /* We had a wrap around in seq no counter */ - received = (~0U - ca->interval_una_seq_start + tp->snd_una); - }else{ - received = (tp->snd_una - ca->interval_una_seq_start); - } + /* Calculate if more bytes was send than received + * in the time interval. + */ + if (before(tp->snd_nxt, ca->interval_snd_seq_start)) { + /* We had a wrap around in seq no counter */ + send = (~0U - ca->interval_snd_seq_start + tp->snd_nxt); + } else { + send = (tp->snd_nxt - ca->interval_snd_seq_start); + } + if (before(tp->snd_una, ca->interval_una_seq_start)) { + /* We had a wrap around in seq no counter */ + received = (~0U - ca->interval_una_seq_start + + tp->snd_una); + } else { + received = (tp->snd_una - ca->interval_una_seq_start); + } - /* Here we use a guard space of 1% of the current cwnd. - * We do this to avoid a false positive evaluation due - * to delays caused by jitter or scheduling. - */ - send_more_than_acked = send > received + ((tcp_snd_cwnd(tp) * tp->mss_cache) / 100 ); + /* Here we use a guard space of 1% of the current cwnd. + * We do this to avoid a false positive evaluation due + * to delays caused by jitter or scheduling. + */ + send_more_than_acked = + send > + received + ((tcp_snd_cwnd(tp) * tp->mss_cache) / 100); - /* Check if it's time to evaluate the srRTT */ - if ((s32)(ca->next_srrtt_check - now) < 0) { - evaluate_srrtt = true; + /* Check if it's time to evaluate the srRTT */ + if ((s32)(ca->next_srrtt_check - now) < 0) { + evaluate_srrtt = true; - /* reset struct and set next end of period */ - ca->next_srrtt_check = now + 5 * ca->curr_rtt; + /* reset struct and set next end of period */ + ca->next_srrtt_check = now + 5 * ca->curr_rtt; - /* Reset Rate calculation */ - ca->interval_snd_seq_start = tp->snd_nxt; - ca->interval_una_seq_start = tp->snd_una; - } + /* Reset Rate calculation */ + ca->interval_snd_seq_start = tp->snd_nxt; + ca->interval_una_seq_start = tp->snd_una; + } - /* Respects the jitter of the connection and add it on top of - * the upper bound for the srRTT. - */ - roccet_xj = div_u64((u64)jitter * 100, ca->curr_min_rtt) + - sr_rtt_upper_bound; - if (roccet_xj < sr_rtt_upper_bound) - roccet_xj = sr_rtt_upper_bound; + /* Respects the jitter of the connection and add it on top of + * the upper bound for the srRTT. + */ + roccet_xj = div_u64((u64)jitter * 100, ca->curr_min_rtt) + + sr_rtt_upper_bound; + if (roccet_xj < sr_rtt_upper_bound) + roccet_xj = sr_rtt_upper_bound; - /* The srRTT exceeds the upper bound if bufferbloat happens. - * Here, we want to reduce the cwnd and drain the buffer. - */ - if (ca->curr_srrtt > roccet_xj && evaluate_srrtt && send_more_than_acked) { - roccet_congestion_event(sk, now); - return; - } + /* The srRTT exceeds the upper bound if bufferbloat happens. + * Here, we want to reduce the cwnd and drain the buffer. + */ + if (ca->curr_srrtt > roccet_xj && evaluate_srrtt && + send_more_than_acked) { + roccet_congestion_event(sk, now); + return; + } - /* Terminates this function if cwnd is not fully utilized. - * In mobile networks like 5G, this termination causes the cwnd to be - * frozen at an excessively high value. This is because slow start or - * HyStart massively exceed the available bandwidth and leave the cwnd - * at an excessively high value. The cwnd cannot therefore be fully - * utilized because it is limited by the connection capacity. - */ - if (!tcp_is_cwnd_limited(sk) || send_more_than_acked) - return; + /* Terminates this function if cwnd is not fully utilized. + * In mobile networks like 5G, this termination causes the + * cwnd to be frozen at an excessively high value. This is + * because slow start or HyStart massively exceed the available + * bandwidth and leave the cwnd at an excessively high value. + * The cwnd cannot therefore be fully utilized because it is + * limited by the connection capacity. + */ + if (!tcp_is_cwnd_limited(sk) || send_more_than_acked) + return; - bictcp_update(ca, tcp_snd_cwnd(tp), acked); - tcp_cong_avoid_ai(tp, max(1, ca->cnt), acked); - } + bictcp_update(ca, tcp_snd_cwnd(tp), acked); + tcp_cong_avoid_ai(tp, max(1, ca->cnt), acked); + } } static u32 roccettcp_recalc_ssthresh(struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); struct roccettcp *ca = inet_csk_ca(sk); - u32 cwnd; + u32 cwnd = tcp_snd_cwnd(tp); - /* If a loss/ECN occurs in the refill pahse of min RTT probing - * we reduce the cwnd and abort the refill. - */ - if (ca->state == RTT_PROBE_REFILL) - ca->state = ORBITER; + /* If a loss/ECN occurs in the refill phase of min RTT probing + * we reduce the cwnd and abort the refill. + */ + if (ca->state == RTT_PROBE_REFILL) + ca->state = ORBITER; - /* If ROCCET is in min RTT probing and a loss/ECN occurs, - * we use the cwnd before the probing interval to - * calculate the cwnd reduction and continue probing. - * After min RTT probing the cwnd is set to the reduced - * value. During min RTT probing it is very likely that - * congestion was caused by the cwnd value before min - * RTT probing. - */ - if (ca->state == RTT_PROBE){ - /* Handle ECN as cubic congestion event in min - * RTT probe. - */ - ca->ece_received = false; + /* If ROCCET is in min RTT probing and a loss/ECN occurs, + * we use the cwnd before the probing interval to + * calculate the cwnd reduction and continue probing. + * After min RTT probing the cwnd is set to the reduced + * value. During min RTT probing it is very likely that + * congestion was caused by the cwnd value before min + * RTT probing. + */ + if (ca->state == RTT_PROBE) { + /* Handle ECN as cubic congestion event in min + * RTT probe. + */ + ca->ece_received = false; - ca->epoch_start = 0; /* end of epoch */ + ca->epoch_start = 0; /* end of epoch */ - /* Wmax and fast convergence */ - if (cwnd < ca->last_max_cwnd && fast_convergence) - ca->last_max_cwnd = - (cwnd * (BICTCP_BETA_SCALE + beta)) / - (2 * BICTCP_BETA_SCALE); - else - ca->last_max_cwnd = cwnd; + /* Wmax and fast convergence */ + if (cwnd < ca->last_max_cwnd && fast_convergence) + ca->last_max_cwnd = + (cwnd * (BICTCP_BETA_SCALE + beta)) / + (2 * BICTCP_BETA_SCALE); + else + ca->last_max_cwnd = cwnd; - cwnd = ca->cwnd_before_min_rtt_probe; - ca->cwnd_before_min_rtt_probe = max((cwnd * beta) / BICTCP_BETA_SCALE, 2U); - return cwnd; - } + cwnd = ca->cwnd_before_min_rtt_probe; + ca->cwnd_before_min_rtt_probe = + max((cwnd * beta) / BICTCP_BETA_SCALE, 2U); + + return cwnd; + } /* Handle ECN as ROCCET congestion event. */ if (ca->ece_received) { @@ -696,19 +717,17 @@ static u32 roccettcp_recalc_ssthresh(struct sock *sk) static void roccettcp_state(struct sock *sk, u8 new_state) { struct roccettcp *ca = inet_csk_ca(sk); - struct tcp_sock *tp = tcp_sk(sk); + struct tcp_sock *tp = tcp_sk(sk); - if (new_state == TCP_CA_Loss){ + if (new_state == TCP_CA_Loss) { roccettcp_reset(ca); - } - else if(new_state == TCP_CA_Recovery){ - tcp_sk(sk)->snd_ssthresh = roccettcp_recalc_ssthresh(sk); - tcp_snd_cwnd_set(tp, tcp_sk(sk)->snd_ssthresh); - } + } else if (new_state == TCP_CA_Recovery) { + tcp_sk(sk)->snd_ssthresh = roccettcp_recalc_ssthresh(sk); + tcp_snd_cwnd_set(tp, tcp_sk(sk)->snd_ssthresh); + } } -static void roccettcp_acked(struct sock *sk, - const struct ack_sample *sample) +static void roccettcp_acked(struct sock *sk, const struct ack_sample *sample) { struct roccettcp *ca = inet_csk_ca(sk); @@ -747,71 +766,76 @@ static void roccet_in_ack_event(struct sock *sk, u32 flags) ca->ece_received = true; } -__bpf_kfunc static void roccet_control(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs) { - struct tcp_sock *tp = tcp_sk(sk); +static void roccet_control(struct sock *sk, u32 ack, int flag, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); struct roccettcp *ca = inet_csk_ca(sk); u32 now = jiffies_to_usecs(tcp_jiffies32); - u64 rate; + u64 rate; - /* Update roccet parameters */ + /* Update roccet parameters */ update_ack_rate(sk, rs->acked_sacked, now); update_min_rtt(sk); update_srrtt(sk); - /* Set values for send and receive rate */ - if (ca->interval_snd_seq_start == 0){ - ca->interval_snd_seq_start = tp->snd_nxt; - ca->interval_una_seq_start = tp->snd_una; - } + /* Set values for send and receive rate */ + if (ca->interval_snd_seq_start == 0) { + ca->interval_snd_seq_start = tp->snd_nxt; + ca->interval_una_seq_start = tp->snd_una; + } - /* Update roccet state */ - if(tcp_in_slow_start(tp)){ - ca->state = LAUNCH; - }else if((s32) now - ca->roccet_last_event_time_us <= 100 * USEC_PER_MSEC){ - ca->state = DRAIN; - } - else if(after(now, ca->next_min_rtt_probe) || ca->state == RTT_PROBE || ca->state == RTT_PROBE_REFILL){ - if(ca->state != RTT_PROBE_REFILL) - ca->state = RTT_PROBE; - roccet_min_rtt_probe(sk, now); - } - else{ - ca->state = ORBITER; - } + /* Update roccet state */ + if (tcp_in_slow_start(tp)) { + ca->state = LAUNCH; + } else if ((s32)now - ca->roccet_last_event_time_us <= + 100 * USEC_PER_MSEC) { + ca->state = DRAIN; + } else if (after(now, ca->next_min_rtt_probe) || + ca->state == RTT_PROBE || ca->state == RTT_PROBE_REFILL) { + if (ca->state != RTT_PROBE_REFILL) + ca->state = RTT_PROBE; + roccet_min_rtt_probe(sk, now); + } else { + ca->state = ORBITER; + } - /* If nothing was fully acked do not increase the cwnd */ - if (!rs->acked_sacked) - return; + /* If nothing was fully acked do not increase the cwnd */ + if (!rs->acked_sacked) + return; - /* Increase the cwnd. - * Loss recovery is handled in roccettcp_state() - */ - roccettcp_cong_avoid(sk, ack, rs->acked_sacked); + /* Increase the cwnd. + * Loss recovery is handled in roccettcp_state() + */ + roccettcp_cong_avoid(sk, ack, rs->acked_sacked); - /* Adjust pacing rate. The code here is similar to the - * pacing rate adjustemnts in tcp_input.c tcp_cong_control(). - * In LAUNCH (slow start) we want a pacing of 200% and - * in ORBITER (congestion avoidance) we adjust the pacing - * to 100% and do not use the sysctl_tcp_pacing_ca_ratio. - */ + /* Adjust pacing rate. The code here is similar to the + * pacing rate adjustments in tcp_input.c tcp_cong_control(). + * In LAUNCH (slow start) we want a pacing of 200% and + * in ORBITER (congestion avoidance) we adjust the pacing + * to 100% and do not use the sysctl_tcp_pacing_ca_ratio. + */ - /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */ + /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */ rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3); /* current rate is (cwnd * mss) / srtt * In Slow Start [1], set sk_pacing_rate to 200 % the current rate. * In Congestion Avoidance phase, set it to 120 % the current rate. * - * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh) + * [1]: Normal Slow Start cond is (tp->snd_cwnd < tp->snd_ssthresh) * If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching * end of slow start and should slow down. */ if (tcp_snd_cwnd(tp) < tp->snd_ssthresh / 2) - rate *= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio); + rate *= READ_ONCE + (sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio); else - /* Pacing rate of 100% (instead of ipv4.sysctl_tcp_pacing_ca_ratio) */ - rate *= 100; + /* Pacing rate of 100% + * (instead of ipv4.sysctl_tcp_pacing_ca_ratio) + */ + rate *= 100; rate *= max(tcp_snd_cwnd(tp), tp->packets_out); @@ -819,7 +843,7 @@ __bpf_kfunc static void roccet_control(struct sock *sk, u32 ack, int flag, const do_div(rate, tp->srtt_us); /* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate - * without any lock. We want to make sure compiler wont store + * without any lock. We want to make sure compiler won't store * intermediate values in this location. */ WRITE_ONCE(sk->sk_pacing_rate, @@ -827,16 +851,16 @@ __bpf_kfunc static void roccet_control(struct sock *sk, u32 ack, int flag, const } static struct tcp_congestion_ops roccet_tcp __read_mostly = { - .init = roccettcp_init, - .ssthresh = roccettcp_recalc_ssthresh, - .set_state = roccettcp_state, - .undo_cwnd = tcp_reno_undo_cwnd, - .cwnd_event_tx_start = roccettcp_cwnd_event_tx_start, - .pkts_acked = roccettcp_acked, - .in_ack_event = roccet_in_ack_event, - .cong_control = roccet_control, - .owner = THIS_MODULE, - .name = "roccet", + .init = roccettcp_init, + .ssthresh = roccettcp_recalc_ssthresh, + .set_state = roccettcp_state, + .undo_cwnd = tcp_reno_undo_cwnd, + .cwnd_event_tx_start = roccettcp_cwnd_event_tx_start, + .pkts_acked = roccettcp_acked, + .in_ack_event = roccet_in_ack_event, + .cong_control = roccet_control, + .owner = THIS_MODULE, + .name = "roccet", }; static int __init roccettcp_register(void) diff --git a/tcp_roccet.h b/tcp_roccet.h index d5784cd..b50e8ea 100644 --- a/tcp_roccet.h +++ b/tcp_roccet.h @@ -7,59 +7,55 @@ #include -/* State in which roccet currently opperates */ +/* State in which roccet currently operates */ enum roccet_state { - LAUNCH, - ORBITER, - RTT_PROBE, - RTT_PROBE_REFILL, - DRAIN -}; - -/* TCP ROCCET helper type, storing data related to ack counting */ -struct ack_rate { - u32 last_rate; /* Last ACK-rate */ - u32 last_rate_time; /* Timestamp of the last ACK-rate */ - u32 curr_rate; /* Current ACK-rate */ - u32 cnt; /* Used for counting acks */ + LAUNCH, + ORBITER, + RTT_PROBE, + RTT_PROBE_REFILL, + DRAIN }; /* TCP ROCCET struct based on the original BICTCP struct with * additions specific to the ROCCET-Algorithm. */ struct roccettcp { - u32 cnt; /* increase cwnd by 1 after ACKs */ + u32 cnt; /* increase cwnd by 1 after ACKs */ u32 last_max_cwnd; /* last maximum snd_cwnd */ - u32 last_cwnd; /* the last snd_cwnd */ - u32 last_time; /* time when updated last_cwnd */ + u32 last_cwnd; /* the last snd_cwnd */ + u32 last_time; /* time when updated last_cwnd */ u32 bic_origin_point; /* origin point of bic function */ - u32 bic_K; /* time to origin point from the - * beginning of the current epoch - */ - u32 delay_min; /* min delay (usec) */ + u32 bic_K; /* time to origin point from the + * beginning of the current epoch + */ + u32 delay_min; /* min delay (usec) */ u32 epoch_start; /* beginning of an epoch */ - u32 ack_cnt; /* number of acks */ - u32 tcp_cwnd; /* estimated tcp cwnd */ - u32 curr_rtt; /* last sample rtt of current round */ + u32 ack_cnt; /* number of acks */ + u32 tcp_cwnd; /* estimated tcp cwnd */ + u32 curr_rtt; /* last sample rtt of current round */ - u32 roccet_last_event_time_us; /* The last time ROCCET was - * triggered - */ - bool ece_received; /* Set to true if an ECE bit was received */ + u32 roccet_last_event_time_us; /* The last time ROCCET was triggered */ u32 curr_min_rtt; /* The current observed minRTT */ - u32 next_min_rtt_probe; /* Next time to probe the minRTT */ + u32 next_min_rtt_probe; /* Next time to probe the minRTT */ u32 probe_min_rtt_until; /* End of minRTT probing period */ - u32 refill_until; /* End of pipe refill after minRTT probe */ + u32 refill_until; /* End of pipe refill after minRTT probe */ u32 cwnd_before_min_rtt_probe; /* cwnd before min RTT probeing */ - u32 curr_srrtt; /* srRTT calculated based on the latest ACK */ + u32 curr_srrtt; /* srRTT calculated based on the latest ACK */ u32 next_srrtt_check; /* Next check for srRTT */ - struct ack_rate ack_rate; /* The last and the current ACK rate */ - u32 last_rtt; /* sample rtt of previous round. - * Used for jitter calculation - */ - enum roccet_state state; /* State in which roccet currently opperates */ - u32 interval_snd_seq_start; - u32 interval_una_seq_start; + u32 last_rtt; /* sample rtt of previous round. + * Used for jitter calculation + */ + + u32 interval_snd_seq_start; + u32 interval_una_seq_start; + + u32 ack_rate_last_rate_time; /* Timestamp of the last ACK-rate */ + u16 ack_rate_last_rate; /* Last ACK-rate */ + u16 ack_rate_curr_rate; /* Current ACK-rate */ + u16 ack_rate_cnt; /* Used for counting acks */ + + bool ece_received; /* Set to true if an ECE bit was received */ + enum roccet_state state; /* State in which roccet currently operates */ }; #endif /* __TCP_ROCCET_H */