linux tcp GSO和TSO实现 linux tcp GSO和TSO实现

 1 #define NETIF_F_SOFT_FEATURES           (NETIF_F_GSO | NETIF_F_GRO)
 2 
 3 int register_netdevice(struct net_device *dev)
 4 
 5 {
 6 
 7               ...
 8 
 9               /* Transfer changeable features to wanted_features and enable
10 
11                * software offloads (GSO and GRO).
12 
13                */
14 
15               dev->hw_features |= NETIF_F_SOFT_FEATURES;
16 
17               dev->features |= NETIF_F_SOFT_FEATURES;         //默认开启GRO/GSO
18 
19               dev->wanted_features = dev->features & dev->hw_features;
20 
21               ...
22 
23 }
24 
25 static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
26 
27 {
28 
29               ...
30 
31               netdev->features = NETIF_F_SG |
32 
33                                 NETIF_F_TSO |
34 
35                                 NETIF_F_TSO6 |
36 
37                                 NETIF_F_RXHASH |
38 
39                                 NETIF_F_RXCSUM |
40 
41                                 NETIF_F_HW_CSUM;
42 
43               register_netdev(netdev);
44 
45               ...
46 
47 }

 1 #define NETIF_F_GSO_SOFTWARE            (NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
 2 
 3 #define NETIF_F_TSO                    (SKB_GSO_TCPV4 << NETIF_F_GSO_SHIFT)
 4 
 5 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
 6 
 7 {
 8 
 9        __sk_dst_set(sk, dst);
10 
11        sk->sk_route_caps = dst->dev->features;
12 
13        if (sk->sk_route_caps & NETIF_F_GSO)   /*GSO默认都会开启*/
14 
15               sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;  /*打开TSO*/
16 
17        if (sk_can_gso(sk)) {  /*对于tcp这里会成立*/
18 
19               if (dst->header_len) {
20 
21                     sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
22 
23               } else {
24 
25                    sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
26 
27                    sk->sk_gso_max_size = dst->dev->gso_max_size;  /*GSO_MAX_SIZE=65536*/
28 
29              }
30  
31      }
32 
33 }

1 static inline int sk_can_gso(const struct sock *sk)
2 
3 {
4 
5     /*对于tcp，在tcp_v4_connect中被设置：sk->sk_gso_type = SKB_GSO_TCPV4*/
6 
7     return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
8 
9 }

1 static inline int net_gso_ok(int features, int gso_type)
2 
3 {
4 
5       int feature = gso_type << NETIF_F_GSO_SHIFT;
6 
7       return (features & feature) == feature;
8 
9 }

 1 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 2 
 3 {
 4 
 5        int mss_now;
 6 
 7  
 8 
 9        mss_now = tcp_current_mss(sk);/*通过ip option，SACKs及pmtu确定当前的mss*/
10 
11        *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
12 
13  
14 
15        return mss_now;
16 
17 }

 1 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, int large_allowed)
 2 {
 3     struct tcp_sock *tp = tcp_sk(sk);
 4     u32 xmit_size_goal, old_size_goal;
 5 
 6     xmit_size_goal = mss_now;
 7     /*这里large_allowed表示是否是紧急数据*/
 8     if (large_allowed && sk_can_gso(sk)) {  /*如果不是紧急数据且支持GSO*/
 9         xmit_size_goal = ((sk->sk_gso_max_size - 1) -
10                   inet_csk(sk)->icsk_af_ops->net_header_len -
11                   inet_csk(sk)->icsk_ext_hdr_len -
12                   tp->tcp_header_len);/*xmit_size_goal为gso最大分段大小减去tcp和ip头部长度*/
13 
14         xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal);/*最多达到收到的最大rwnd窗口通告的一半*/
15 
16         /* We try hard to avoid divides here */
17         old_size_goal = tp->xmit_size_goal_segs * mss_now;
18 
19         if (likely(old_size_goal <= xmit_size_goal &&
20                old_size_goal + mss_now > xmit_size_goal)) {
21             xmit_size_goal = old_size_goal; /*使用老的xmit_size*/
22         } else {
23             tp->xmit_size_goal_segs = xmit_size_goal / mss_now;
24             xmit_size_goal = tp->xmit_size_goal_segs * mss_now; /*使用新的xmit_size*/
25         }
26     }
27 
28     return max(xmit_size_goal, mss_now);
29 }

  1 int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t size)
  2 {
  3     struct sock *sk = sock->sk;
  4     struct iovec *iov;
  5     struct tcp_sock *tp = tcp_sk(sk);
  6     struct sk_buff *skb;
  7     int iovlen, flags;
  8     int mss_now, size_goal;
  9     int err, copied;
 10     long timeo;
 11 
 12     lock_sock(sk);
 13     TCP_CHECK_TIMER(sk);
 14 
 15     flags = msg->msg_flags;
 16     timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 17 
 18     /* Wait for a connection to finish. */
 19     if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 20         if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 21             goto out_err;
 22 
 23     /* This should be in poll */
 24     clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 25     /* size_goal表示GSO支持的大小，为mss的整数倍，不支持GSO时则和mss相等 */
 26     mss_now = tcp_send_mss(sk, &size_goal, flags);/*返回值mss_now为真实mss*/
 27 
 28     /* Ok commence sending. */
 29     iovlen = msg->msg_iovlen;
 30     iov = msg->msg_iov;
 31     copied = 0;
 32 
 33     err = -EPIPE;
 34     if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 35         goto out_err;
 36 
 37     while (--iovlen >= 0) {
 38         size_t seglen = iov->iov_len;
 39         unsigned char __user *from = iov->iov_base;
 40 
 41         iov++;
 42 
 43         while (seglen > 0) {
 44             int copy = 0;
 45             int max = size_goal; /*每个skb中填充的数据长度初始化为size_goal*/
 46             /* 从sk->sk_write_queue中取出队尾的skb，因为这个skb可能还没有被填满 */
 47             skb = tcp_write_queue_tail(sk);
 48             if (tcp_send_head(sk)) { /*如果之前还有未发送的数据*/
 49                 if (skb->ip_summed == CHECKSUM_NONE)  /*比如路由变更，之前的不支持TSO,现在的支持了*/
 50                     max = mss_now; /*上一个不支持GSO的skb，继续不支持*/
 51                 copy = max - skb->len; /*copy为每次想skb中拷贝的数据长度*/
 52             }
 53            /*copy<=0表示不能合并到之前skb做GSO*/
 54             if (copy <= 0) {
 55 new_segment:
 56                 /* Allocate new segment. If the interface is SG,
 57                  * allocate skb fitting to single page.
 58                  */
 59                  /* 内存不足，需要等待 */
 60                 if (!sk_stream_memory_free(sk))
 61                     goto wait_for_sndbuf;
 62                 /* 分配新的skb */
 63                 skb = sk_stream_alloc_skb(sk, select_size(sk),
 64                         sk->sk_allocation);
 65                 if (!skb)
 66                     goto wait_for_memory;
 67 
 68                 /*
 69                  * Check whether we can use HW checksum.
 70                  */
 71                 /*如果硬件支持checksum，则将skb->ip_summed设置为CHECKSUM_PARTIAL，表示由硬件计算校验和*/
 72                 if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
 73                     skb->ip_summed = CHECKSUM_PARTIAL;
 74                 /*将skb加入sk->sk_write_queue队尾, 同时去掉skb的TCP_NAGLE_PUSH标记*/
 75                 skb_entail(sk, skb);
 76                 copy = size_goal;  /*这里将每次copy的大小设置为size_goal，即GSO支持的大小*/
 77                 max = size_goal;
 78             }
 79 
 80             /* Try to append data to the end of skb. */
 81             if (copy > seglen)
 82                 copy = seglen;
 83 
 84             /* Where to copy to? */
 85             if (skb_tailroom(skb) > 0) { /*如果skb的线性区还有空间，则先填充skb的线性区*/
 86                 /* We have some space in skb head. Superb! */
 87                 if (copy > skb_tailroom(skb))
 88                     copy = skb_tailroom(skb);
 89                 if ((err = skb_add_data(skb, from, copy)) != 0) /*copy用户态数据到skb线性区*/
 90                     goto do_fault;
 91             } else {  /*否则尝试向SG的frags中拷贝*/
 92                 int merge = 0;
 93                 int i = skb_shinfo(skb)->nr_frags;
 94                 struct page *page = TCP_PAGE(sk);
 95                 int off = TCP_OFF(sk);
 96 
 97                 if (skb_can_coalesce(skb, i, page, off) &&
 98                     off != PAGE_SIZE) {/*pfrag->page和frags[i-1]是否使用相同页，并且page_offset相同*/
 99                     /* We can extend the last page
100                      * fragment. */
101                     merge = 1; /*说明和之前frags中是同一个page，需要merge*/
102                 } else if (i == MAX_SKB_FRAGS ||
103                        (!i && !(sk->sk_route_caps & NETIF_F_SG))) {
104                     /* Need to add new fragment and cannot
105                      * do this because interface is non-SG,
106                      * or because all the page slots are
107                      * busy. */
108                      /*如果设备不支持SG，或者非线性区frags已经达到最大，则创建新的skb分段*/
109                     tcp_mark_push(tp, skb); /*标记push flag*/
110                     goto new_segment;
111                 } else if (page) {
112                     if (off == PAGE_SIZE) {
113                         put_page(page); /*增加page引用计数*/
114                         TCP_PAGE(sk) = page = NULL;
115                         off = 0;
116                     }
117                 } else
118                     off = 0;
119 
120                 if (copy > PAGE_SIZE - off)
121                     copy = PAGE_SIZE - off;
122 
123                 if (!sk_wmem_schedule(sk, copy))
124                     goto wait_for_memory;
125 
126                 if (!page) {
127                     /* Allocate new cache page. */
128                     if (!(page = sk_stream_alloc_page(sk)))
129                         goto wait_for_memory;
130                 }
131 
132                 /* Time to copy data. We are close to
133                  * the end! */
134                 err = skb_copy_to_page(sk, from, skb, page, off, copy); /*拷贝数据到page中*/
135                 if (err) {
136                     /* If this page was new, give it to the
137                      * socket so it does not get leaked.
138                      */
139                     if (!TCP_PAGE(sk)) {
140                         TCP_PAGE(sk) = page;
141                         TCP_OFF(sk) = 0;
142                     }
143                     goto do_error;
144                 }
145 
146                 /* Update the skb. */
147                 if (merge) { /*pfrag和frags[i - 1]是相同的*/
148                     skb_shinfo(skb)->frags[i - 1].size += copy;
149                 } else {
150                     skb_fill_page_desc(skb, i, page, off, copy);
151                     if (TCP_PAGE(sk)) {
152                         get_page(page);
153                     } else if (off + copy < PAGE_SIZE) {
154                         get_page(page);
155                         TCP_PAGE(sk) = page;
156                     }
157                 }
158 
159                 TCP_OFF(sk) = off + copy;
160             }
161 
162             if (!copied)
163                 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
164 
165             tp->write_seq += copy;
166             TCP_SKB_CB(skb)->end_seq += copy;
167             skb_shinfo(skb)->gso_segs = 0; /*清零tso分段数，让tcp_write_xmit去计算*/
168 
169             from += copy;
170             copied += copy;
171             if ((seglen -= copy) == 0 && iovlen == 0)
172                 goto out;
173             /* 还有数据没copy，并且没有达到最大可拷贝的大小(注意这里max之前被赋值为size_goal，即GSO支持的大小)， 尝试往该skb继续添加数据*/
174             if (skb->len < max || (flags & MSG_OOB))
175                 continue;
176             /*下面的逻辑就是：还有数据没copy，但是当前skb已经满了，所以可以发送了(但不是一定要发送)*/
177             if (forced_push(tp)) { /*超过最大窗口的一半没有设置push了*/
178                 tcp_mark_push(tp, skb); /*设置push标记，更新pushed_seq*/
179                 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); /*调用tcp_write_xmit马上发送*/
180             } else if (skb == tcp_send_head(sk)) /*第一个包，直接发送*/
181                 tcp_push_one(sk, mss_now);
182             continue; /*说明发送队列前面还有skb等待发送，且距离之前push的包还不是非常久*/
183 
184 wait_for_sndbuf:
185             set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
186 wait_for_memory:
187             if (copied)/*先把copied的发出去再等内存*/
188                 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
189             /*阻塞等待内存*/
190             if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
191                 goto do_error;
192 
193             mss_now = tcp_send_mss(sk, &size_goal, flags);
194         }
195     }
196 
197 out:
198     if (copied) /*所有数据都放到发送队列中了，调用tcp_push发送*/
199         tcp_push(sk, flags, mss_now, tp->nonagle);
200     TCP_CHECK_TIMER(sk);
201     release_sock(sk);
202     return copied;
203 
204 do_fault:
205     if (!skb->len) {
206         tcp_unlink_write_queue(skb, sk);
207         /* It is the one place in all of TCP, except connection
208          * reset, where we can be unlinking the send_head.
209          */
210         tcp_check_send_head(sk, skb);
211         sk_wmem_free_skb(sk, skb);
212     }
213 
214 do_error:
215     if (copied)
216         goto out;
217 out_err:
218     err = sk_stream_error(sk, flags, err);
219     TCP_CHECK_TIMER(sk);
220     release_sock(sk);
221     return err;
222 }

 1 static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
 2               int push_one, gfp_t gfp)
 3 {
 4     struct tcp_sock *tp = tcp_sk(sk);
 5     struct sk_buff *skb;
 6     unsigned int tso_segs, sent_pkts;
 7     int cwnd_quota;
 8     int result;
 9 
10     sent_pkts = 0;
11 
12     if (!push_one) {
13         /* Do MTU probing. */
14         result = tcp_mtu_probe(sk);
15         if (!result) {
16             return 0;
17         } else if (result > 0) {
18             sent_pkts = 1;
19         }
20     }
21     /*遍历发送队列*/
22     while ((skb = tcp_send_head(sk))) {
23         unsigned int limit;
24 
25         tso_segs = tcp_init_tso_segs(sk, skb, mss_now); /*skb->len/mss，重新设置tcp_gso_segs，因为在tcp_sendmsg中被清零了*/
26         BUG_ON(!tso_segs);
27 
28         cwnd_quota = tcp_cwnd_test(tp, skb);
29         if (!cwnd_quota)
30             break;
31 
32         if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
33             break;
34 
35         if (tso_segs == 1) {  /*tso_segs=1表示无需tso分段*/
36             /* 根据nagle算法，计算是否需要推迟发送数据 */
37             if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
38                              (tcp_skb_is_last(sk, skb) ? /*last skb就直接发送*/
39                               nonagle : TCP_NAGLE_PUSH))))
40                 break;
41         } else {/*有多个tso分段*/
42             if (!push_one /*push所有skb*/
43                 && tcp_tso_should_defer(sk, skb))/*/如果发送窗口剩余不多，并且预计下一个ack将很快到来(意味着可用窗口会增加)，则推迟发送*/
44                 break;
45         }
46         /*下面的逻辑是：不用推迟发送，马上发送的情况*/
47         limit = mss_now;
48 /*由于tso_segs被设置为skb->len/mss_now，所以开启gso时一定大于1*/
49         if (tso_segs > 1 && !tcp_urg_mode(tp)) /*tso分段大于1且非urg模式*/
50             limit = tcp_mss_split_point(sk, skb, mss_now, cwnd_quota);/*返回当前skb中可以发送的数据大小，通过mss和cwnd*/
51         /* 当skb的长度大于限制时，需要调用tso_fragment分片,如果分段失败则暂不发送 */
52         if (skb->len > limit &&
53             unlikely(tso_fragment(sk, skb, limit, mss_now))) /*/按limit切割成多个skb*/
54             break;
55 
56         TCP_SKB_CB(skb)->when = tcp_time_stamp;
57         /*发送，如果包被qdisc丢了，则退出循环，不继续发送了*/
58         if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
59             break;
60 
61         /* Advance the send_head.  This one is sent out.
62          * This call will increment packets_out.
63          */
64          /*更新sk_send_head和packets_out*/
65         tcp_event_new_data_sent(sk, skb);
66 
67         tcp_minshall_update(tp, mss_now, skb);
68         sent_pkts++;
69 
70         if (push_one)
71             break;
72     }
73 
74     if (likely(sent_pkts)) {
75         tcp_cwnd_validate(sk);
76         return 0;
77     }
78     return !tp->packets_out && tcp_send_head(sk);
79 }

 1 static int tcp_init_tso_segs(struct sock *sk, struct sk_buff *skb,
 2                  unsigned int mss_now)
 3 {
 4     int tso_segs = tcp_skb_pcount(skb); /*skb_shinfo(skb)->gso_seg之前被初始化为0*/
 5 
 6     if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
 7         tcp_set_skb_tso_segs(sk, skb, mss_now);
 8         tso_segs = tcp_skb_pcount(skb);
 9     }
10     return tso_segs;
11 }
12 
13 static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb,
14                  unsigned int mss_now)
15 {
16     /* Make sure we own this skb before messing gso_size/gso_segs */
17     WARN_ON_ONCE(skb_cloned(skb));
18 
19     if (skb->len <= mss_now || !sk_can_gso(sk) ||
20         skb->ip_summed == CHECKSUM_NONE) {/*不支持gso的情况*/
21         /* Avoid the costly divide in the normal
22          * non-TSO case.
23          */
24         skb_shinfo(skb)->gso_segs = 1;
25         skb_shinfo(skb)->gso_size = 0;
26         skb_shinfo(skb)->gso_type = 0;
27     } else {
28         skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss_now); /*被设置为skb->len/mss_now*/
29         skb_shinfo(skb)->gso_size = mss_now;   /*注意mss_now为真实的mss，这里保存以供gso分段使用*/
30         skb_shinfo(skb)->gso_type = sk->sk_gso_type;
31     }
32 }
33