#define TCP_NAGLE_OFF        1    /* Nagle's algo is disabled */
#define TCP_NAGLE_CORK        2    /* Socket is corked        */
#define TCP_NAGLE_PUSH        4    /* Cork is overridden for already queued data */
case TCP_CORK:
        /* When set indicates to always queue non-full frames.
         * Later the user clears this option and we transmit
         * any pending partial frames in the queue.  This is
         * meant to be used alongside sendfile() to get properly
         * filled frames when the user (for example) must write
         * out headers with a write() call first and then use
         * sendfile to send out the data parts.
         *
         * TCP_CORK can be set together with TCP_NODELAY and it is
         * stronger than TCP_NODELAY.
         */
        if (val) {
            tp->nonagle |= TCP_NAGLE_CORK;
        } else {
            tp->nonagle &= ~TCP_NAGLE_CORK;
            if (tp->nonagle&TCP_NAGLE_OFF)
                tp->nonagle |= TCP_NAGLE_PUSH;
            tcp_push_pending_frames(sk);
        }
        break;
case TCP_NODELAY:
        if (val) {
            /* TCP_NODELAY is weaker than TCP_CORK, so that
             * this option on corked socket is remembered, but
             * it is not activated until cork is cleared.
             *
             * However, when TCP_NODELAY is set we make
             * an explicit push, which overrides even TCP_CORK
             * for currently queued segments.
             */
            tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
            tcp_push_pending_frames(sk);
        } else {
            tp->nonagle &= ~TCP_NAGLE_OFF;
        }

int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
{
    //size_goal表示GSO支持的大小，为mss_now的整数倍，不支持GSO时则相等
    mss_now = tcp_send_mss(sk, &size_goal, flags);
    // 把msg的用户态数据，按照GSO支持的最大大小，尽量copy到一个skb中
    //skb_entail(sk,skb)到发送队列
    //还有数据没copy，但是当前skb已经满了，可以发送了
    if (forced_push(tp)) {    //超过最大窗口的一半没有设置push了
        tcp_mark_push(tp, skb);    //设置push标记，更新pushed_seq
        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);    //调用tcp_write_xmit马上发送
    } else if (skb == tcp_send_head(sk))    //第一个包，直接发送
        tcp_push_one(sk, mss_now);
    else{
        //说明发送队列前面还有skb等待发送，且距离之前push的包还不是非常久, 则只是继续放到队列中，继续开始创建下一个skb copy
        continue
    }
out:    
    //最后的包调用tcp_push发送
    tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
    ...
}
static void skb_entail(struct sock *sk, struct sk_buff *skb)
{
    ...
    tcp_add_write_queue_tail(sk, skb);
    
    if (tp->nonagle & TCP_NAGLE_PUSH)
        tp->nonagle &= ~TCP_NAGLE_PUSH;    //创建新的skb放入发送队列，立刻清楚push选项
}

static void tcp_push(struct sock *sk, int flags, int mss_now,
             int nonagle, int size_goal)
{
    struct tcp_sock *tp = tcp_sk(sk);
    struct sk_buff *skb;
    if (!tcp_send_head(sk))
        return;
    skb = tcp_write_queue_tail(sk);
    if (!(flags & MSG_MORE) || forced_push(tp))
        tcp_mark_push(tp, skb);
    tcp_mark_urg(tp, flags);
    if (tcp_should_autocork(sk, skb, size_goal)) {
        //利用tsq机制延后发送
        /* avoid atomic op if TSQ_THROTTLED bit is already set */
        if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
            NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
            set_bit(TSQ_THROTTLED, &tp->tsq_flags);
        }
        /* It is possible TX completion already happened
         * before we set TSQ_THROTTLED.
         */
        if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
            return;
    }
    if (flags & MSG_MORE)    //应用程序标记了很快有新的数据到来，则标记cork，不发送小包
        nonagle = TCP_NAGLE_CORK;
    __tcp_push_pending_frames(sk, mss_now, nonagle);    //最终调用tcp_write_xmit
}

/* If a not yet filled skb is pushed, do not send it if
 * we have data packets in Qdisc or NIC queues :
 * Because TX completion will happen shortly, it gives a chance
 * to coalesce future sendmsg() payload into this skb, without
 * need for a timer, and with no latency trade off.
 * As packets containing data payload have a bigger truesize
 * than pure acks (dataless) packets, the last checks prevent
 * autocorking if we only have an ACK in Qdisc/NIC queues,
 * or if TX completion was delayed after we processed ACK packet.
 */
static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
                int size_goal)
{
    return skb->len < size_goal &&    //不到最大GSO size
           sysctl_tcp_autocorking &&    //默认开启
           skb != tcp_write_queue_head(sk) &&    //发送队列前面还有其他skb
           atomic_read(&sk->sk_wmem_alloc) > skb->truesize;    //qdisc中有数据， 说明网卡发送后完成中断释放内存，会很快有新的数据到来
}

static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
               int push_one, gfp_t gfp)
{
    
    max_segs = tcp_tso_segs(sk, mss_now);    //当前tso支持的最大segs数量
    while ((skb = tcp_send_head(sk))) {    //遍历发送队列
        tso_segs = tcp_init_tso_segs(skb, mss_now);    //skb->len/mss，重新设置tcp_gso_segs，因为在tcp_sendmsg中被清零了    
        ...
        if (tso_segs == 1) {//tso_segs=1表示无需tso分段
            /* 根据nagle算法，计算是否需要推迟发送数据 */
            if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
                             (tcp_skb_is_last(sk, skb) ?
                              nonagle : TCP_NAGLE_PUSH))))    //last skb就直接发送
                break;    //推迟发送
        } else {    //tso分段
            if (!push_one &&    //不只一个skb
                tcp_tso_should_defer(sk, skb, &is_cwnd_limited, //如果发送窗口剩余不多，并且预计下一个ack将很快到来(意味着可用窗口会增加)，则推迟发送
                         max_segs))
                break;    //可以推迟
        }
        //不用推迟发送，马上发送
        limit = mss_now;
        ...
        if (tcp_small_queue_check(sk, skb, 0))    //tsq检查，qdisc是否达到限制
            break;
        if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))    //发送，如果包被qdisc丢了，则退出循环，不继续发送了
            break;
        tcp_event_new_data_sent(sk, skb);//更新sk_send_head和packets_out
        /* 更新struct tcp_sock中的snd_sml字段。记录非全尺寸发送的最后一个字节序号，主要用来做nagle测试
         */
        tcp_minshall_update(tp, mss_now, skb);
        sent_pkts += tcp_skb_pcount(skb);
        if (push_one)    //只发一个skb的则退出循环
            break;
    }
    ...
    //没有数据包inflight，并且有数据等待发送，则准备尝试0窗口探测
    return !tp->packets_out && tcp_send_head(sk);
}

/* Return true if the Nagle test allows this packet to be
 * sent now.
 */
static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
                  unsigned int cur_mss, int nonagle)
{
    /* Nagle rule does not apply to frames, which sit in the middle of the
     * write_queue (they have no chances to get new data).
     *
     * This is implemented in the callers, where they modify the 'nonagle'
     * argument based upon the location of SKB in the send queue.
     */
    if (nonagle & TCP_NAGLE_PUSH)
        return true;
    /* Don't use the nagle rule for urgent data (or for the final FIN). */
    if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
        return true;
    if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
        return true;
    //skb->len < cur_mss且设置了TCP_NAGLE_CORK， 或者上一个发送的小包还未被确认， 则推迟发送
    return false;
}
static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
                int nonagle)
{
    return partial &&    //skb->len < mss, 也就是说>=mss就直接发送
        ((nonagle & TCP_NAGLE_CORK) ||    //设置了cork则使用nagle
         (!nonagle && tp->packets_out && tcp_minshall_check(tp)));    //有inflight数据且上一个发送的小包还没被确认则进入nagle
}
/* Minshall's variant of the Nagle send check. */
static bool tcp_minshall_check(const struct tcp_sock *tp)
{
    return after(tp->snd_sml, tp->snd_una) &&    //上一个发送的小包还没确认
        !after(tp->snd_sml, tp->snd_nxt);    //没有回绕
}
static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
                const struct sk_buff *skb)
{
    if (skb->len < tcp_skb_pcount(skb) * mss_now)
        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
}

static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
                 bool *is_cwnd_limited, u32 max_segs)
{
    const struct inet_connection_sock *icsk = inet_csk(sk);
    u32 age, send_win, cong_win, limit, in_flight;
    struct tcp_sock *tp = tcp_sk(sk);
    struct skb_mstamp now;
    struct sk_buff *head;
    int win_divisor;
    if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
        goto send_now;
    if (icsk->icsk_ca_state >= TCP_CA_Recovery)
        goto send_now;
    /* Avoid bursty behavior by allowing defer
     * only if the last write was recent.
     */
    if ((s32)(tcp_time_stamp - tp->lsndtime) > 0)
        goto send_now;
    in_flight = tcp_packets_in_flight(tp);
    BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
    send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;    //发送窗口
    /* From in_flight test above, we know that cwnd > in_flight.  */
    cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;    //拥塞窗口
    limit = min(send_win, cong_win);    //最大发送窗口剩余
    /* If a full-sized TSO skb can be sent, do it. */
    if (limit >= max_segs * tp->mss_cache)    //支持最大尺寸的tso发送
        goto send_now;
    /* Middle in queue won't get any more data, full sendable already? */
    if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))    //不是发送队列的最后一个，且满足发送窗口
        goto send_now;    //直接发送，不会有数据被添加到这个skb了
    win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);
    if (win_divisor) {
        u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
        /* If at least some fraction of a window is available,
         * just use it.
         */
        chunk /= win_divisor;
        if (limit >= chunk)    //剩余的窗口大于总窗口的比例， 默认1/3
            goto send_now;
    } else {
        /* Different approach, try not to defer past a single
         * ACK.  Receiver should ACK every other full sized
         * frame, so if we have space for more than 3 frames
         * then send now.
         */
        if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
            goto send_now;
    }
    head = tcp_write_queue_head(sk);
    skb_mstamp_get(&now);
    age = skb_mstamp_us_delta(&now, &head->skb_mstamp);    //最早的未确认包的距离现在的时间
    /* If next ACK is likely to come too late (half srtt), do not defer */
    if (age < (tp->srtt_us >> 4))    // 也就是说下一个ack的到来很可能大于1/2的srtt，直接发送
        goto send_now;
    /* Ok, it looks like it is advisable to defer. */
    //当前skb的收到cwnd限制
    if (cong_win < send_win && cong_win <= skb->len)
        *is_cwnd_limited = true;
    //可以推迟发送了
    return true;
send_now:
    return false;
}