linux/net/ipv4/tcp.c
<<
>>
Prefs
   1/*
   2 * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3 *              operating system.  INET is implemented using the  BSD Socket
   4 *              interface as the means of communication with the user level.
   5 *
   6 *              Implementation of the Transmission Control Protocol(TCP).
   7 *
   8 * Authors:     Ross Biro
   9 *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10 *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  11 *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  12 *              Florian La Roche, <flla@stud.uni-sb.de>
  13 *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  14 *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  15 *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  16 *              Matthew Dillon, <dillon@apollo.west.oic.com>
  17 *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  18 *              Jorge Cwik, <jorge@laser.satlink.net>
  19 *
  20 * Fixes:
  21 *              Alan Cox        :       Numerous verify_area() calls
  22 *              Alan Cox        :       Set the ACK bit on a reset
  23 *              Alan Cox        :       Stopped it crashing if it closed while
  24 *                                      sk->inuse=1 and was trying to connect
  25 *                                      (tcp_err()).
  26 *              Alan Cox        :       All icmp error handling was broken
  27 *                                      pointers passed where wrong and the
  28 *                                      socket was looked up backwards. Nobody
  29 *                                      tested any icmp error code obviously.
  30 *              Alan Cox        :       tcp_err() now handled properly. It
  31 *                                      wakes people on errors. poll
  32 *                                      behaves and the icmp error race
  33 *                                      has gone by moving it into sock.c
  34 *              Alan Cox        :       tcp_send_reset() fixed to work for
  35 *                                      everything not just packets for
  36 *                                      unknown sockets.
  37 *              Alan Cox        :       tcp option processing.
  38 *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  39 *                                      syn rule wrong]
  40 *              Herp Rosmanith  :       More reset fixes
  41 *              Alan Cox        :       No longer acks invalid rst frames.
  42 *                                      Acking any kind of RST is right out.
  43 *              Alan Cox        :       Sets an ignore me flag on an rst
  44 *                                      receive otherwise odd bits of prattle
  45 *                                      escape still
  46 *              Alan Cox        :       Fixed another acking RST frame bug.
  47 *                                      Should stop LAN workplace lockups.
  48 *              Alan Cox        :       Some tidyups using the new skb list
  49 *                                      facilities
  50 *              Alan Cox        :       sk->keepopen now seems to work
  51 *              Alan Cox        :       Pulls options out correctly on accepts
  52 *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  53 *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  54 *                                      bit to skb ops.
  55 *              Alan Cox        :       Tidied tcp_data to avoid a potential
  56 *                                      nasty.
  57 *              Alan Cox        :       Added some better commenting, as the
  58 *                                      tcp is hard to follow
  59 *              Alan Cox        :       Removed incorrect check for 20 * psh
  60 *      Michael O'Reilly        :       ack < copied bug fix.
  61 *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  62 *              Alan Cox        :       FIN with no memory -> CRASH
  63 *              Alan Cox        :       Added socket option proto entries.
  64 *                                      Also added awareness of them to accept.
  65 *              Alan Cox        :       Added TCP options (SOL_TCP)
  66 *              Alan Cox        :       Switched wakeup calls to callbacks,
  67 *                                      so the kernel can layer network
  68 *                                      sockets.
  69 *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  70 *              Alan Cox        :       Handle FIN (more) properly (we hope).
  71 *              Alan Cox        :       RST frames sent on unsynchronised
  72 *                                      state ack error.
  73 *              Alan Cox        :       Put in missing check for SYN bit.
  74 *              Alan Cox        :       Added tcp_select_window() aka NET2E
  75 *                                      window non shrink trick.
  76 *              Alan Cox        :       Added a couple of small NET2E timer
  77 *                                      fixes
  78 *              Charles Hedrick :       TCP fixes
  79 *              Toomas Tamm     :       TCP window fixes
  80 *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  81 *              Charles Hedrick :       Rewrote most of it to actually work
  82 *              Linus           :       Rewrote tcp_read() and URG handling
  83 *                                      completely
  84 *              Gerhard Koerting:       Fixed some missing timer handling
  85 *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  86 *              Gerhard Koerting:       PC/TCP workarounds
  87 *              Adam Caldwell   :       Assorted timer/timing errors
  88 *              Matthew Dillon  :       Fixed another RST bug
  89 *              Alan Cox        :       Move to kernel side addressing changes.
  90 *              Alan Cox        :       Beginning work on TCP fastpathing
  91 *                                      (not yet usable)
  92 *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  93 *              Alan Cox        :       TCP fast path debugging
  94 *              Alan Cox        :       Window clamping
  95 *              Michael Riepe   :       Bug in tcp_check()
  96 *              Matt Dillon     :       More TCP improvements and RST bug fixes
  97 *              Matt Dillon     :       Yet more small nasties remove from the
  98 *                                      TCP code (Be very nice to this man if
  99 *                                      tcp finally works 100%) 8)
 100 *              Alan Cox        :       BSD accept semantics.
 101 *              Alan Cox        :       Reset on closedown bug.
 102 *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 103 *              Michael Pall    :       Handle poll() after URG properly in
 104 *                                      all cases.
 105 *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 106 *                                      (multi URG PUSH broke rlogin).
 107 *              Michael Pall    :       Fix the multi URG PUSH problem in
 108 *                                      tcp_readable(), poll() after URG
 109 *                                      works now.
 110 *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 111 *                                      BSD api.
 112 *              Alan Cox        :       Changed the semantics of sk->socket to
 113 *                                      fix a race and a signal problem with
 114 *                                      accept() and async I/O.
 115 *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 116 *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 117 *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 118 *                                      clients/servers which listen in on
 119 *                                      fixed ports.
 120 *              Alan Cox        :       Cleaned the above up and shrank it to
 121 *                                      a sensible code size.
 122 *              Alan Cox        :       Self connect lockup fix.
 123 *              Alan Cox        :       No connect to multicast.
 124 *              Ross Biro       :       Close unaccepted children on master
 125 *                                      socket close.
 126 *              Alan Cox        :       Reset tracing code.
 127 *              Alan Cox        :       Spurious resets on shutdown.
 128 *              Alan Cox        :       Giant 15 minute/60 second timer error
 129 *              Alan Cox        :       Small whoops in polling before an
 130 *                                      accept.
 131 *              Alan Cox        :       Kept the state trace facility since
 132 *                                      it's handy for debugging.
 133 *              Alan Cox        :       More reset handler fixes.
 134 *              Alan Cox        :       Started rewriting the code based on
 135 *                                      the RFC's for other useful protocol
 136 *                                      references see: Comer, KA9Q NOS, and
 137 *                                      for a reference on the difference
 138 *                                      between specifications and how BSD
 139 *                                      works see the 4.4lite source.
 140 *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 141 *                                      close.
 142 *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 143 *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 144 *              Alan Cox        :       Reimplemented timers as per the RFC
 145 *                                      and using multiple timers for sanity.
 146 *              Alan Cox        :       Small bug fixes, and a lot of new
 147 *                                      comments.
 148 *              Alan Cox        :       Fixed dual reader crash by locking
 149 *                                      the buffers (much like datagram.c)
 150 *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 151 *                                      now gets fed up of retrying without
 152 *                                      (even a no space) answer.
 153 *              Alan Cox        :       Extracted closing code better
 154 *              Alan Cox        :       Fixed the closing state machine to
 155 *                                      resemble the RFC.
 156 *              Alan Cox        :       More 'per spec' fixes.
 157 *              Jorge Cwik      :       Even faster checksumming.
 158 *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 159 *                                      only frames. At least one pc tcp stack
 160 *                                      generates them.
 161 *              Alan Cox        :       Cache last socket.
 162 *              Alan Cox        :       Per route irtt.
 163 *              Matt Day        :       poll()->select() match BSD precisely on error
 164 *              Alan Cox        :       New buffers
 165 *              Marc Tamsky     :       Various sk->prot->retransmits and
 166 *                                      sk->retransmits misupdating fixed.
 167 *                                      Fixed tcp_write_timeout: stuck close,
 168 *                                      and TCP syn retries gets used now.
 169 *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 170 *                                      ack if state is TCP_CLOSED.
 171 *              Alan Cox        :       Look up device on a retransmit - routes may
 172 *                                      change. Doesn't yet cope with MSS shrink right
 173 *                                      but it's a start!
 174 *              Marc Tamsky     :       Closing in closing fixes.
 175 *              Mike Shaver     :       RFC1122 verifications.
 176 *              Alan Cox        :       rcv_saddr errors.
 177 *              Alan Cox        :       Block double connect().
 178 *              Alan Cox        :       Small hooks for enSKIP.
 179 *              Alexey Kuznetsov:       Path MTU discovery.
 180 *              Alan Cox        :       Support soft errors.
 181 *              Alan Cox        :       Fix MTU discovery pathological case
 182 *                                      when the remote claims no mtu!
 183 *              Marc Tamsky     :       TCP_CLOSE fix.
 184 *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 185 *                                      window but wrong (fixes NT lpd problems)
 186 *              Pedro Roque     :       Better TCP window handling, delayed ack.
 187 *              Joerg Reuter    :       No modification of locked buffers in
 188 *                                      tcp_do_retransmit()
 189 *              Eric Schenk     :       Changed receiver side silly window
 190 *                                      avoidance algorithm to BSD style
 191 *                                      algorithm. This doubles throughput
 192 *                                      against machines running Solaris,
 193 *                                      and seems to result in general
 194 *                                      improvement.
 195 *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 196 *      Willy Konynenberg       :       Transparent proxying support.
 197 *      Mike McLagan            :       Routing by source
 198 *              Keith Owens     :       Do proper merging with partial SKB's in
 199 *                                      tcp_do_sendmsg to avoid burstiness.
 200 *              Eric Schenk     :       Fix fast close down bug with
 201 *                                      shutdown() followed by close().
 202 *              Andi Kleen      :       Make poll agree with SIGIO
 203 *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 204 *                                      lingertime == 0 (RFC 793 ABORT Call)
 205 *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 206 *                                      csum_and_copy_from_user() if possible.
 207 *
 208 *              This program is free software; you can redistribute it and/or
 209 *              modify it under the terms of the GNU General Public License
 210 *              as published by the Free Software Foundation; either version
 211 *              2 of the License, or(at your option) any later version.
 212 *
 213 * Description of States:
 214 *
 215 *      TCP_SYN_SENT            sent a connection request, waiting for ack
 216 *
 217 *      TCP_SYN_RECV            received a connection request, sent ack,
 218 *                              waiting for final ack in three-way handshake.
 219 *
 220 *      TCP_ESTABLISHED         connection established
 221 *
 222 *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 223 *                              transmission of remaining buffered data
 224 *
 225 *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 226 *                              to shutdown
 227 *
 228 *      TCP_CLOSING             both sides have shutdown but we still have
 229 *                              data we have to finish sending
 230 *
 231 *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 232 *                              closed, can only be entered from FIN_WAIT2
 233 *                              or CLOSING.  Required because the other end
 234 *                              may not have gotten our last ACK causing it
 235 *                              to retransmit the data packet (which we ignore)
 236 *
 237 *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 238 *                              us to finish writing our data and to shutdown
 239 *                              (we have to close() to move on to LAST_ACK)
 240 *
 241 *      TCP_LAST_ACK            out side has shutdown after remote has
 242 *                              shutdown.  There may still be data in our
 243 *                              buffer that we have to finish sending
 244 *
 245 *      TCP_CLOSE               socket is finished
 246 */
 247
 248#define pr_fmt(fmt) "TCP: " fmt
 249
 250#include <crypto/hash.h>
 251#include <linux/kernel.h>
 252#include <linux/module.h>
 253#include <linux/types.h>
 254#include <linux/fcntl.h>
 255#include <linux/poll.h>
 256#include <linux/inet_diag.h>
 257#include <linux/init.h>
 258#include <linux/fs.h>
 259#include <linux/skbuff.h>
 260#include <linux/scatterlist.h>
 261#include <linux/splice.h>
 262#include <linux/net.h>
 263#include <linux/socket.h>
 264#include <linux/random.h>
 265#include <linux/bootmem.h>
 266#include <linux/highmem.h>
 267#include <linux/swap.h>
 268#include <linux/cache.h>
 269#include <linux/err.h>
 270#include <linux/time.h>
 271#include <linux/slab.h>
 272
 273#include <net/icmp.h>
 274#include <net/inet_common.h>
 275#include <net/tcp.h>
 276#include <net/xfrm.h>
 277#include <net/ip.h>
 278#include <net/sock.h>
 279
 280#include <asm/uaccess.h>
 281#include <asm/ioctls.h>
 282#include <asm/unaligned.h>
 283#include <net/busy_poll.h>
 284
 285int sysctl_tcp_min_tso_segs __read_mostly = 2;
 286
 287int sysctl_tcp_autocorking __read_mostly = 1;
 288
 289struct percpu_counter tcp_orphan_count;
 290EXPORT_SYMBOL_GPL(tcp_orphan_count);
 291
 292long sysctl_tcp_mem[3] __read_mostly;
 293int sysctl_tcp_wmem[3] __read_mostly;
 294int sysctl_tcp_rmem[3] __read_mostly;
 295
 296EXPORT_SYMBOL(sysctl_tcp_mem);
 297EXPORT_SYMBOL(sysctl_tcp_rmem);
 298EXPORT_SYMBOL(sysctl_tcp_wmem);
 299
 300atomic_long_t tcp_memory_allocated;     /* Current allocated memory. */
 301EXPORT_SYMBOL(tcp_memory_allocated);
 302
 303/*
 304 * Current number of TCP sockets.
 305 */
 306struct percpu_counter tcp_sockets_allocated;
 307EXPORT_SYMBOL(tcp_sockets_allocated);
 308
 309/*
 310 * TCP splice context
 311 */
 312struct tcp_splice_state {
 313        struct pipe_inode_info *pipe;
 314        size_t len;
 315        unsigned int flags;
 316};
 317
 318/*
 319 * Pressure flag: try to collapse.
 320 * Technical note: it is used by multiple contexts non atomically.
 321 * All the __sk_mem_schedule() is of this nature: accounting
 322 * is strict, actions are advisory and have some latency.
 323 */
 324int tcp_memory_pressure __read_mostly;
 325EXPORT_SYMBOL(tcp_memory_pressure);
 326
 327void tcp_enter_memory_pressure(struct sock *sk)
 328{
 329        if (!tcp_memory_pressure) {
 330                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
 331                tcp_memory_pressure = 1;
 332        }
 333}
 334EXPORT_SYMBOL(tcp_enter_memory_pressure);
 335
 336/* Convert seconds to retransmits based on initial and max timeout */
 337static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
 338{
 339        u8 res = 0;
 340
 341        if (seconds > 0) {
 342                int period = timeout;
 343
 344                res = 1;
 345                while (seconds > period && res < 255) {
 346                        res++;
 347                        timeout <<= 1;
 348                        if (timeout > rto_max)
 349                                timeout = rto_max;
 350                        period += timeout;
 351                }
 352        }
 353        return res;
 354}
 355
 356/* Convert retransmits to seconds based on initial and max timeout */
 357static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
 358{
 359        int period = 0;
 360
 361        if (retrans > 0) {
 362                period = timeout;
 363                while (--retrans) {
 364                        timeout <<= 1;
 365                        if (timeout > rto_max)
 366                                timeout = rto_max;
 367                        period += timeout;
 368                }
 369        }
 370        return period;
 371}
 372
 373/* Address-family independent initialization for a tcp_sock.
 374 *
 375 * NOTE: A lot of things set to zero explicitly by call to
 376 *       sk_alloc() so need not be done here.
 377 */
 378void tcp_init_sock(struct sock *sk)
 379{
 380        struct inet_connection_sock *icsk = inet_csk(sk);
 381        struct tcp_sock *tp = tcp_sk(sk);
 382
 383        __skb_queue_head_init(&tp->out_of_order_queue);
 384        tcp_init_xmit_timers(sk);
 385        tcp_prequeue_init(tp);
 386        INIT_LIST_HEAD(&tp->tsq_node);
 387
 388        icsk->icsk_rto = TCP_TIMEOUT_INIT;
 389        tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
 390        tp->rtt_min[0].rtt = ~0U;
 391
 392        /* So many TCP implementations out there (incorrectly) count the
 393         * initial SYN frame in their delayed-ACK and congestion control
 394         * algorithms that we must have the following bandaid to talk
 395         * efficiently to them.  -DaveM
 396         */
 397        tp->snd_cwnd = TCP_INIT_CWND;
 398
 399        /* See draft-stevens-tcpca-spec-01 for discussion of the
 400         * initialization of these values.
 401         */
 402        tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 403        tp->snd_cwnd_clamp = ~0;
 404        tp->mss_cache = TCP_MSS_DEFAULT;
 405        u64_stats_init(&tp->syncp);
 406
 407        tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
 408        tcp_enable_early_retrans(tp);
 409        tcp_assign_congestion_control(sk);
 410
 411        tp->tsoffset = 0;
 412
 413        sk->sk_state = TCP_CLOSE;
 414
 415        sk->sk_write_space = sk_stream_write_space;
 416        sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
 417
 418        icsk->icsk_sync_mss = tcp_sync_mss;
 419
 420        sk->sk_sndbuf = sysctl_tcp_wmem[1];
 421        sk->sk_rcvbuf = sysctl_tcp_rmem[1];
 422
 423        local_bh_disable();
 424        if (mem_cgroup_sockets_enabled)
 425                sock_update_memcg(sk);
 426        sk_sockets_allocated_inc(sk);
 427        local_bh_enable();
 428}
 429EXPORT_SYMBOL(tcp_init_sock);
 430
 431static void tcp_tx_timestamp(struct sock *sk, u16 tsflags, struct sk_buff *skb)
 432{
 433        if (tsflags) {
 434                struct skb_shared_info *shinfo = skb_shinfo(skb);
 435                struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 436
 437                sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
 438                if (tsflags & SOF_TIMESTAMPING_TX_ACK)
 439                        tcb->txstamp_ack = 1;
 440                if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
 441                        shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
 442        }
 443}
 444
 445/*
 446 *      Wait for a TCP event.
 447 *
 448 *      Note that we don't need to lock the socket, as the upper poll layers
 449 *      take care of normal races (between the test and the event) and we don't
 450 *      go look at any of the socket buffers directly.
 451 */
 452unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 453{
 454        unsigned int mask;
 455        struct sock *sk = sock->sk;
 456        const struct tcp_sock *tp = tcp_sk(sk);
 457        int state;
 458
 459        sock_rps_record_flow(sk);
 460
 461        sock_poll_wait(file, sk_sleep(sk), wait);
 462
 463        state = sk_state_load(sk);
 464        if (state == TCP_LISTEN)
 465                return inet_csk_listen_poll(sk);
 466
 467        /* Socket is not locked. We are protected from async events
 468         * by poll logic and correct handling of state changes
 469         * made by other threads is impossible in any case.
 470         */
 471
 472        mask = 0;
 473
 474        /*
 475         * POLLHUP is certainly not done right. But poll() doesn't
 476         * have a notion of HUP in just one direction, and for a
 477         * socket the read side is more interesting.
 478         *
 479         * Some poll() documentation says that POLLHUP is incompatible
 480         * with the POLLOUT/POLLWR flags, so somebody should check this
 481         * all. But careful, it tends to be safer to return too many
 482         * bits than too few, and you can easily break real applications
 483         * if you don't tell them that something has hung up!
 484         *
 485         * Check-me.
 486         *
 487         * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 488         * our fs/select.c). It means that after we received EOF,
 489         * poll always returns immediately, making impossible poll() on write()
 490         * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 491         * if and only if shutdown has been made in both directions.
 492         * Actually, it is interesting to look how Solaris and DUX
 493         * solve this dilemma. I would prefer, if POLLHUP were maskable,
 494         * then we could set it on SND_SHUTDOWN. BTW examples given
 495         * in Stevens' books assume exactly this behaviour, it explains
 496         * why POLLHUP is incompatible with POLLOUT.    --ANK
 497         *
 498         * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 499         * blocking on fresh not-connected or disconnected socket. --ANK
 500         */
 501        if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 502                mask |= POLLHUP;
 503        if (sk->sk_shutdown & RCV_SHUTDOWN)
 504                mask |= POLLIN | POLLRDNORM | POLLRDHUP;
 505
 506        /* Connected or passive Fast Open socket? */
 507        if (state != TCP_SYN_SENT &&
 508            (state != TCP_SYN_RECV || tp->fastopen_rsk)) {
 509                int target = sock_rcvlowat(sk, 0, INT_MAX);
 510
 511                if (tp->urg_seq == tp->copied_seq &&
 512                    !sock_flag(sk, SOCK_URGINLINE) &&
 513                    tp->urg_data)
 514                        target++;
 515
 516                if (tp->rcv_nxt - tp->copied_seq >= target)
 517                        mask |= POLLIN | POLLRDNORM;
 518
 519                if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 520                        if (sk_stream_is_writeable(sk)) {
 521                                mask |= POLLOUT | POLLWRNORM;
 522                        } else {  /* send SIGIO later */
 523                                sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 524                                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 525
 526                                /* Race breaker. If space is freed after
 527                                 * wspace test but before the flags are set,
 528                                 * IO signal will be lost. Memory barrier
 529                                 * pairs with the input side.
 530                                 */
 531                                smp_mb__after_atomic();
 532                                if (sk_stream_is_writeable(sk))
 533                                        mask |= POLLOUT | POLLWRNORM;
 534                        }
 535                } else
 536                        mask |= POLLOUT | POLLWRNORM;
 537
 538                if (tp->urg_data & TCP_URG_VALID)
 539                        mask |= POLLPRI;
 540        }
 541        /* This barrier is coupled with smp_wmb() in tcp_reset() */
 542        smp_rmb();
 543        if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
 544                mask |= POLLERR;
 545
 546        return mask;
 547}
 548EXPORT_SYMBOL(tcp_poll);
 549
 550int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 551{
 552        struct tcp_sock *tp = tcp_sk(sk);
 553        int answ;
 554        bool slow;
 555
 556        switch (cmd) {
 557        case SIOCINQ:
 558                if (sk->sk_state == TCP_LISTEN)
 559                        return -EINVAL;
 560
 561                slow = lock_sock_fast(sk);
 562                answ = tcp_inq(sk);
 563                unlock_sock_fast(sk, slow);
 564                break;
 565        case SIOCATMARK:
 566                answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 567                break;
 568        case SIOCOUTQ:
 569                if (sk->sk_state == TCP_LISTEN)
 570                        return -EINVAL;
 571
 572                if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 573                        answ = 0;
 574                else
 575                        answ = tp->write_seq - tp->snd_una;
 576                break;
 577        case SIOCOUTQNSD:
 578                if (sk->sk_state == TCP_LISTEN)
 579                        return -EINVAL;
 580
 581                if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 582                        answ = 0;
 583                else
 584                        answ = tp->write_seq - tp->snd_nxt;
 585                break;
 586        default:
 587                return -ENOIOCTLCMD;
 588        }
 589
 590        return put_user(answ, (int __user *)arg);
 591}
 592EXPORT_SYMBOL(tcp_ioctl);
 593
 594static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 595{
 596        TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
 597        tp->pushed_seq = tp->write_seq;
 598}
 599
 600static inline bool forced_push(const struct tcp_sock *tp)
 601{
 602        return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 603}
 604
 605static void skb_entail(struct sock *sk, struct sk_buff *skb)
 606{
 607        struct tcp_sock *tp = tcp_sk(sk);
 608        struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 609
 610        skb->csum    = 0;
 611        tcb->seq     = tcb->end_seq = tp->write_seq;
 612        tcb->tcp_flags = TCPHDR_ACK;
 613        tcb->sacked  = 0;
 614        __skb_header_release(skb);
 615        tcp_add_write_queue_tail(sk, skb);
 616        sk->sk_wmem_queued += skb->truesize;
 617        sk_mem_charge(sk, skb->truesize);
 618        if (tp->nonagle & TCP_NAGLE_PUSH)
 619                tp->nonagle &= ~TCP_NAGLE_PUSH;
 620
 621        tcp_slow_start_after_idle_check(sk);
 622}
 623
 624static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
 625{
 626        if (flags & MSG_OOB)
 627                tp->snd_up = tp->write_seq;
 628}
 629
 630/* If a not yet filled skb is pushed, do not send it if
 631 * we have data packets in Qdisc or NIC queues :
 632 * Because TX completion will happen shortly, it gives a chance
 633 * to coalesce future sendmsg() payload into this skb, without
 634 * need for a timer, and with no latency trade off.
 635 * As packets containing data payload have a bigger truesize
 636 * than pure acks (dataless) packets, the last checks prevent
 637 * autocorking if we only have an ACK in Qdisc/NIC queues,
 638 * or if TX completion was delayed after we processed ACK packet.
 639 */
 640static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
 641                                int size_goal)
 642{
 643        return skb->len < size_goal &&
 644               sysctl_tcp_autocorking &&
 645               skb != tcp_write_queue_head(sk) &&
 646               atomic_read(&sk->sk_wmem_alloc) > skb->truesize;
 647}
 648
 649static void tcp_push(struct sock *sk, int flags, int mss_now,
 650                     int nonagle, int size_goal)
 651{
 652        struct tcp_sock *tp = tcp_sk(sk);
 653        struct sk_buff *skb;
 654
 655        if (!tcp_send_head(sk))
 656                return;
 657
 658        skb = tcp_write_queue_tail(sk);
 659        if (!(flags & MSG_MORE) || forced_push(tp))
 660                tcp_mark_push(tp, skb);
 661
 662        tcp_mark_urg(tp, flags);
 663
 664        if (tcp_should_autocork(sk, skb, size_goal)) {
 665
 666                /* avoid atomic op if TSQ_THROTTLED bit is already set */
 667                if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
 668                        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
 669                        set_bit(TSQ_THROTTLED, &tp->tsq_flags);
 670                }
 671                /* It is possible TX completion already happened
 672                 * before we set TSQ_THROTTLED.
 673                 */
 674                if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
 675                        return;
 676        }
 677
 678        if (flags & MSG_MORE)
 679                nonagle = TCP_NAGLE_CORK;
 680
 681        __tcp_push_pending_frames(sk, mss_now, nonagle);
 682}
 683
 684static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 685                                unsigned int offset, size_t len)
 686{
 687        struct tcp_splice_state *tss = rd_desc->arg.data;
 688        int ret;
 689
 690        ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
 691                              min(rd_desc->count, len), tss->flags,
 692                              skb_socket_splice);
 693        if (ret > 0)
 694                rd_desc->count -= ret;
 695        return ret;
 696}
 697
 698static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 699{
 700        /* Store TCP splice context information in read_descriptor_t. */
 701        read_descriptor_t rd_desc = {
 702                .arg.data = tss,
 703                .count    = tss->len,
 704        };
 705
 706        return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 707}
 708
 709/**
 710 *  tcp_splice_read - splice data from TCP socket to a pipe
 711 * @sock:       socket to splice from
 712 * @ppos:       position (not valid)
 713 * @pipe:       pipe to splice to
 714 * @len:        number of bytes to splice
 715 * @flags:      splice modifier flags
 716 *
 717 * Description:
 718 *    Will read pages from given socket and fill them into a pipe.
 719 *
 720 **/
 721ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 722                        struct pipe_inode_info *pipe, size_t len,
 723                        unsigned int flags)
 724{
 725        struct sock *sk = sock->sk;
 726        struct tcp_splice_state tss = {
 727                .pipe = pipe,
 728                .len = len,
 729                .flags = flags,
 730        };
 731        long timeo;
 732        ssize_t spliced;
 733        int ret;
 734
 735        sock_rps_record_flow(sk);
 736        /*
 737         * We can't seek on a socket input
 738         */
 739        if (unlikely(*ppos))
 740                return -ESPIPE;
 741
 742        ret = spliced = 0;
 743
 744        lock_sock(sk);
 745
 746        timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
 747        while (tss.len) {
 748                ret = __tcp_splice_read(sk, &tss);
 749                if (ret < 0)
 750                        break;
 751                else if (!ret) {
 752                        if (spliced)
 753                                break;
 754                        if (sock_flag(sk, SOCK_DONE))
 755                                break;
 756                        if (sk->sk_err) {
 757                                ret = sock_error(sk);
 758                                break;
 759                        }
 760                        if (sk->sk_shutdown & RCV_SHUTDOWN)
 761                                break;
 762                        if (sk->sk_state == TCP_CLOSE) {
 763                                /*
 764                                 * This occurs when user tries to read
 765                                 * from never connected socket.
 766                                 */
 767                                if (!sock_flag(sk, SOCK_DONE))
 768                                        ret = -ENOTCONN;
 769                                break;
 770                        }
 771                        if (!timeo) {
 772                                ret = -EAGAIN;
 773                                break;
 774                        }
 775                        sk_wait_data(sk, &timeo, NULL);
 776                        if (signal_pending(current)) {
 777                                ret = sock_intr_errno(timeo);
 778                                break;
 779                        }
 780                        continue;
 781                }
 782                tss.len -= ret;
 783                spliced += ret;
 784
 785                if (!timeo)
 786                        break;
 787                release_sock(sk);
 788                lock_sock(sk);
 789
 790                if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 791                    (sk->sk_shutdown & RCV_SHUTDOWN) ||
 792                    signal_pending(current))
 793                        break;
 794        }
 795
 796        release_sock(sk);
 797
 798        if (spliced)
 799                return spliced;
 800
 801        return ret;
 802}
 803EXPORT_SYMBOL(tcp_splice_read);
 804
 805struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
 806                                    bool force_schedule)
 807{
 808        struct sk_buff *skb;
 809
 810        /* The TCP header must be at least 32-bit aligned.  */
 811        size = ALIGN(size, 4);
 812
 813        if (unlikely(tcp_under_memory_pressure(sk)))
 814                sk_mem_reclaim_partial(sk);
 815
 816        skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 817        if (likely(skb)) {
 818                bool mem_scheduled;
 819
 820                if (force_schedule) {
 821                        mem_scheduled = true;
 822                        sk_forced_mem_schedule(sk, skb->truesize);
 823                } else {
 824                        mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
 825                }
 826                if (likely(mem_scheduled)) {
 827                        skb_reserve(skb, sk->sk_prot->max_header);
 828                        /*
 829                         * Make sure that we have exactly size bytes
 830                         * available to the caller, no more, no less.
 831                         */
 832                        skb->reserved_tailroom = skb->end - skb->tail - size;
 833                        return skb;
 834                }
 835                __kfree_skb(skb);
 836        } else {
 837                sk->sk_prot->enter_memory_pressure(sk);
 838                sk_stream_moderate_sndbuf(sk);
 839        }
 840        return NULL;
 841}
 842
 843static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
 844                                       int large_allowed)
 845{
 846        struct tcp_sock *tp = tcp_sk(sk);
 847        u32 new_size_goal, size_goal;
 848
 849        if (!large_allowed || !sk_can_gso(sk))
 850                return mss_now;
 851
 852        /* Note : tcp_tso_autosize() will eventually split this later */
 853        new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
 854        new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
 855
 856        /* We try hard to avoid divides here */
 857        size_goal = tp->gso_segs * mss_now;
 858        if (unlikely(new_size_goal < size_goal ||
 859                     new_size_goal >= size_goal + mss_now)) {
 860                tp->gso_segs = min_t(u16, new_size_goal / mss_now,
 861                                     sk->sk_gso_max_segs);
 862                size_goal = tp->gso_segs * mss_now;
 863        }
 864
 865        return max(size_goal, mss_now);
 866}
 867
 868static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 869{
 870        int mss_now;
 871
 872        mss_now = tcp_current_mss(sk);
 873        *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
 874
 875        return mss_now;
 876}
 877
 878static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 879                                size_t size, int flags)
 880{
 881        struct tcp_sock *tp = tcp_sk(sk);
 882        int mss_now, size_goal;
 883        int err;
 884        ssize_t copied;
 885        long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 886
 887        /* Wait for a connection to finish. One exception is TCP Fast Open
 888         * (passive side) where data is allowed to be sent before a connection
 889         * is fully established.
 890         */
 891        if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
 892            !tcp_passive_fastopen(sk)) {
 893                err = sk_stream_wait_connect(sk, &timeo);
 894                if (err != 0)
 895                        goto out_err;
 896        }
 897
 898        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 899
 900        mss_now = tcp_send_mss(sk, &size_goal, flags);
 901        copied = 0;
 902
 903        err = -EPIPE;
 904        if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 905                goto out_err;
 906
 907        while (size > 0) {
 908                struct sk_buff *skb = tcp_write_queue_tail(sk);
 909                int copy, i;
 910                bool can_coalesce;
 911
 912                if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0 ||
 913                    !tcp_skb_can_collapse_to(skb)) {
 914new_segment:
 915                        if (!sk_stream_memory_free(sk))
 916                                goto wait_for_sndbuf;
 917
 918                        skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
 919                                                  skb_queue_empty(&sk->sk_write_queue));
 920                        if (!skb)
 921                                goto wait_for_memory;
 922
 923                        skb_entail(sk, skb);
 924                        copy = size_goal;
 925                }
 926
 927                if (copy > size)
 928                        copy = size;
 929
 930                i = skb_shinfo(skb)->nr_frags;
 931                can_coalesce = skb_can_coalesce(skb, i, page, offset);
 932                if (!can_coalesce && i >= sysctl_max_skb_frags) {
 933                        tcp_mark_push(tp, skb);
 934                        goto new_segment;
 935                }
 936                if (!sk_wmem_schedule(sk, copy))
 937                        goto wait_for_memory;
 938
 939                if (can_coalesce) {
 940                        skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
 941                } else {
 942                        get_page(page);
 943                        skb_fill_page_desc(skb, i, page, offset, copy);
 944                }
 945                skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
 946
 947                skb->len += copy;
 948                skb->data_len += copy;
 949                skb->truesize += copy;
 950                sk->sk_wmem_queued += copy;
 951                sk_mem_charge(sk, copy);
 952                skb->ip_summed = CHECKSUM_PARTIAL;
 953                tp->write_seq += copy;
 954                TCP_SKB_CB(skb)->end_seq += copy;
 955                tcp_skb_pcount_set(skb, 0);
 956
 957                if (!copied)
 958                        TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
 959
 960                copied += copy;
 961                offset += copy;
 962                size -= copy;
 963                if (!size) {
 964                        tcp_tx_timestamp(sk, sk->sk_tsflags, skb);
 965                        goto out;
 966                }
 967
 968                if (skb->len < size_goal || (flags & MSG_OOB))
 969                        continue;
 970
 971                if (forced_push(tp)) {
 972                        tcp_mark_push(tp, skb);
 973                        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 974                } else if (skb == tcp_send_head(sk))
 975                        tcp_push_one(sk, mss_now);
 976                continue;
 977
 978wait_for_sndbuf:
 979                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 980wait_for_memory:
 981                tcp_push(sk, flags & ~MSG_MORE, mss_now,
 982                         TCP_NAGLE_PUSH, size_goal);
 983
 984                err = sk_stream_wait_memory(sk, &timeo);
 985                if (err != 0)
 986                        goto do_error;
 987
 988                mss_now = tcp_send_mss(sk, &size_goal, flags);
 989        }
 990
 991out:
 992        if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
 993                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
 994        return copied;
 995
 996do_error:
 997        if (copied)
 998                goto out;
 999out_err:
1000        /* make sure we wake any epoll edge trigger waiter */
1001        if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1002                sk->sk_write_space(sk);
1003        return sk_stream_error(sk, flags, err);
1004}
1005
1006int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1007                 size_t size, int flags)
1008{
1009        ssize_t res;
1010
1011        if (!(sk->sk_route_caps & NETIF_F_SG) ||
1012            !sk_check_csum_caps(sk))
1013                return sock_no_sendpage(sk->sk_socket, page, offset, size,
1014                                        flags);
1015
1016        lock_sock(sk);
1017        res = do_tcp_sendpages(sk, page, offset, size, flags);
1018        release_sock(sk);
1019        return res;
1020}
1021EXPORT_SYMBOL(tcp_sendpage);
1022
1023static inline int select_size(const struct sock *sk, bool sg)
1024{
1025        const struct tcp_sock *tp = tcp_sk(sk);
1026        int tmp = tp->mss_cache;
1027
1028        if (sg) {
1029                if (sk_can_gso(sk)) {
1030                        /* Small frames wont use a full page:
1031                         * Payload will immediately follow tcp header.
1032                         */
1033                        tmp = SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
1034                } else {
1035                        int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
1036
1037                        if (tmp >= pgbreak &&
1038                            tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
1039                                tmp = pgbreak;
1040                }
1041        }
1042
1043        return tmp;
1044}
1045
1046void tcp_free_fastopen_req(struct tcp_sock *tp)
1047{
1048        if (tp->fastopen_req) {
1049                kfree(tp->fastopen_req);
1050                tp->fastopen_req = NULL;
1051        }
1052}
1053
1054static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1055                                int *copied, size_t size)
1056{
1057        struct tcp_sock *tp = tcp_sk(sk);
1058        int err, flags;
1059
1060        if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE))
1061                return -EOPNOTSUPP;
1062        if (tp->fastopen_req)
1063                return -EALREADY; /* Another Fast Open is in progress */
1064
1065        tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1066                                   sk->sk_allocation);
1067        if (unlikely(!tp->fastopen_req))
1068                return -ENOBUFS;
1069        tp->fastopen_req->data = msg;
1070        tp->fastopen_req->size = size;
1071
1072        flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1073        err = __inet_stream_connect(sk->sk_socket, msg->msg_name,
1074                                    msg->msg_namelen, flags);
1075        *copied = tp->fastopen_req->copied;
1076        tcp_free_fastopen_req(tp);
1077        return err;
1078}
1079
1080int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1081{
1082        struct tcp_sock *tp = tcp_sk(sk);
1083        struct sk_buff *skb;
1084        struct sockcm_cookie sockc;
1085        int flags, err, copied = 0;
1086        int mss_now = 0, size_goal, copied_syn = 0;
1087        bool process_backlog = false;
1088        bool sg;
1089        long timeo;
1090
1091        lock_sock(sk);
1092
1093        flags = msg->msg_flags;
1094        if (flags & MSG_FASTOPEN) {
1095                err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
1096                if (err == -EINPROGRESS && copied_syn > 0)
1097                        goto out;
1098                else if (err)
1099                        goto out_err;
1100        }
1101
1102        timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1103
1104        /* Wait for a connection to finish. One exception is TCP Fast Open
1105         * (passive side) where data is allowed to be sent before a connection
1106         * is fully established.
1107         */
1108        if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1109            !tcp_passive_fastopen(sk)) {
1110                err = sk_stream_wait_connect(sk, &timeo);
1111                if (err != 0)
1112                        goto do_error;
1113        }
1114
1115        if (unlikely(tp->repair)) {
1116                if (tp->repair_queue == TCP_RECV_QUEUE) {
1117                        copied = tcp_send_rcvq(sk, msg, size);
1118                        goto out_nopush;
1119                }
1120
1121                err = -EINVAL;
1122                if (tp->repair_queue == TCP_NO_QUEUE)
1123                        goto out_err;
1124
1125                /* 'common' sending to sendq */
1126        }
1127
1128        sockc.tsflags = sk->sk_tsflags;
1129        if (msg->msg_controllen) {
1130                err = sock_cmsg_send(sk, msg, &sockc);
1131                if (unlikely(err)) {
1132                        err = -EINVAL;
1133                        goto out_err;
1134                }
1135        }
1136
1137        /* This should be in poll */
1138        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1139
1140        /* Ok commence sending. */
1141        copied = 0;
1142
1143restart:
1144        mss_now = tcp_send_mss(sk, &size_goal, flags);
1145
1146        err = -EPIPE;
1147        if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1148                goto out_err;
1149
1150        sg = !!(sk->sk_route_caps & NETIF_F_SG);
1151
1152        while (msg_data_left(msg)) {
1153                int copy = 0;
1154                int max = size_goal;
1155
1156                skb = tcp_write_queue_tail(sk);
1157                if (tcp_send_head(sk)) {
1158                        if (skb->ip_summed == CHECKSUM_NONE)
1159                                max = mss_now;
1160                        copy = max - skb->len;
1161                }
1162
1163                if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1164new_segment:
1165                        /* Allocate new segment. If the interface is SG,
1166                         * allocate skb fitting to single page.
1167                         */
1168                        if (!sk_stream_memory_free(sk))
1169                                goto wait_for_sndbuf;
1170
1171                        if (process_backlog && sk_flush_backlog(sk)) {
1172                                process_backlog = false;
1173                                goto restart;
1174                        }
1175                        skb = sk_stream_alloc_skb(sk,
1176                                                  select_size(sk, sg),
1177                                                  sk->sk_allocation,
1178                                                  skb_queue_empty(&sk->sk_write_queue));
1179                        if (!skb)
1180                                goto wait_for_memory;
1181
1182                        process_backlog = true;
1183                        /*
1184                         * Check whether we can use HW checksum.
1185                         */
1186                        if (sk_check_csum_caps(sk))
1187                                skb->ip_summed = CHECKSUM_PARTIAL;
1188
1189                        skb_entail(sk, skb);
1190                        copy = size_goal;
1191                        max = size_goal;
1192
1193                        /* All packets are restored as if they have
1194                         * already been sent. skb_mstamp isn't set to
1195                         * avoid wrong rtt estimation.
1196                         */
1197                        if (tp->repair)
1198                                TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1199                }
1200
1201                /* Try to append data to the end of skb. */
1202                if (copy > msg_data_left(msg))
1203                        copy = msg_data_left(msg);
1204
1205                /* Where to copy to? */
1206                if (skb_availroom(skb) > 0) {
1207                        /* We have some space in skb head. Superb! */
1208                        copy = min_t(int, copy, skb_availroom(skb));
1209                        err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1210                        if (err)
1211                                goto do_fault;
1212                } else {
1213                        bool merge = true;
1214                        int i = skb_shinfo(skb)->nr_frags;
1215                        struct page_frag *pfrag = sk_page_frag(sk);
1216
1217                        if (!sk_page_frag_refill(sk, pfrag))
1218                                goto wait_for_memory;
1219
1220                        if (!skb_can_coalesce(skb, i, pfrag->page,
1221                                              pfrag->offset)) {
1222                                if (i == sysctl_max_skb_frags || !sg) {
1223                                        tcp_mark_push(tp, skb);
1224                                        goto new_segment;
1225                                }
1226                                merge = false;
1227                        }
1228
1229                        copy = min_t(int, copy, pfrag->size - pfrag->offset);
1230
1231                        if (!sk_wmem_schedule(sk, copy))
1232                                goto wait_for_memory;
1233
1234                        err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1235                                                       pfrag->page,
1236                                                       pfrag->offset,
1237                                                       copy);
1238                        if (err)
1239                                goto do_error;
1240
1241                        /* Update the skb. */
1242                        if (merge) {
1243                                skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1244                        } else {
1245                                skb_fill_page_desc(skb, i, pfrag->page,
1246                                                   pfrag->offset, copy);
1247                                get_page(pfrag->page);
1248                        }
1249                        pfrag->offset += copy;
1250                }
1251
1252                if (!copied)
1253                        TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1254
1255                tp->write_seq += copy;
1256                TCP_SKB_CB(skb)->end_seq += copy;
1257                tcp_skb_pcount_set(skb, 0);
1258
1259                copied += copy;
1260                if (!msg_data_left(msg)) {
1261                        tcp_tx_timestamp(sk, sockc.tsflags, skb);
1262                        if (unlikely(flags & MSG_EOR))
1263                                TCP_SKB_CB(skb)->eor = 1;
1264                        goto out;
1265                }
1266
1267                if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))
1268                        continue;
1269
1270                if (forced_push(tp)) {
1271                        tcp_mark_push(tp, skb);
1272                        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1273                } else if (skb == tcp_send_head(sk))
1274                        tcp_push_one(sk, mss_now);
1275                continue;
1276
1277wait_for_sndbuf:
1278                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1279wait_for_memory:
1280                if (copied)
1281                        tcp_push(sk, flags & ~MSG_MORE, mss_now,
1282                                 TCP_NAGLE_PUSH, size_goal);
1283
1284                err = sk_stream_wait_memory(sk, &timeo);
1285                if (err != 0)
1286                        goto do_error;
1287
1288                mss_now = tcp_send_mss(sk, &size_goal, flags);
1289        }
1290
1291out:
1292        if (copied)
1293                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1294out_nopush:
1295        release_sock(sk);
1296        return copied + copied_syn;
1297
1298do_fault:
1299        if (!skb->len) {
1300                tcp_unlink_write_queue(skb, sk);
1301                /* It is the one place in all of TCP, except connection
1302                 * reset, where we can be unlinking the send_head.
1303                 */
1304                tcp_check_send_head(sk, skb);
1305                sk_wmem_free_skb(sk, skb);
1306        }
1307
1308do_error:
1309        if (copied + copied_syn)
1310                goto out;
1311out_err:
1312        err = sk_stream_error(sk, flags, err);
1313        /* make sure we wake any epoll edge trigger waiter */
1314        if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1315                sk->sk_write_space(sk);
1316        release_sock(sk);
1317        return err;
1318}
1319EXPORT_SYMBOL(tcp_sendmsg);
1320
1321/*
1322 *      Handle reading urgent data. BSD has very simple semantics for
1323 *      this, no blocking and very strange errors 8)
1324 */
1325
1326static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1327{
1328        struct tcp_sock *tp = tcp_sk(sk);
1329
1330        /* No URG data to read. */
1331        if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1332            tp->urg_data == TCP_URG_READ)
1333                return -EINVAL; /* Yes this is right ! */
1334
1335        if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1336                return -ENOTCONN;
1337
1338        if (tp->urg_data & TCP_URG_VALID) {
1339                int err = 0;
1340                char c = tp->urg_data;
1341
1342                if (!(flags & MSG_PEEK))
1343                        tp->urg_data = TCP_URG_READ;
1344
1345                /* Read urgent data. */
1346                msg->msg_flags |= MSG_OOB;
1347
1348                if (len > 0) {
1349                        if (!(flags & MSG_TRUNC))
1350                                err = memcpy_to_msg(msg, &c, 1);
1351                        len = 1;
1352                } else
1353                        msg->msg_flags |= MSG_TRUNC;
1354
1355                return err ? -EFAULT : len;
1356        }
1357
1358        if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1359                return 0;
1360
1361        /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1362         * the available implementations agree in this case:
1363         * this call should never block, independent of the
1364         * blocking state of the socket.
1365         * Mike <pall@rz.uni-karlsruhe.de>
1366         */
1367        return -EAGAIN;
1368}
1369
1370static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1371{
1372        struct sk_buff *skb;
1373        int copied = 0, err = 0;
1374
1375        /* XXX -- need to support SO_PEEK_OFF */
1376
1377        skb_queue_walk(&sk->sk_write_queue, skb) {
1378                err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1379                if (err)
1380                        break;
1381
1382                copied += skb->len;
1383        }
1384
1385        return err ?: copied;
1386}
1387
1388/* Clean up the receive buffer for full frames taken by the user,
1389 * then send an ACK if necessary.  COPIED is the number of bytes
1390 * tcp_recvmsg has given to the user so far, it speeds up the
1391 * calculation of whether or not we must ACK for the sake of
1392 * a window update.
1393 */
1394static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1395{
1396        struct tcp_sock *tp = tcp_sk(sk);
1397        bool time_to_ack = false;
1398
1399        struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1400
1401        WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1402             "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1403             tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1404
1405        if (inet_csk_ack_scheduled(sk)) {
1406                const struct inet_connection_sock *icsk = inet_csk(sk);
1407                   /* Delayed ACKs frequently hit locked sockets during bulk
1408                    * receive. */
1409                if (icsk->icsk_ack.blocked ||
1410                    /* Once-per-two-segments ACK was not sent by tcp_input.c */
1411                    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1412                    /*
1413                     * If this read emptied read buffer, we send ACK, if
1414                     * connection is not bidirectional, user drained
1415                     * receive buffer and there was a small segment
1416                     * in queue.
1417                     */
1418                    (copied > 0 &&
1419                     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1420                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1421                       !icsk->icsk_ack.pingpong)) &&
1422                      !atomic_read(&sk->sk_rmem_alloc)))
1423                        time_to_ack = true;
1424        }
1425
1426        /* We send an ACK if we can now advertise a non-zero window
1427         * which has been raised "significantly".
1428         *
1429         * Even if window raised up to infinity, do not send window open ACK
1430         * in states, where we will not receive more. It is useless.
1431         */
1432        if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1433                __u32 rcv_window_now = tcp_receive_window(tp);
1434
1435                /* Optimize, __tcp_select_window() is not cheap. */
1436                if (2*rcv_window_now <= tp->window_clamp) {
1437                        __u32 new_window = __tcp_select_window(sk);
1438
1439                        /* Send ACK now, if this read freed lots of space
1440                         * in our buffer. Certainly, new_window is new window.
1441                         * We can advertise it now, if it is not less than current one.
1442                         * "Lots" means "at least twice" here.
1443                         */
1444                        if (new_window && new_window >= 2 * rcv_window_now)
1445                                time_to_ack = true;
1446                }
1447        }
1448        if (time_to_ack)
1449                tcp_send_ack(sk);
1450}
1451
1452static void tcp_prequeue_process(struct sock *sk)
1453{
1454        struct sk_buff *skb;
1455        struct tcp_sock *tp = tcp_sk(sk);
1456
1457        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1458
1459        while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1460                sk_backlog_rcv(sk, skb);
1461
1462        /* Clear memory counter. */
1463        tp->ucopy.memory = 0;
1464}
1465
1466static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1467{
1468        struct sk_buff *skb;
1469        u32 offset;
1470
1471        while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1472                offset = seq - TCP_SKB_CB(skb)->seq;
1473                if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1474                        pr_err_once("%s: found a SYN, please report !\n", __func__);
1475                        offset--;
1476                }
1477                if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1478                        *off = offset;
1479                        return skb;
1480                }
1481                /* This looks weird, but this can happen if TCP collapsing
1482                 * splitted a fat GRO packet, while we released socket lock
1483                 * in skb_splice_bits()
1484                 */
1485                sk_eat_skb(sk, skb);
1486        }
1487        return NULL;
1488}
1489
1490/*
1491 * This routine provides an alternative to tcp_recvmsg() for routines
1492 * that would like to handle copying from skbuffs directly in 'sendfile'
1493 * fashion.
1494 * Note:
1495 *      - It is assumed that the socket was locked by the caller.
1496 *      - The routine does not block.
1497 *      - At present, there is no support for reading OOB data
1498 *        or for 'peeking' the socket using this routine
1499 *        (although both would be easy to implement).
1500 */
1501int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1502                  sk_read_actor_t recv_actor)
1503{
1504        struct sk_buff *skb;
1505        struct tcp_sock *tp = tcp_sk(sk);
1506        u32 seq = tp->copied_seq;
1507        u32 offset;
1508        int copied = 0;
1509
1510        if (sk->sk_state == TCP_LISTEN)
1511                return -ENOTCONN;
1512        while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1513                if (offset < skb->len) {
1514                        int used;
1515                        size_t len;
1516
1517                        len = skb->len - offset;
1518                        /* Stop reading if we hit a patch of urgent data */
1519                        if (tp->urg_data) {
1520                                u32 urg_offset = tp->urg_seq - seq;
1521                                if (urg_offset < len)
1522                                        len = urg_offset;
1523                                if (!len)
1524                                        break;
1525                        }
1526                        used = recv_actor(desc, skb, offset, len);
1527                        if (used <= 0) {
1528                                if (!copied)
1529                                        copied = used;
1530                                break;
1531                        } else if (used <= len) {
1532                                seq += used;
1533                                copied += used;
1534                                offset += used;
1535                        }
1536                        /* If recv_actor drops the lock (e.g. TCP splice
1537                         * receive) the skb pointer might be invalid when
1538                         * getting here: tcp_collapse might have deleted it
1539                         * while aggregating skbs from the socket queue.
1540                         */
1541                        skb = tcp_recv_skb(sk, seq - 1, &offset);
1542                        if (!skb)
1543                                break;
1544                        /* TCP coalescing might have appended data to the skb.
1545                         * Try to splice more frags
1546                         */
1547                        if (offset + 1 != skb->len)
1548                                continue;
1549                }
1550                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1551                        sk_eat_skb(sk, skb);
1552                        ++seq;
1553                        break;
1554                }
1555                sk_eat_skb(sk, skb);
1556                if (!desc->count)
1557                        break;
1558                tp->copied_seq = seq;
1559        }
1560        tp->copied_seq = seq;
1561
1562        tcp_rcv_space_adjust(sk);
1563
1564        /* Clean up data we have read: This will do ACK frames. */
1565        if (copied > 0) {
1566                tcp_recv_skb(sk, seq, &offset);
1567                tcp_cleanup_rbuf(sk, copied);
1568        }
1569        return copied;
1570}
1571EXPORT_SYMBOL(tcp_read_sock);
1572
1573/*
1574 *      This routine copies from a sock struct into the user buffer.
1575 *
1576 *      Technical note: in 2.3 we work on _locked_ socket, so that
1577 *      tricks with *seq access order and skb->users are not required.
1578 *      Probably, code can be easily improved even more.
1579 */
1580
1581int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
1582                int flags, int *addr_len)
1583{
1584        struct tcp_sock *tp = tcp_sk(sk);
1585        int copied = 0;
1586        u32 peek_seq;
1587        u32 *seq;
1588        unsigned long used;
1589        int err;
1590        int target;             /* Read at least this many bytes */
1591        long timeo;
1592        struct task_struct *user_recv = NULL;
1593        struct sk_buff *skb, *last;
1594        u32 urg_hole = 0;
1595
1596        if (unlikely(flags & MSG_ERRQUEUE))
1597                return inet_recv_error(sk, msg, len, addr_len);
1598
1599        if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1600            (sk->sk_state == TCP_ESTABLISHED))
1601                sk_busy_loop(sk, nonblock);
1602
1603        lock_sock(sk);
1604
1605        err = -ENOTCONN;
1606        if (sk->sk_state == TCP_LISTEN)
1607                goto out;
1608
1609        timeo = sock_rcvtimeo(sk, nonblock);
1610
1611        /* Urgent data needs to be handled specially. */
1612        if (flags & MSG_OOB)
1613                goto recv_urg;
1614
1615        if (unlikely(tp->repair)) {
1616                err = -EPERM;
1617                if (!(flags & MSG_PEEK))
1618                        goto out;
1619
1620                if (tp->repair_queue == TCP_SEND_QUEUE)
1621                        goto recv_sndq;
1622
1623                err = -EINVAL;
1624                if (tp->repair_queue == TCP_NO_QUEUE)
1625                        goto out;
1626
1627                /* 'common' recv queue MSG_PEEK-ing */
1628        }
1629
1630        seq = &tp->copied_seq;
1631        if (flags & MSG_PEEK) {
1632                peek_seq = tp->copied_seq;
1633                seq = &peek_seq;
1634        }
1635
1636        target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1637
1638        do {
1639                u32 offset;
1640
1641                /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1642                if (tp->urg_data && tp->urg_seq == *seq) {
1643                        if (copied)
1644                                break;
1645                        if (signal_pending(current)) {
1646                                copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1647                                break;
1648                        }
1649                }
1650
1651                /* Next get a buffer. */
1652
1653                last = skb_peek_tail(&sk->sk_receive_queue);
1654                skb_queue_walk(&sk->sk_receive_queue, skb) {
1655                        last = skb;
1656                        /* Now that we have two receive queues this
1657                         * shouldn't happen.
1658                         */
1659                        if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1660                                 "recvmsg bug: copied %X seq %X rcvnxt %X fl %X\n",
1661                                 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1662                                 flags))
1663                                break;
1664
1665                        offset = *seq - TCP_SKB_CB(skb)->seq;
1666                        if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1667                                pr_err_once("%s: found a SYN, please report !\n", __func__);
1668                                offset--;
1669                        }
1670                        if (offset < skb->len)
1671                                goto found_ok_skb;
1672                        if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1673                                goto found_fin_ok;
1674                        WARN(!(flags & MSG_PEEK),
1675                             "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
1676                             *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1677                }
1678
1679                /* Well, if we have backlog, try to process it now yet. */
1680
1681                if (copied >= target && !sk->sk_backlog.tail)
1682                        break;
1683
1684                if (copied) {
1685                        if (sk->sk_err ||
1686                            sk->sk_state == TCP_CLOSE ||
1687                            (sk->sk_shutdown & RCV_SHUTDOWN) ||
1688                            !timeo ||
1689                            signal_pending(current))
1690                                break;
1691                } else {
1692                        if (sock_flag(sk, SOCK_DONE))
1693                                break;
1694
1695                        if (sk->sk_err) {
1696                                copied = sock_error(sk);
1697                                break;
1698                        }
1699
1700                        if (sk->sk_shutdown & RCV_SHUTDOWN)
1701                                break;
1702
1703                        if (sk->sk_state == TCP_CLOSE) {
1704                                if (!sock_flag(sk, SOCK_DONE)) {
1705                                        /* This occurs when user tries to read
1706                                         * from never connected socket.
1707                                         */
1708                                        copied = -ENOTCONN;
1709                                        break;
1710                                }
1711                                break;
1712                        }
1713
1714                        if (!timeo) {
1715                                copied = -EAGAIN;
1716                                break;
1717                        }
1718
1719                        if (signal_pending(current)) {
1720                                copied = sock_intr_errno(timeo);
1721                                break;
1722                        }
1723                }
1724
1725                tcp_cleanup_rbuf(sk, copied);
1726
1727                if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1728                        /* Install new reader */
1729                        if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1730                                user_recv = current;
1731                                tp->ucopy.task = user_recv;
1732                                tp->ucopy.msg = msg;
1733                        }
1734
1735                        tp->ucopy.len = len;
1736
1737                        WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1738                                !(flags & (MSG_PEEK | MSG_TRUNC)));
1739
1740                        /* Ugly... If prequeue is not empty, we have to
1741                         * process it before releasing socket, otherwise
1742                         * order will be broken at second iteration.
1743                         * More elegant solution is required!!!
1744                         *
1745                         * Look: we have the following (pseudo)queues:
1746                         *
1747                         * 1. packets in flight
1748                         * 2. backlog
1749                         * 3. prequeue
1750                         * 4. receive_queue
1751                         *
1752                         * Each queue can be processed only if the next ones
1753                         * are empty. At this point we have empty receive_queue.
1754                         * But prequeue _can_ be not empty after 2nd iteration,
1755                         * when we jumped to start of loop because backlog
1756                         * processing added something to receive_queue.
1757                         * We cannot release_sock(), because backlog contains
1758                         * packets arrived _after_ prequeued ones.
1759                         *
1760                         * Shortly, algorithm is clear --- to process all
1761                         * the queues in order. We could make it more directly,
1762                         * requeueing packets from backlog to prequeue, if
1763                         * is not empty. It is more elegant, but eats cycles,
1764                         * unfortunately.
1765                         */
1766                        if (!skb_queue_empty(&tp->ucopy.prequeue))
1767                                goto do_prequeue;
1768
1769                        /* __ Set realtime policy in scheduler __ */
1770                }
1771
1772                if (copied >= target) {
1773                        /* Do not sleep, just process backlog. */
1774                        release_sock(sk);
1775                        lock_sock(sk);
1776                } else {
1777                        sk_wait_data(sk, &timeo, last);
1778                }
1779
1780                if (user_recv) {
1781                        int chunk;
1782
1783                        /* __ Restore normal policy in scheduler __ */
1784
1785                        chunk = len - tp->ucopy.len;
1786                        if (chunk != 0) {
1787                                NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1788                                len -= chunk;
1789                                copied += chunk;
1790                        }
1791
1792                        if (tp->rcv_nxt == tp->copied_seq &&
1793                            !skb_queue_empty(&tp->ucopy.prequeue)) {
1794do_prequeue:
1795                                tcp_prequeue_process(sk);
1796
1797                                chunk = len - tp->ucopy.len;
1798                                if (chunk != 0) {
1799                                        NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1800                                        len -= chunk;
1801                                        copied += chunk;
1802                                }
1803                        }
1804                }
1805                if ((flags & MSG_PEEK) &&
1806                    (peek_seq - copied - urg_hole != tp->copied_seq)) {
1807                        net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
1808                                            current->comm,
1809                                            task_pid_nr(current));
1810                        peek_seq = tp->copied_seq;
1811                }
1812                continue;
1813
1814        found_ok_skb:
1815                /* Ok so how much can we use? */
1816                used = skb->len - offset;
1817                if (len < used)
1818                        used = len;
1819
1820                /* Do we have urgent data here? */
1821                if (tp->urg_data) {
1822                        u32 urg_offset = tp->urg_seq - *seq;
1823                        if (urg_offset < used) {
1824                                if (!urg_offset) {
1825                                        if (!sock_flag(sk, SOCK_URGINLINE)) {
1826                                                ++*seq;
1827                                                urg_hole++;
1828                                                offset++;
1829                                                used--;
1830                                                if (!used)
1831                                                        goto skip_copy;
1832                                        }
1833                                } else
1834                                        used = urg_offset;
1835                        }
1836                }
1837
1838                if (!(flags & MSG_TRUNC)) {
1839                        err = skb_copy_datagram_msg(skb, offset, msg, used);
1840                        if (err) {
1841                                /* Exception. Bailout! */
1842                                if (!copied)
1843                                        copied = -EFAULT;
1844                                break;
1845                        }
1846                }
1847
1848                *seq += used;
1849                copied += used;
1850                len -= used;
1851
1852                tcp_rcv_space_adjust(sk);
1853
1854skip_copy:
1855                if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1856                        tp->urg_data = 0;
1857                        tcp_fast_path_check(sk);
1858                }
1859                if (used + offset < skb->len)
1860                        continue;
1861
1862                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1863                        goto found_fin_ok;
1864                if (!(flags & MSG_PEEK))
1865                        sk_eat_skb(sk, skb);
1866                continue;
1867
1868        found_fin_ok:
1869                /* Process the FIN. */
1870                ++*seq;
1871                if (!(flags & MSG_PEEK))
1872                        sk_eat_skb(sk, skb);
1873                break;
1874        } while (len > 0);
1875
1876        if (user_recv) {
1877                if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1878                        int chunk;
1879
1880                        tp->ucopy.len = copied > 0 ? len : 0;
1881
1882                        tcp_prequeue_process(sk);
1883
1884                        if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1885                                NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1886                                len -= chunk;
1887                                copied += chunk;
1888                        }
1889                }
1890
1891                tp->ucopy.task = NULL;
1892                tp->ucopy.len = 0;
1893        }
1894
1895        /* According to UNIX98, msg_name/msg_namelen are ignored
1896         * on connected socket. I was just happy when found this 8) --ANK
1897         */
1898
1899        /* Clean up data we have read: This will do ACK frames. */
1900        tcp_cleanup_rbuf(sk, copied);
1901
1902        release_sock(sk);
1903        return copied;
1904
1905out:
1906        release_sock(sk);
1907        return err;
1908
1909recv_urg:
1910        err = tcp_recv_urg(sk, msg, len, flags);
1911        goto out;
1912
1913recv_sndq:
1914        err = tcp_peek_sndq(sk, msg, len);
1915        goto out;
1916}
1917EXPORT_SYMBOL(tcp_recvmsg);
1918
1919void tcp_set_state(struct sock *sk, int state)
1920{
1921        int oldstate = sk->sk_state;
1922
1923        switch (state) {
1924        case TCP_ESTABLISHED:
1925                if (oldstate != TCP_ESTABLISHED)
1926                        TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1927                break;
1928
1929        case TCP_CLOSE:
1930                if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1931                        TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1932
1933                sk->sk_prot->unhash(sk);
1934                if (inet_csk(sk)->icsk_bind_hash &&
1935                    !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1936                        inet_put_port(sk);
1937                /* fall through */
1938        default:
1939                if (oldstate == TCP_ESTABLISHED)
1940                        TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1941        }
1942
1943        /* Change state AFTER socket is unhashed to avoid closed
1944         * socket sitting in hash tables.
1945         */
1946        sk_state_store(sk, state);
1947
1948#ifdef STATE_TRACE
1949        SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1950#endif
1951}
1952EXPORT_SYMBOL_GPL(tcp_set_state);
1953
1954/*
1955 *      State processing on a close. This implements the state shift for
1956 *      sending our FIN frame. Note that we only send a FIN for some
1957 *      states. A shutdown() may have already sent the FIN, or we may be
1958 *      closed.
1959 */
1960
1961static const unsigned char new_state[16] = {
1962  /* current state:        new state:      action:      */
1963  [0 /* (Invalid) */]   = TCP_CLOSE,
1964  [TCP_ESTABLISHED]     = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1965  [TCP_SYN_SENT]        = TCP_CLOSE,
1966  [TCP_SYN_RECV]        = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1967  [TCP_FIN_WAIT1]       = TCP_FIN_WAIT1,
1968  [TCP_FIN_WAIT2]       = TCP_FIN_WAIT2,
1969  [TCP_TIME_WAIT]       = TCP_CLOSE,
1970  [TCP_CLOSE]           = TCP_CLOSE,
1971  [TCP_CLOSE_WAIT]      = TCP_LAST_ACK  | TCP_ACTION_FIN,
1972  [TCP_LAST_ACK]        = TCP_LAST_ACK,
1973  [TCP_LISTEN]          = TCP_CLOSE,
1974  [TCP_CLOSING]         = TCP_CLOSING,
1975  [TCP_NEW_SYN_RECV]    = TCP_CLOSE,    /* should not happen ! */
1976};
1977
1978static int tcp_close_state(struct sock *sk)
1979{
1980        int next = (int)new_state[sk->sk_state];
1981        int ns = next & TCP_STATE_MASK;
1982
1983        tcp_set_state(sk, ns);
1984
1985        return next & TCP_ACTION_FIN;
1986}
1987
1988/*
1989 *      Shutdown the sending side of a connection. Much like close except
1990 *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1991 */
1992
1993void tcp_shutdown(struct sock *sk, int how)
1994{
1995        /*      We need to grab some memory, and put together a FIN,
1996         *      and then put it into the queue to be sent.
1997         *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1998         */
1999        if (!(how & SEND_SHUTDOWN))
2000                return;
2001
2002        /* If we've already sent a FIN, or it's a closed state, skip this. */
2003        if ((1 << sk->sk_state) &
2004            (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2005             TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2006                /* Clear out any half completed packets.  FIN if needed. */
2007                if (tcp_close_state(sk))
2008                        tcp_send_fin(sk);
2009        }
2010}
2011EXPORT_SYMBOL(tcp_shutdown);
2012
2013bool tcp_check_oom(struct sock *sk, int shift)
2014{
2015        bool too_many_orphans, out_of_socket_memory;
2016
2017        too_many_orphans = tcp_too_many_orphans(sk, shift);
2018        out_of_socket_memory = tcp_out_of_memory(sk);
2019
2020        if (too_many_orphans)
2021                net_info_ratelimited("too many orphaned sockets\n");
2022        if (out_of_socket_memory)
2023                net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2024        return too_many_orphans || out_of_socket_memory;
2025}
2026
2027void tcp_close(struct sock *sk, long timeout)
2028{
2029        struct sk_buff *skb;
2030        int data_was_unread = 0;
2031        int state;
2032
2033        lock_sock(sk);
2034        sk->sk_shutdown = SHUTDOWN_MASK;
2035
2036        if (sk->sk_state == TCP_LISTEN) {
2037                tcp_set_state(sk, TCP_CLOSE);
2038
2039                /* Special case. */
2040                inet_csk_listen_stop(sk);
2041
2042                goto adjudge_to_death;
2043        }
2044
2045        /*  We need to flush the recv. buffs.  We do this only on the
2046         *  descriptor close, not protocol-sourced closes, because the
2047         *  reader process may not have drained the data yet!
2048         */
2049        while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2050                u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2051
2052                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2053                        len--;
2054                data_was_unread += len;
2055                __kfree_skb(skb);
2056        }
2057
2058        sk_mem_reclaim(sk);
2059
2060        /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2061        if (sk->sk_state == TCP_CLOSE)
2062                goto adjudge_to_death;
2063
2064        /* As outlined in RFC 2525, section 2.17, we send a RST here because
2065         * data was lost. To witness the awful effects of the old behavior of
2066         * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2067         * GET in an FTP client, suspend the process, wait for the client to
2068         * advertise a zero window, then kill -9 the FTP client, wheee...
2069         * Note: timeout is always zero in such a case.
2070         */
2071        if (unlikely(tcp_sk(sk)->repair)) {
2072                sk->sk_prot->disconnect(sk, 0);
2073        } else if (data_was_unread) {
2074                /* Unread data was tossed, zap the connection. */
2075                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2076                tcp_set_state(sk, TCP_CLOSE);
2077                tcp_send_active_reset(sk, sk->sk_allocation);
2078        } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2079                /* Check zero linger _after_ checking for unread data. */
2080                sk->sk_prot->disconnect(sk, 0);
2081                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2082        } else if (tcp_close_state(sk)) {
2083                /* We FIN if the application ate all the data before
2084                 * zapping the connection.
2085                 */
2086
2087                /* RED-PEN. Formally speaking, we have broken TCP state
2088                 * machine. State transitions:
2089                 *
2090                 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2091                 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
2092                 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2093                 *
2094                 * are legal only when FIN has been sent (i.e. in window),
2095                 * rather than queued out of window. Purists blame.
2096                 *
2097                 * F.e. "RFC state" is ESTABLISHED,
2098                 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2099                 *
2100                 * The visible declinations are that sometimes
2101                 * we enter time-wait state, when it is not required really
2102                 * (harmless), do not send active resets, when they are
2103                 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2104                 * they look as CLOSING or LAST_ACK for Linux)
2105                 * Probably, I missed some more holelets.
2106                 *                                              --ANK
2107                 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2108                 * in a single packet! (May consider it later but will
2109                 * probably need API support or TCP_CORK SYN-ACK until
2110                 * data is written and socket is closed.)
2111                 */
2112                tcp_send_fin(sk);
2113        }
2114
2115        sk_stream_wait_close(sk, timeout);
2116
2117adjudge_to_death:
2118        state = sk->sk_state;
2119        sock_hold(sk);
2120        sock_orphan(sk);
2121
2122        /* It is the last release_sock in its life. It will remove backlog. */
2123        release_sock(sk);
2124
2125
2126        /* Now socket is owned by kernel and we acquire BH lock
2127           to finish close. No need to check for user refs.
2128         */
2129        local_bh_disable();
2130        bh_lock_sock(sk);
2131        WARN_ON(sock_owned_by_user(sk));
2132
2133        percpu_counter_inc(sk->sk_prot->orphan_count);
2134
2135        /* Have we already been destroyed by a softirq or backlog? */
2136        if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2137                goto out;
2138
2139        /*      This is a (useful) BSD violating of the RFC. There is a
2140         *      problem with TCP as specified in that the other end could
2141         *      keep a socket open forever with no application left this end.
2142         *      We use a 1 minute timeout (about the same as BSD) then kill
2143         *      our end. If they send after that then tough - BUT: long enough
2144         *      that we won't make the old 4*rto = almost no time - whoops
2145         *      reset mistake.
2146         *
2147         *      Nope, it was not mistake. It is really desired behaviour
2148         *      f.e. on http servers, when such sockets are useless, but
2149         *      consume significant resources. Let's do it with special
2150         *      linger2 option.                                 --ANK
2151         */
2152
2153        if (sk->sk_state == TCP_FIN_WAIT2) {
2154                struct tcp_sock *tp = tcp_sk(sk);
2155                if (tp->linger2 < 0) {
2156                        tcp_set_state(sk, TCP_CLOSE);
2157                        tcp_send_active_reset(sk, GFP_ATOMIC);
2158                        __NET_INC_STATS(sock_net(sk),
2159                                        LINUX_MIB_TCPABORTONLINGER);
2160                } else {
2161                        const int tmo = tcp_fin_time(sk);
2162
2163                        if (tmo > TCP_TIMEWAIT_LEN) {
2164                                inet_csk_reset_keepalive_timer(sk,
2165                                                tmo - TCP_TIMEWAIT_LEN);
2166                        } else {
2167                                tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2168                                goto out;
2169                        }
2170                }
2171        }
2172        if (sk->sk_state != TCP_CLOSE) {
2173                sk_mem_reclaim(sk);
2174                if (tcp_check_oom(sk, 0)) {
2175                        tcp_set_state(sk, TCP_CLOSE);
2176                        tcp_send_active_reset(sk, GFP_ATOMIC);
2177                        __NET_INC_STATS(sock_net(sk),
2178                                        LINUX_MIB_TCPABORTONMEMORY);
2179                }
2180        }
2181
2182        if (sk->sk_state == TCP_CLOSE) {
2183                struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
2184                /* We could get here with a non-NULL req if the socket is
2185                 * aborted (e.g., closed with unread data) before 3WHS
2186                 * finishes.
2187                 */
2188                if (req)
2189                        reqsk_fastopen_remove(sk, req, false);
2190                inet_csk_destroy_sock(sk);
2191        }
2192        /* Otherwise, socket is reprieved until protocol close. */
2193
2194out:
2195        bh_unlock_sock(sk);
2196        local_bh_enable();
2197        sock_put(sk);
2198}
2199EXPORT_SYMBOL(tcp_close);
2200
2201/* These states need RST on ABORT according to RFC793 */
2202
2203static inline bool tcp_need_reset(int state)
2204{
2205        return (1 << state) &
2206               (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2207                TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2208}
2209
2210int tcp_disconnect(struct sock *sk, int flags)
2211{
2212        struct inet_sock *inet = inet_sk(sk);
2213        struct inet_connection_sock *icsk = inet_csk(sk);
2214        struct tcp_sock *tp = tcp_sk(sk);
2215        int err = 0;
2216        int old_state = sk->sk_state;
2217
2218        if (old_state != TCP_CLOSE)
2219                tcp_set_state(sk, TCP_CLOSE);
2220
2221        /* ABORT function of RFC793 */
2222        if (old_state == TCP_LISTEN) {
2223                inet_csk_listen_stop(sk);
2224        } else if (unlikely(tp->repair)) {
2225                sk->sk_err = ECONNABORTED;
2226        } else if (tcp_need_reset(old_state) ||
2227                   (tp->snd_nxt != tp->write_seq &&
2228                    (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2229                /* The last check adjusts for discrepancy of Linux wrt. RFC
2230                 * states
2231                 */
2232                tcp_send_active_reset(sk, gfp_any());
2233                sk->sk_err = ECONNRESET;
2234        } else if (old_state == TCP_SYN_SENT)
2235                sk->sk_err = ECONNRESET;
2236
2237        tcp_clear_xmit_timers(sk);
2238        __skb_queue_purge(&sk->sk_receive_queue);
2239        tcp_write_queue_purge(sk);
2240        __skb_queue_purge(&tp->out_of_order_queue);
2241
2242        inet->inet_dport = 0;
2243
2244        if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2245                inet_reset_saddr(sk);
2246
2247        sk->sk_shutdown = 0;
2248        sock_reset_flag(sk, SOCK_DONE);
2249        tp->srtt_us = 0;
2250        tp->write_seq += tp->max_window + 2;
2251        if (tp->write_seq == 0)
2252                tp->write_seq = 1;
2253        icsk->icsk_backoff = 0;
2254        tp->snd_cwnd = 2;
2255        icsk->icsk_probes_out = 0;
2256        tp->packets_out = 0;
2257        tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2258        tp->snd_cwnd_cnt = 0;
2259        tp->window_clamp = 0;
2260        tcp_set_ca_state(sk, TCP_CA_Open);
2261        tcp_clear_retrans(tp);
2262        inet_csk_delack_init(sk);
2263        tcp_init_send_head(sk);
2264        memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2265        __sk_dst_reset(sk);
2266
2267        WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2268
2269        sk->sk_error_report(sk);
2270        return err;
2271}
2272EXPORT_SYMBOL(tcp_disconnect);
2273
2274static inline bool tcp_can_repair_sock(const struct sock *sk)
2275{
2276        return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2277                ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_ESTABLISHED));
2278}
2279
2280static int tcp_repair_options_est(struct tcp_sock *tp,
2281                struct tcp_repair_opt __user *optbuf, unsigned int len)
2282{
2283        struct tcp_repair_opt opt;
2284
2285        while (len >= sizeof(opt)) {
2286                if (copy_from_user(&opt, optbuf, sizeof(opt)))
2287                        return -EFAULT;
2288
2289                optbuf++;
2290                len -= sizeof(opt);
2291
2292                switch (opt.opt_code) {
2293                case TCPOPT_MSS:
2294                        tp->rx_opt.mss_clamp = opt.opt_val;
2295                        break;
2296                case TCPOPT_WINDOW:
2297                        {
2298                                u16 snd_wscale = opt.opt_val & 0xFFFF;
2299                                u16 rcv_wscale = opt.opt_val >> 16;
2300
2301                                if (snd_wscale > 14 || rcv_wscale > 14)
2302                                        return -EFBIG;
2303
2304                                tp->rx_opt.snd_wscale = snd_wscale;
2305                                tp->rx_opt.rcv_wscale = rcv_wscale;
2306                                tp->rx_opt.wscale_ok = 1;
2307                        }
2308                        break;
2309                case TCPOPT_SACK_PERM:
2310                        if (opt.opt_val != 0)
2311                                return -EINVAL;
2312
2313                        tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2314                        if (sysctl_tcp_fack)
2315                                tcp_enable_fack(tp);
2316                        break;
2317                case TCPOPT_TIMESTAMP:
2318                        if (opt.opt_val != 0)
2319                                return -EINVAL;
2320
2321                        tp->rx_opt.tstamp_ok = 1;
2322                        break;
2323                }
2324        }
2325
2326        return 0;
2327}
2328
2329/*
2330 *      Socket option code for TCP.
2331 */
2332static int do_tcp_setsockopt(struct sock *sk, int level,
2333                int optname, char __user *optval, unsigned int optlen)
2334{
2335        struct tcp_sock *tp = tcp_sk(sk);
2336        struct inet_connection_sock *icsk = inet_csk(sk);
2337        struct net *net = sock_net(sk);
2338        int val;
2339        int err = 0;
2340
2341        /* These are data/string values, all the others are ints */
2342        switch (optname) {
2343        case TCP_CONGESTION: {
2344                char name[TCP_CA_NAME_MAX];
2345
2346                if (optlen < 1)
2347                        return -EINVAL;
2348
2349                val = strncpy_from_user(name, optval,
2350                                        min_t(long, TCP_CA_NAME_MAX-1, optlen));
2351                if (val < 0)
2352                        return -EFAULT;
2353                name[val] = 0;
2354
2355                lock_sock(sk);
2356                err = tcp_set_congestion_control(sk, name);
2357                release_sock(sk);
2358                return err;
2359        }
2360        default:
2361                /* fallthru */
2362                break;
2363        }
2364
2365        if (optlen < sizeof(int))
2366                return -EINVAL;
2367
2368        if (get_user(val, (int __user *)optval))
2369                return -EFAULT;
2370
2371        lock_sock(sk);
2372
2373        switch (optname) {
2374        case TCP_MAXSEG:
2375                /* Values greater than interface MTU won't take effect. However
2376                 * at the point when this call is done we typically don't yet
2377                 * know which interface is going to be used */
2378                if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2379                        err = -EINVAL;
2380                        break;
2381                }
2382                tp->rx_opt.user_mss = val;
2383                break;
2384
2385        case TCP_NODELAY:
2386                if (val) {
2387                        /* TCP_NODELAY is weaker than TCP_CORK, so that
2388                         * this option on corked socket is remembered, but
2389                         * it is not activated until cork is cleared.
2390                         *
2391                         * However, when TCP_NODELAY is set we make
2392                         * an explicit push, which overrides even TCP_CORK
2393                         * for currently queued segments.
2394                         */
2395                        tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2396                        tcp_push_pending_frames(sk);
2397                } else {
2398                        tp->nonagle &= ~TCP_NAGLE_OFF;
2399                }
2400                break;
2401
2402        case TCP_THIN_LINEAR_TIMEOUTS:
2403                if (val < 0 || val > 1)
2404                        err = -EINVAL;
2405                else
2406                        tp->thin_lto = val;
2407                break;
2408
2409        case TCP_THIN_DUPACK:
2410                if (val < 0 || val > 1)
2411                        err = -EINVAL;
2412                else {
2413                        tp->thin_dupack = val;
2414                        if (tp->thin_dupack)
2415                                tcp_disable_early_retrans(tp);
2416                }
2417                break;
2418
2419        case TCP_REPAIR:
2420                if (!tcp_can_repair_sock(sk))
2421                        err = -EPERM;
2422                else if (val == 1) {
2423                        tp->repair = 1;
2424                        sk->sk_reuse = SK_FORCE_REUSE;
2425                        tp->repair_queue = TCP_NO_QUEUE;
2426                } else if (val == 0) {
2427                        tp->repair = 0;
2428                        sk->sk_reuse = SK_NO_REUSE;
2429                        tcp_send_window_probe(sk);
2430                } else
2431                        err = -EINVAL;
2432
2433                break;
2434
2435        case TCP_REPAIR_QUEUE:
2436                if (!tp->repair)
2437                        err = -EPERM;
2438                else if (val < TCP_QUEUES_NR)
2439                        tp->repair_queue = val;
2440                else
2441                        err = -EINVAL;
2442                break;
2443
2444        case TCP_QUEUE_SEQ:
2445                if (sk->sk_state != TCP_CLOSE)
2446                        err = -EPERM;
2447                else if (tp->repair_queue == TCP_SEND_QUEUE)
2448                        tp->write_seq = val;
2449                else if (tp->repair_queue == TCP_RECV_QUEUE)
2450                        tp->rcv_nxt = val;
2451                else
2452                        err = -EINVAL;
2453                break;
2454
2455        case TCP_REPAIR_OPTIONS:
2456                if (!tp->repair)
2457                        err = -EINVAL;
2458                else if (sk->sk_state == TCP_ESTABLISHED)
2459                        err = tcp_repair_options_est(tp,
2460                                        (struct tcp_repair_opt __user *)optval,
2461                                        optlen);
2462                else
2463                        err = -EPERM;
2464                break;
2465
2466        case TCP_CORK:
2467                /* When set indicates to always queue non-full frames.
2468                 * Later the user clears this option and we transmit
2469                 * any pending partial frames in the queue.  This is
2470                 * meant to be used alongside sendfile() to get properly
2471                 * filled frames when the user (for example) must write
2472                 * out headers with a write() call first and then use
2473                 * sendfile to send out the data parts.
2474                 *
2475                 * TCP_CORK can be set together with TCP_NODELAY and it is
2476                 * stronger than TCP_NODELAY.
2477                 */
2478                if (val) {
2479                        tp->nonagle |= TCP_NAGLE_CORK;
2480                } else {
2481                        tp->nonagle &= ~TCP_NAGLE_CORK;
2482                        if (tp->nonagle&TCP_NAGLE_OFF)
2483                                tp->nonagle |= TCP_NAGLE_PUSH;
2484                        tcp_push_pending_frames(sk);
2485                }
2486                break;
2487
2488        case TCP_KEEPIDLE:
2489                if (val < 1 || val > MAX_TCP_KEEPIDLE)
2490                        err = -EINVAL;
2491                else {
2492                        tp->keepalive_time = val * HZ;
2493                        if (sock_flag(sk, SOCK_KEEPOPEN) &&
2494                            !((1 << sk->sk_state) &
2495                              (TCPF_CLOSE | TCPF_LISTEN))) {
2496                                u32 elapsed = keepalive_time_elapsed(tp);
2497                                if (tp->keepalive_time > elapsed)
2498                                        elapsed = tp->keepalive_time - elapsed;
2499                                else
2500                                        elapsed = 0;
2501                                inet_csk_reset_keepalive_timer(sk, elapsed);
2502                        }
2503                }
2504                break;
2505        case TCP_KEEPINTVL:
2506                if (val < 1 || val > MAX_TCP_KEEPINTVL)
2507                        err = -EINVAL;
2508                else
2509                        tp->keepalive_intvl = val * HZ;
2510                break;
2511        case TCP_KEEPCNT:
2512                if (val < 1 || val > MAX_TCP_KEEPCNT)
2513                        err = -EINVAL;
2514                else
2515                        tp->keepalive_probes = val;
2516                break;
2517        case TCP_SYNCNT:
2518                if (val < 1 || val > MAX_TCP_SYNCNT)
2519                        err = -EINVAL;
2520                else
2521                        icsk->icsk_syn_retries = val;
2522                break;
2523
2524        case TCP_SAVE_SYN:
2525                if (val < 0 || val > 1)
2526                        err = -EINVAL;
2527                else
2528                        tp->save_syn = val;
2529                break;
2530
2531        case TCP_LINGER2:
2532                if (val < 0)
2533                        tp->linger2 = -1;
2534                else if (val > net->ipv4.sysctl_tcp_fin_timeout / HZ)
2535                        tp->linger2 = 0;
2536                else
2537                        tp->linger2 = val * HZ;
2538                break;
2539
2540        case TCP_DEFER_ACCEPT:
2541                /* Translate value in seconds to number of retransmits */
2542                icsk->icsk_accept_queue.rskq_defer_accept =
2543                        secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2544                                        TCP_RTO_MAX / HZ);
2545                break;
2546
2547        case TCP_WINDOW_CLAMP:
2548                if (!val) {
2549                        if (sk->sk_state != TCP_CLOSE) {
2550                                err = -EINVAL;
2551                                break;
2552                        }
2553                        tp->window_clamp = 0;
2554                } else
2555                        tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2556                                                SOCK_MIN_RCVBUF / 2 : val;
2557                break;
2558
2559        case TCP_QUICKACK:
2560                if (!val) {
2561                        icsk->icsk_ack.pingpong = 1;
2562                } else {
2563                        icsk->icsk_ack.pingpong = 0;
2564                        if ((1 << sk->sk_state) &
2565                            (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2566                            inet_csk_ack_scheduled(sk)) {
2567                                icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2568                                tcp_cleanup_rbuf(sk, 1);
2569                                if (!(val & 1))
2570                                        icsk->icsk_ack.pingpong = 1;
2571                        }
2572                }
2573                break;
2574
2575#ifdef CONFIG_TCP_MD5SIG
2576        case TCP_MD5SIG:
2577                /* Read the IP->Key mappings from userspace */
2578                err = tp->af_specific->md5_parse(sk, optval, optlen);
2579                break;
2580#endif
2581        case TCP_USER_TIMEOUT:
2582                /* Cap the max time in ms TCP will retry or probe the window
2583                 * before giving up and aborting (ETIMEDOUT) a connection.
2584                 */
2585                if (val < 0)
2586                        err = -EINVAL;
2587                else
2588                        icsk->icsk_user_timeout = msecs_to_jiffies(val);
2589                break;
2590
2591        case TCP_FASTOPEN:
2592                if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
2593                    TCPF_LISTEN))) {
2594                        tcp_fastopen_init_key_once(true);
2595
2596                        fastopen_queue_tune(sk, val);
2597                } else {
2598                        err = -EINVAL;
2599                }
2600                break;
2601        case TCP_TIMESTAMP:
2602                if (!tp->repair)
2603                        err = -EPERM;
2604                else
2605                        tp->tsoffset = val - tcp_time_stamp;
2606                break;
2607        case TCP_NOTSENT_LOWAT:
2608                tp->notsent_lowat = val;
2609                sk->sk_write_space(sk);
2610                break;
2611        default:
2612                err = -ENOPROTOOPT;
2613                break;
2614        }
2615
2616        release_sock(sk);
2617        return err;
2618}
2619
2620int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2621                   unsigned int optlen)
2622{
2623        const struct inet_connection_sock *icsk = inet_csk(sk);
2624
2625        if (level != SOL_TCP)
2626                return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2627                                                     optval, optlen);
2628        return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2629}
2630EXPORT_SYMBOL(tcp_setsockopt);
2631
2632#ifdef CONFIG_COMPAT
2633int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2634                          char __user *optval, unsigned int optlen)
2635{
2636        if (level != SOL_TCP)
2637                return inet_csk_compat_setsockopt(sk, level, optname,
2638                                                  optval, optlen);
2639        return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2640}
2641EXPORT_SYMBOL(compat_tcp_setsockopt);
2642#endif
2643
2644/* Return information about state of tcp endpoint in API format. */
2645void tcp_get_info(struct sock *sk, struct tcp_info *info)
2646{
2647        const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
2648        const struct inet_connection_sock *icsk = inet_csk(sk);
2649        u32 now = tcp_time_stamp;
2650        unsigned int start;
2651        int notsent_bytes;
2652        u64 rate64;
2653        u32 rate;
2654
2655        memset(info, 0, sizeof(*info));
2656        if (sk->sk_type != SOCK_STREAM)
2657                return;
2658
2659        info->tcpi_state = sk_state_load(sk);
2660
2661        info->tcpi_ca_state = icsk->icsk_ca_state;
2662        info->tcpi_retransmits = icsk->icsk_retransmits;
2663        info->tcpi_probes = icsk->icsk_probes_out;
2664        info->tcpi_backoff = icsk->icsk_backoff;
2665
2666        if (tp->rx_opt.tstamp_ok)
2667                info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2668        if (tcp_is_sack(tp))
2669                info->tcpi_options |= TCPI_OPT_SACK;
2670        if (tp->rx_opt.wscale_ok) {
2671                info->tcpi_options |= TCPI_OPT_WSCALE;
2672                info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2673                info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2674        }
2675
2676        if (tp->ecn_flags & TCP_ECN_OK)
2677                info->tcpi_options |= TCPI_OPT_ECN;
2678        if (tp->ecn_flags & TCP_ECN_SEEN)
2679                info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2680        if (tp->syn_data_acked)
2681                info->tcpi_options |= TCPI_OPT_SYN_DATA;
2682
2683        info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2684        info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2685        info->tcpi_snd_mss = tp->mss_cache;
2686        info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2687
2688        if (info->tcpi_state == TCP_LISTEN) {
2689                info->tcpi_unacked = sk->sk_ack_backlog;
2690                info->tcpi_sacked = sk->sk_max_ack_backlog;
2691        } else {
2692                info->tcpi_unacked = tp->packets_out;
2693                info->tcpi_sacked = tp->sacked_out;
2694        }
2695        info->tcpi_lost = tp->lost_out;
2696        info->tcpi_retrans = tp->retrans_out;
2697        info->tcpi_fackets = tp->fackets_out;
2698
2699        info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2700        info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2701        info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2702
2703        info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2704        info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2705        info->tcpi_rtt = tp->srtt_us >> 3;
2706        info->tcpi_rttvar = tp->mdev_us >> 2;
2707        info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2708        info->tcpi_snd_cwnd = tp->snd_cwnd;
2709        info->tcpi_advmss = tp->advmss;
2710        info->tcpi_reordering = tp->reordering;
2711
2712        info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2713        info->tcpi_rcv_space = tp->rcvq_space.space;
2714
2715        info->tcpi_total_retrans = tp->total_retrans;
2716
2717        rate = READ_ONCE(sk->sk_pacing_rate);
2718        rate64 = rate != ~0U ? rate : ~0ULL;
2719        put_unaligned(rate64, &info->tcpi_pacing_rate);
2720
2721        rate = READ_ONCE(sk->sk_max_pacing_rate);
2722        rate64 = rate != ~0U ? rate : ~0ULL;
2723        put_unaligned(rate64, &info->tcpi_max_pacing_rate);
2724
2725        do {
2726                start = u64_stats_fetch_begin_irq(&tp->syncp);
2727                put_unaligned(tp->bytes_acked, &info->tcpi_bytes_acked);
2728                put_unaligned(tp->bytes_received, &info->tcpi_bytes_received);
2729        } while (u64_stats_fetch_retry_irq(&tp->syncp, start));
2730        info->tcpi_segs_out = tp->segs_out;
2731        info->tcpi_segs_in = tp->segs_in;
2732
2733        notsent_bytes = READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt);
2734        info->tcpi_notsent_bytes = max(0, notsent_bytes);
2735
2736        info->tcpi_min_rtt = tcp_min_rtt(tp);
2737        info->tcpi_data_segs_in = tp->data_segs_in;
2738        info->tcpi_data_segs_out = tp->data_segs_out;
2739}
2740EXPORT_SYMBOL_GPL(tcp_get_info);
2741
2742static int do_tcp_getsockopt(struct sock *sk, int level,
2743                int optname, char __user *optval, int __user *optlen)
2744{
2745        struct inet_connection_sock *icsk = inet_csk(sk);
2746        struct tcp_sock *tp = tcp_sk(sk);
2747        struct net *net = sock_net(sk);
2748        int val, len;
2749
2750        if (get_user(len, optlen))
2751                return -EFAULT;
2752
2753        len = min_t(unsigned int, len, sizeof(int));
2754
2755        if (len < 0)
2756                return -EINVAL;
2757
2758        switch (optname) {
2759        case TCP_MAXSEG:
2760                val = tp->mss_cache;
2761                if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2762                        val = tp->rx_opt.user_mss;
2763                if (tp->repair)
2764                        val = tp->rx_opt.mss_clamp;
2765                break;
2766        case TCP_NODELAY:
2767                val = !!(tp->nonagle&TCP_NAGLE_OFF);
2768                break;
2769        case TCP_CORK:
2770                val = !!(tp->nonagle&TCP_NAGLE_CORK);
2771                break;
2772        case TCP_KEEPIDLE:
2773                val = keepalive_time_when(tp) / HZ;
2774                break;
2775        case TCP_KEEPINTVL:
2776                val = keepalive_intvl_when(tp) / HZ;
2777                break;
2778        case TCP_KEEPCNT:
2779                val = keepalive_probes(tp);
2780                break;
2781        case TCP_SYNCNT:
2782                val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries;
2783                break;
2784        case TCP_LINGER2:
2785                val = tp->linger2;
2786                if (val >= 0)
2787                        val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
2788                break;
2789        case TCP_DEFER_ACCEPT:
2790                val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2791                                      TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2792                break;
2793        case TCP_WINDOW_CLAMP:
2794                val = tp->window_clamp;
2795                break;
2796        case TCP_INFO: {
2797                struct tcp_info info;
2798
2799                if (get_user(len, optlen))
2800                        return -EFAULT;
2801
2802                tcp_get_info(sk, &info);
2803
2804                len = min_t(unsigned int, len, sizeof(info));
2805                if (put_user(len, optlen))
2806                        return -EFAULT;
2807                if (copy_to_user(optval, &info, len))
2808                        return -EFAULT;
2809                return 0;
2810        }
2811        case TCP_CC_INFO: {
2812                const struct tcp_congestion_ops *ca_ops;
2813                union tcp_cc_info info;
2814                size_t sz = 0;
2815                int attr;
2816
2817                if (get_user(len, optlen))
2818                        return -EFAULT;
2819
2820                ca_ops = icsk->icsk_ca_ops;
2821                if (ca_ops && ca_ops->get_info)
2822                        sz = ca_ops->get_info(sk, ~0U, &attr, &info);
2823
2824                len = min_t(unsigned int, len, sz);
2825                if (put_user(len, optlen))
2826                        return -EFAULT;
2827                if (copy_to_user(optval, &info, len))
2828                        return -EFAULT;
2829                return 0;
2830        }
2831        case TCP_QUICKACK:
2832                val = !icsk->icsk_ack.pingpong;
2833                break;
2834
2835        case TCP_CONGESTION:
2836                if (get_user(len, optlen))
2837                        return -EFAULT;
2838                len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2839                if (put_user(len, optlen))
2840                        return -EFAULT;
2841                if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2842                        return -EFAULT;
2843                return 0;
2844
2845        case TCP_THIN_LINEAR_TIMEOUTS:
2846                val = tp->thin_lto;
2847                break;
2848        case TCP_THIN_DUPACK:
2849                val = tp->thin_dupack;
2850                break;
2851
2852        case TCP_REPAIR:
2853                val = tp->repair;
2854                break;
2855
2856        case TCP_REPAIR_QUEUE:
2857                if (tp->repair)
2858                        val = tp->repair_queue;
2859                else
2860                        return -EINVAL;
2861                break;
2862
2863        case TCP_QUEUE_SEQ:
2864                if (tp->repair_queue == TCP_SEND_QUEUE)
2865                        val = tp->write_seq;
2866                else if (tp->repair_queue == TCP_RECV_QUEUE)
2867                        val = tp->rcv_nxt;
2868                else
2869                        return -EINVAL;
2870                break;
2871
2872        case TCP_USER_TIMEOUT:
2873                val = jiffies_to_msecs(icsk->icsk_user_timeout);
2874                break;
2875
2876        case TCP_FASTOPEN:
2877                val = icsk->icsk_accept_queue.fastopenq.max_qlen;
2878                break;
2879
2880        case TCP_TIMESTAMP:
2881                val = tcp_time_stamp + tp->tsoffset;
2882                break;
2883        case TCP_NOTSENT_LOWAT:
2884                val = tp->notsent_lowat;
2885                break;
2886        case TCP_SAVE_SYN:
2887                val = tp->save_syn;
2888                break;
2889        case TCP_SAVED_SYN: {
2890                if (get_user(len, optlen))
2891                        return -EFAULT;
2892
2893                lock_sock(sk);
2894                if (tp->saved_syn) {
2895                        if (len < tp->saved_syn[0]) {
2896                                if (put_user(tp->saved_syn[0], optlen)) {
2897                                        release_sock(sk);
2898                                        return -EFAULT;
2899                                }
2900                                release_sock(sk);
2901                                return -EINVAL;
2902                        }
2903                        len = tp->saved_syn[0];
2904                        if (put_user(len, optlen)) {
2905                                release_sock(sk);
2906                                return -EFAULT;
2907                        }
2908                        if (copy_to_user(optval, tp->saved_syn + 1, len)) {
2909                                release_sock(sk);
2910                                return -EFAULT;
2911                        }
2912                        tcp_saved_syn_free(tp);
2913                        release_sock(sk);
2914                } else {
2915                        release_sock(sk);
2916                        len = 0;
2917                        if (put_user(len, optlen))
2918                                return -EFAULT;
2919                }
2920                return 0;
2921        }
2922        default:
2923                return -ENOPROTOOPT;
2924        }
2925
2926        if (put_user(len, optlen))
2927                return -EFAULT;
2928        if (copy_to_user(optval, &val, len))
2929                return -EFAULT;
2930        return 0;
2931}
2932
2933int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2934                   int __user *optlen)
2935{
2936        struct inet_connection_sock *icsk = inet_csk(sk);
2937
2938        if (level != SOL_TCP)
2939                return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2940                                                     optval, optlen);
2941        return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2942}
2943EXPORT_SYMBOL(tcp_getsockopt);
2944
2945#ifdef CONFIG_COMPAT
2946int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2947                          char __user *optval, int __user *optlen)
2948{
2949        if (level != SOL_TCP)
2950                return inet_csk_compat_getsockopt(sk, level, optname,
2951                                                  optval, optlen);
2952        return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2953}
2954EXPORT_SYMBOL(compat_tcp_getsockopt);
2955#endif
2956
2957#ifdef CONFIG_TCP_MD5SIG
2958static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
2959static DEFINE_MUTEX(tcp_md5sig_mutex);
2960static bool tcp_md5sig_pool_populated = false;
2961
2962static void __tcp_alloc_md5sig_pool(void)
2963{
2964        struct crypto_ahash *hash;
2965        int cpu;
2966
2967        hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
2968        if (IS_ERR(hash))
2969                return;
2970
2971        for_each_possible_cpu(cpu) {
2972                struct ahash_request *req;
2973
2974                if (per_cpu(tcp_md5sig_pool, cpu).md5_req)
2975                        continue;
2976
2977                req = ahash_request_alloc(hash, GFP_KERNEL);
2978                if (!req)
2979                        return;
2980
2981                ahash_request_set_callback(req, 0, NULL, NULL);
2982
2983                per_cpu(tcp_md5sig_pool, cpu).md5_req = req;
2984        }
2985        /* before setting tcp_md5sig_pool_populated, we must commit all writes
2986         * to memory. See smp_rmb() in tcp_get_md5sig_pool()
2987         */
2988        smp_wmb();
2989        tcp_md5sig_pool_populated = true;
2990}
2991
2992bool tcp_alloc_md5sig_pool(void)
2993{
2994        if (unlikely(!tcp_md5sig_pool_populated)) {
2995                mutex_lock(&tcp_md5sig_mutex);
2996
2997                if (!tcp_md5sig_pool_populated)
2998                        __tcp_alloc_md5sig_pool();
2999
3000                mutex_unlock(&tcp_md5sig_mutex);
3001        }
3002        return tcp_md5sig_pool_populated;
3003}
3004EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3005
3006
3007/**
3008 *      tcp_get_md5sig_pool - get md5sig_pool for this user
3009 *
3010 *      We use percpu structure, so if we succeed, we exit with preemption
3011 *      and BH disabled, to make sure another thread or softirq handling
3012 *      wont try to get same context.
3013 */
3014struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3015{
3016        local_bh_disable();
3017
3018        if (tcp_md5sig_pool_populated) {
3019                /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3020                smp_rmb();
3021                return this_cpu_ptr(&tcp_md5sig_pool);
3022        }
3023        local_bh_enable();
3024        return NULL;
3025}
3026EXPORT_SYMBOL(tcp_get_md5sig_pool);
3027
3028int tcp_md5_hash_header(struct tcp_md5sig_pool *hp,
3029                        const struct tcphdr *th)
3030{
3031        struct scatterlist sg;
3032        struct tcphdr hdr;
3033
3034        /* We are not allowed to change tcphdr, make a local copy */
3035        memcpy(&hdr, th, sizeof(hdr));
3036        hdr.check = 0;
3037
3038        /* options aren't included in the hash */
3039        sg_init_one(&sg, &hdr, sizeof(hdr));
3040        ahash_request_set_crypt(hp->md5_req, &sg, NULL, sizeof(hdr));
3041        return crypto_ahash_update(hp->md5_req);
3042}
3043EXPORT_SYMBOL(tcp_md5_hash_header);
3044
3045int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3046                          const struct sk_buff *skb, unsigned int header_len)
3047{
3048        struct scatterlist sg;
3049        const struct tcphdr *tp = tcp_hdr(skb);
3050        struct ahash_request *req = hp->md5_req;
3051        unsigned int i;
3052        const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3053                                           skb_headlen(skb) - header_len : 0;
3054        const struct skb_shared_info *shi = skb_shinfo(skb);
3055        struct sk_buff *frag_iter;
3056
3057        sg_init_table(&sg, 1);
3058
3059        sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3060        ahash_request_set_crypt(req, &sg, NULL, head_data_len);
3061        if (crypto_ahash_update(req))
3062                return 1;
3063
3064        for (i = 0; i < shi->nr_frags; ++i) {
3065                const struct skb_frag_struct *f = &shi->frags[i];
3066                unsigned int offset = f->page_offset;
3067                struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3068
3069                sg_set_page(&sg, page, skb_frag_size(f),
3070                            offset_in_page(offset));
3071                ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
3072                if (crypto_ahash_update(req))
3073                        return 1;
3074        }
3075
3076        skb_walk_frags(skb, frag_iter)
3077                if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3078                        return 1;
3079
3080        return 0;
3081}
3082EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3083
3084int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3085{
3086        struct scatterlist sg;
3087
3088        sg_init_one(&sg, key->key, key->keylen);
3089        ahash_request_set_crypt(hp->md5_req, &sg, NULL, key->keylen);
3090        return crypto_ahash_update(hp->md5_req);
3091}
3092EXPORT_SYMBOL(tcp_md5_hash_key);
3093
3094#endif
3095
3096void tcp_done(struct sock *sk)
3097{
3098        struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
3099
3100        if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3101                TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3102
3103        tcp_set_state(sk, TCP_CLOSE);
3104        tcp_clear_xmit_timers(sk);
3105        if (req)
3106                reqsk_fastopen_remove(sk, req, false);
3107
3108        sk->sk_shutdown = SHUTDOWN_MASK;
3109
3110        if (!sock_flag(sk, SOCK_DEAD))
3111                sk->sk_state_change(sk);
3112        else
3113                inet_csk_destroy_sock(sk);
3114}
3115EXPORT_SYMBOL_GPL(tcp_done);
3116
3117int tcp_abort(struct sock *sk, int err)
3118{
3119        if (!sk_fullsock(sk)) {
3120                if (sk->sk_state == TCP_NEW_SYN_RECV) {
3121                        struct request_sock *req = inet_reqsk(sk);
3122
3123                        local_bh_disable();
3124                        inet_csk_reqsk_queue_drop_and_put(req->rsk_listener,
3125                                                          req);
3126                        local_bh_enable();
3127                        return 0;
3128                }
3129                sock_gen_put(sk);
3130                return -EOPNOTSUPP;
3131        }
3132
3133        /* Don't race with userspace socket closes such as tcp_close. */
3134        lock_sock(sk);
3135
3136        if (sk->sk_state == TCP_LISTEN) {
3137                tcp_set_state(sk, TCP_CLOSE);
3138                inet_csk_listen_stop(sk);
3139        }
3140
3141        /* Don't race with BH socket closes such as inet_csk_listen_stop. */
3142        local_bh_disable();
3143        bh_lock_sock(sk);
3144
3145        if (!sock_flag(sk, SOCK_DEAD)) {
3146                sk->sk_err = err;
3147                /* This barrier is coupled with smp_rmb() in tcp_poll() */
3148                smp_wmb();
3149                sk->sk_error_report(sk);
3150                if (tcp_need_reset(sk->sk_state))
3151                        tcp_send_active_reset(sk, GFP_ATOMIC);
3152                tcp_done(sk);
3153        }
3154
3155        bh_unlock_sock(sk);
3156        local_bh_enable();
3157        release_sock(sk);
3158        sock_put(sk);
3159        return 0;
3160}
3161EXPORT_SYMBOL_GPL(tcp_abort);
3162
3163extern struct tcp_congestion_ops tcp_reno;
3164
3165static __initdata unsigned long thash_entries;
3166static int __init set_thash_entries(char *str)
3167{
3168        ssize_t ret;
3169
3170        if (!str)
3171                return 0;
3172
3173        ret = kstrtoul(str, 0, &thash_entries);
3174        if (ret)
3175                return 0;
3176
3177        return 1;
3178}
3179__setup("thash_entries=", set_thash_entries);
3180
3181static void __init tcp_init_mem(void)
3182{
3183        unsigned long limit = nr_free_buffer_pages() / 16;
3184
3185        limit = max(limit, 128UL);
3186        sysctl_tcp_mem[0] = limit / 4 * 3;              /* 4.68 % */
3187        sysctl_tcp_mem[1] = limit;                      /* 6.25 % */
3188        sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;      /* 9.37 % */
3189}
3190
3191void __init tcp_init(void)
3192{
3193        unsigned long limit;
3194        int max_rshare, max_wshare, cnt;
3195        unsigned int i;
3196
3197        sock_skb_cb_check_size(sizeof(struct tcp_skb_cb));
3198
3199        percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3200        percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
3201        tcp_hashinfo.bind_bucket_cachep =
3202                kmem_cache_create("tcp_bind_bucket",
3203                                  sizeof(struct inet_bind_bucket), 0,
3204                                  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3205
3206        /* Size and allocate the main established and bind bucket
3207         * hash tables.
3208         *
3209         * The methodology is similar to that of the buffer cache.
3210         */
3211        tcp_hashinfo.ehash =
3212                alloc_large_system_hash("TCP established",
3213                                        sizeof(struct inet_ehash_bucket),
3214                                        thash_entries,
3215                                        17, /* one slot per 128 KB of memory */
3216                                        0,
3217                                        NULL,
3218                                        &tcp_hashinfo.ehash_mask,
3219                                        0,
3220                                        thash_entries ? 0 : 512 * 1024);
3221        for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3222                INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3223
3224        if (inet_ehash_locks_alloc(&tcp_hashinfo))
3225                panic("TCP: failed to alloc ehash_locks");
3226        tcp_hashinfo.bhash =
3227                alloc_large_system_hash("TCP bind",
3228                                        sizeof(struct inet_bind_hashbucket),
3229                                        tcp_hashinfo.ehash_mask + 1,
3230                                        17, /* one slot per 128 KB of memory */
3231                                        0,
3232                                        &tcp_hashinfo.bhash_size,
3233                                        NULL,
3234                                        0,
3235                                        64 * 1024);
3236        tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3237        for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3238                spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3239                INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3240        }
3241
3242
3243        cnt = tcp_hashinfo.ehash_mask + 1;
3244
3245        tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3246        sysctl_tcp_max_orphans = cnt / 2;
3247        sysctl_max_syn_backlog = max(128, cnt / 256);
3248
3249        tcp_init_mem();
3250        /* Set per-socket limits to no more than 1/128 the pressure threshold */
3251        limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3252        max_wshare = min(4UL*1024*1024, limit);
3253        max_rshare = min(6UL*1024*1024, limit);
3254
3255        sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
3256        sysctl_tcp_wmem[1] = 16*1024;
3257        sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
3258
3259        sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
3260        sysctl_tcp_rmem[1] = 87380;
3261        sysctl_tcp_rmem[2] = max(87380, max_rshare);
3262
3263        pr_info("Hash tables configured (established %u bind %u)\n",
3264                tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
3265
3266        tcp_metrics_init();
3267        BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
3268        tcp_tasklet_init();
3269}
3270
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.