linux/net/ipv4/tcp.c
<<
>>
Prefs
   1/*
   2 * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3 *              operating system.  INET is implemented using the  BSD Socket
   4 *              interface as the means of communication with the user level.
   5 *
   6 *              Implementation of the Transmission Control Protocol(TCP).
   7 *
   8 * Authors:     Ross Biro
   9 *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10 *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  11 *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  12 *              Florian La Roche, <flla@stud.uni-sb.de>
  13 *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  14 *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  15 *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  16 *              Matthew Dillon, <dillon@apollo.west.oic.com>
  17 *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  18 *              Jorge Cwik, <jorge@laser.satlink.net>
  19 *
  20 * Fixes:
  21 *              Alan Cox        :       Numerous verify_area() calls
  22 *              Alan Cox        :       Set the ACK bit on a reset
  23 *              Alan Cox        :       Stopped it crashing if it closed while
  24 *                                      sk->inuse=1 and was trying to connect
  25 *                                      (tcp_err()).
  26 *              Alan Cox        :       All icmp error handling was broken
  27 *                                      pointers passed where wrong and the
  28 *                                      socket was looked up backwards. Nobody
  29 *                                      tested any icmp error code obviously.
  30 *              Alan Cox        :       tcp_err() now handled properly. It
  31 *                                      wakes people on errors. poll
  32 *                                      behaves and the icmp error race
  33 *                                      has gone by moving it into sock.c
  34 *              Alan Cox        :       tcp_send_reset() fixed to work for
  35 *                                      everything not just packets for
  36 *                                      unknown sockets.
  37 *              Alan Cox        :       tcp option processing.
  38 *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  39 *                                      syn rule wrong]
  40 *              Herp Rosmanith  :       More reset fixes
  41 *              Alan Cox        :       No longer acks invalid rst frames.
  42 *                                      Acking any kind of RST is right out.
  43 *              Alan Cox        :       Sets an ignore me flag on an rst
  44 *                                      receive otherwise odd bits of prattle
  45 *                                      escape still
  46 *              Alan Cox        :       Fixed another acking RST frame bug.
  47 *                                      Should stop LAN workplace lockups.
  48 *              Alan Cox        :       Some tidyups using the new skb list
  49 *                                      facilities
  50 *              Alan Cox        :       sk->keepopen now seems to work
  51 *              Alan Cox        :       Pulls options out correctly on accepts
  52 *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  53 *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  54 *                                      bit to skb ops.
  55 *              Alan Cox        :       Tidied tcp_data to avoid a potential
  56 *                                      nasty.
  57 *              Alan Cox        :       Added some better commenting, as the
  58 *                                      tcp is hard to follow
  59 *              Alan Cox        :       Removed incorrect check for 20 * psh
  60 *      Michael O'Reilly        :       ack < copied bug fix.
  61 *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  62 *              Alan Cox        :       FIN with no memory -> CRASH
  63 *              Alan Cox        :       Added socket option proto entries.
  64 *                                      Also added awareness of them to accept.
  65 *              Alan Cox        :       Added TCP options (SOL_TCP)
  66 *              Alan Cox        :       Switched wakeup calls to callbacks,
  67 *                                      so the kernel can layer network
  68 *                                      sockets.
  69 *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  70 *              Alan Cox        :       Handle FIN (more) properly (we hope).
  71 *              Alan Cox        :       RST frames sent on unsynchronised
  72 *                                      state ack error.
  73 *              Alan Cox        :       Put in missing check for SYN bit.
  74 *              Alan Cox        :       Added tcp_select_window() aka NET2E
  75 *                                      window non shrink trick.
  76 *              Alan Cox        :       Added a couple of small NET2E timer
  77 *                                      fixes
  78 *              Charles Hedrick :       TCP fixes
  79 *              Toomas Tamm     :       TCP window fixes
  80 *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  81 *              Charles Hedrick :       Rewrote most of it to actually work
  82 *              Linus           :       Rewrote tcp_read() and URG handling
  83 *                                      completely
  84 *              Gerhard Koerting:       Fixed some missing timer handling
  85 *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  86 *              Gerhard Koerting:       PC/TCP workarounds
  87 *              Adam Caldwell   :       Assorted timer/timing errors
  88 *              Matthew Dillon  :       Fixed another RST bug
  89 *              Alan Cox        :       Move to kernel side addressing changes.
  90 *              Alan Cox        :       Beginning work on TCP fastpathing
  91 *                                      (not yet usable)
  92 *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  93 *              Alan Cox        :       TCP fast path debugging
  94 *              Alan Cox        :       Window clamping
  95 *              Michael Riepe   :       Bug in tcp_check()
  96 *              Matt Dillon     :       More TCP improvements and RST bug fixes
  97 *              Matt Dillon     :       Yet more small nasties remove from the
  98 *                                      TCP code (Be very nice to this man if
  99 *                                      tcp finally works 100%) 8)
 100 *              Alan Cox        :       BSD accept semantics.
 101 *              Alan Cox        :       Reset on closedown bug.
 102 *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 103 *              Michael Pall    :       Handle poll() after URG properly in
 104 *                                      all cases.
 105 *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 106 *                                      (multi URG PUSH broke rlogin).
 107 *              Michael Pall    :       Fix the multi URG PUSH problem in
 108 *                                      tcp_readable(), poll() after URG
 109 *                                      works now.
 110 *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 111 *                                      BSD api.
 112 *              Alan Cox        :       Changed the semantics of sk->socket to
 113 *                                      fix a race and a signal problem with
 114 *                                      accept() and async I/O.
 115 *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 116 *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 117 *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 118 *                                      clients/servers which listen in on
 119 *                                      fixed ports.
 120 *              Alan Cox        :       Cleaned the above up and shrank it to
 121 *                                      a sensible code size.
 122 *              Alan Cox        :       Self connect lockup fix.
 123 *              Alan Cox        :       No connect to multicast.
 124 *              Ross Biro       :       Close unaccepted children on master
 125 *                                      socket close.
 126 *              Alan Cox        :       Reset tracing code.
 127 *              Alan Cox        :       Spurious resets on shutdown.
 128 *              Alan Cox        :       Giant 15 minute/60 second timer error
 129 *              Alan Cox        :       Small whoops in polling before an
 130 *                                      accept.
 131 *              Alan Cox        :       Kept the state trace facility since
 132 *                                      it's handy for debugging.
 133 *              Alan Cox        :       More reset handler fixes.
 134 *              Alan Cox        :       Started rewriting the code based on
 135 *                                      the RFC's for other useful protocol
 136 *                                      references see: Comer, KA9Q NOS, and
 137 *                                      for a reference on the difference
 138 *                                      between specifications and how BSD
 139 *                                      works see the 4.4lite source.
 140 *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 141 *                                      close.
 142 *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 143 *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 144 *              Alan Cox        :       Reimplemented timers as per the RFC
 145 *                                      and using multiple timers for sanity.
 146 *              Alan Cox        :       Small bug fixes, and a lot of new
 147 *                                      comments.
 148 *              Alan Cox        :       Fixed dual reader crash by locking
 149 *                                      the buffers (much like datagram.c)
 150 *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 151 *                                      now gets fed up of retrying without
 152 *                                      (even a no space) answer.
 153 *              Alan Cox        :       Extracted closing code better
 154 *              Alan Cox        :       Fixed the closing state machine to
 155 *                                      resemble the RFC.
 156 *              Alan Cox        :       More 'per spec' fixes.
 157 *              Jorge Cwik      :       Even faster checksumming.
 158 *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 159 *                                      only frames. At least one pc tcp stack
 160 *                                      generates them.
 161 *              Alan Cox        :       Cache last socket.
 162 *              Alan Cox        :       Per route irtt.
 163 *              Matt Day        :       poll()->select() match BSD precisely on error
 164 *              Alan Cox        :       New buffers
 165 *              Marc Tamsky     :       Various sk->prot->retransmits and
 166 *                                      sk->retransmits misupdating fixed.
 167 *                                      Fixed tcp_write_timeout: stuck close,
 168 *                                      and TCP syn retries gets used now.
 169 *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 170 *                                      ack if state is TCP_CLOSED.
 171 *              Alan Cox        :       Look up device on a retransmit - routes may
 172 *                                      change. Doesn't yet cope with MSS shrink right
 173 *                                      but it's a start!
 174 *              Marc Tamsky     :       Closing in closing fixes.
 175 *              Mike Shaver     :       RFC1122 verifications.
 176 *              Alan Cox        :       rcv_saddr errors.
 177 *              Alan Cox        :       Block double connect().
 178 *              Alan Cox        :       Small hooks for enSKIP.
 179 *              Alexey Kuznetsov:       Path MTU discovery.
 180 *              Alan Cox        :       Support soft errors.
 181 *              Alan Cox        :       Fix MTU discovery pathological case
 182 *                                      when the remote claims no mtu!
 183 *              Marc Tamsky     :       TCP_CLOSE fix.
 184 *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 185 *                                      window but wrong (fixes NT lpd problems)
 186 *              Pedro Roque     :       Better TCP window handling, delayed ack.
 187 *              Joerg Reuter    :       No modification of locked buffers in
 188 *                                      tcp_do_retransmit()
 189 *              Eric Schenk     :       Changed receiver side silly window
 190 *                                      avoidance algorithm to BSD style
 191 *                                      algorithm. This doubles throughput
 192 *                                      against machines running Solaris,
 193 *                                      and seems to result in general
 194 *                                      improvement.
 195 *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 196 *      Willy Konynenberg       :       Transparent proxying support.
 197 *      Mike McLagan            :       Routing by source
 198 *              Keith Owens     :       Do proper merging with partial SKB's in
 199 *                                      tcp_do_sendmsg to avoid burstiness.
 200 *              Eric Schenk     :       Fix fast close down bug with
 201 *                                      shutdown() followed by close().
 202 *              Andi Kleen      :       Make poll agree with SIGIO
 203 *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 204 *                                      lingertime == 0 (RFC 793 ABORT Call)
 205 *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 206 *                                      csum_and_copy_from_user() if possible.
 207 *
 208 *              This program is free software; you can redistribute it and/or
 209 *              modify it under the terms of the GNU General Public License
 210 *              as published by the Free Software Foundation; either version
 211 *              2 of the License, or(at your option) any later version.
 212 *
 213 * Description of States:
 214 *
 215 *      TCP_SYN_SENT            sent a connection request, waiting for ack
 216 *
 217 *      TCP_SYN_RECV            received a connection request, sent ack,
 218 *                              waiting for final ack in three-way handshake.
 219 *
 220 *      TCP_ESTABLISHED         connection established
 221 *
 222 *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 223 *                              transmission of remaining buffered data
 224 *
 225 *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 226 *                              to shutdown
 227 *
 228 *      TCP_CLOSING             both sides have shutdown but we still have
 229 *                              data we have to finish sending
 230 *
 231 *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 232 *                              closed, can only be entered from FIN_WAIT2
 233 *                              or CLOSING.  Required because the other end
 234 *                              may not have gotten our last ACK causing it
 235 *                              to retransmit the data packet (which we ignore)
 236 *
 237 *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 238 *                              us to finish writing our data and to shutdown
 239 *                              (we have to close() to move on to LAST_ACK)
 240 *
 241 *      TCP_LAST_ACK            out side has shutdown after remote has
 242 *                              shutdown.  There may still be data in our
 243 *                              buffer that we have to finish sending
 244 *
 245 *      TCP_CLOSE               socket is finished
 246 */
 247
 248#define pr_fmt(fmt) "TCP: " fmt
 249
 250#include <crypto/hash.h>
 251#include <linux/kernel.h>
 252#include <linux/module.h>
 253#include <linux/types.h>
 254#include <linux/fcntl.h>
 255#include <linux/poll.h>
 256#include <linux/inet_diag.h>
 257#include <linux/init.h>
 258#include <linux/fs.h>
 259#include <linux/skbuff.h>
 260#include <linux/scatterlist.h>
 261#include <linux/splice.h>
 262#include <linux/net.h>
 263#include <linux/socket.h>
 264#include <linux/random.h>
 265#include <linux/bootmem.h>
 266#include <linux/highmem.h>
 267#include <linux/swap.h>
 268#include <linux/cache.h>
 269#include <linux/err.h>
 270#include <linux/time.h>
 271#include <linux/slab.h>
 272
 273#include <net/icmp.h>
 274#include <net/inet_common.h>
 275#include <net/tcp.h>
 276#include <net/xfrm.h>
 277#include <net/ip.h>
 278#include <net/sock.h>
 279
 280#include <linux/uaccess.h>
 281#include <asm/ioctls.h>
 282#include <net/busy_poll.h>
 283
 284int sysctl_tcp_min_tso_segs __read_mostly = 2;
 285
 286int sysctl_tcp_autocorking __read_mostly = 1;
 287
 288struct percpu_counter tcp_orphan_count;
 289EXPORT_SYMBOL_GPL(tcp_orphan_count);
 290
 291long sysctl_tcp_mem[3] __read_mostly;
 292int sysctl_tcp_wmem[3] __read_mostly;
 293int sysctl_tcp_rmem[3] __read_mostly;
 294
 295EXPORT_SYMBOL(sysctl_tcp_mem);
 296EXPORT_SYMBOL(sysctl_tcp_rmem);
 297EXPORT_SYMBOL(sysctl_tcp_wmem);
 298
 299atomic_long_t tcp_memory_allocated;     /* Current allocated memory. */
 300EXPORT_SYMBOL(tcp_memory_allocated);
 301
 302/*
 303 * Current number of TCP sockets.
 304 */
 305struct percpu_counter tcp_sockets_allocated;
 306EXPORT_SYMBOL(tcp_sockets_allocated);
 307
 308/*
 309 * TCP splice context
 310 */
 311struct tcp_splice_state {
 312        struct pipe_inode_info *pipe;
 313        size_t len;
 314        unsigned int flags;
 315};
 316
 317/*
 318 * Pressure flag: try to collapse.
 319 * Technical note: it is used by multiple contexts non atomically.
 320 * All the __sk_mem_schedule() is of this nature: accounting
 321 * is strict, actions are advisory and have some latency.
 322 */
 323int tcp_memory_pressure __read_mostly;
 324EXPORT_SYMBOL(tcp_memory_pressure);
 325
 326void tcp_enter_memory_pressure(struct sock *sk)
 327{
 328        if (!tcp_memory_pressure) {
 329                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
 330                tcp_memory_pressure = 1;
 331        }
 332}
 333EXPORT_SYMBOL(tcp_enter_memory_pressure);
 334
 335/* Convert seconds to retransmits based on initial and max timeout */
 336static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
 337{
 338        u8 res = 0;
 339
 340        if (seconds > 0) {
 341                int period = timeout;
 342
 343                res = 1;
 344                while (seconds > period && res < 255) {
 345                        res++;
 346                        timeout <<= 1;
 347                        if (timeout > rto_max)
 348                                timeout = rto_max;
 349                        period += timeout;
 350                }
 351        }
 352        return res;
 353}
 354
 355/* Convert retransmits to seconds based on initial and max timeout */
 356static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
 357{
 358        int period = 0;
 359
 360        if (retrans > 0) {
 361                period = timeout;
 362                while (--retrans) {
 363                        timeout <<= 1;
 364                        if (timeout > rto_max)
 365                                timeout = rto_max;
 366                        period += timeout;
 367                }
 368        }
 369        return period;
 370}
 371
 372/* Address-family independent initialization for a tcp_sock.
 373 *
 374 * NOTE: A lot of things set to zero explicitly by call to
 375 *       sk_alloc() so need not be done here.
 376 */
 377void tcp_init_sock(struct sock *sk)
 378{
 379        struct inet_connection_sock *icsk = inet_csk(sk);
 380        struct tcp_sock *tp = tcp_sk(sk);
 381
 382        tp->out_of_order_queue = RB_ROOT;
 383        tcp_init_xmit_timers(sk);
 384        tcp_prequeue_init(tp);
 385        INIT_LIST_HEAD(&tp->tsq_node);
 386
 387        icsk->icsk_rto = TCP_TIMEOUT_INIT;
 388        tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
 389        minmax_reset(&tp->rtt_min, tcp_time_stamp, ~0U);
 390
 391        /* So many TCP implementations out there (incorrectly) count the
 392         * initial SYN frame in their delayed-ACK and congestion control
 393         * algorithms that we must have the following bandaid to talk
 394         * efficiently to them.  -DaveM
 395         */
 396        tp->snd_cwnd = TCP_INIT_CWND;
 397
 398        /* There's a bubble in the pipe until at least the first ACK. */
 399        tp->app_limited = ~0U;
 400
 401        /* See draft-stevens-tcpca-spec-01 for discussion of the
 402         * initialization of these values.
 403         */
 404        tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 405        tp->snd_cwnd_clamp = ~0;
 406        tp->mss_cache = TCP_MSS_DEFAULT;
 407
 408        tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
 409        tcp_enable_early_retrans(tp);
 410        tcp_assign_congestion_control(sk);
 411
 412        tp->tsoffset = 0;
 413
 414        sk->sk_state = TCP_CLOSE;
 415
 416        sk->sk_write_space = sk_stream_write_space;
 417        sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
 418
 419        icsk->icsk_sync_mss = tcp_sync_mss;
 420
 421        sk->sk_sndbuf = sysctl_tcp_wmem[1];
 422        sk->sk_rcvbuf = sysctl_tcp_rmem[1];
 423
 424        local_bh_disable();
 425        sk_sockets_allocated_inc(sk);
 426        local_bh_enable();
 427}
 428EXPORT_SYMBOL(tcp_init_sock);
 429
 430static void tcp_tx_timestamp(struct sock *sk, u16 tsflags, struct sk_buff *skb)
 431{
 432        if (tsflags) {
 433                struct skb_shared_info *shinfo = skb_shinfo(skb);
 434                struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 435
 436                sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
 437                if (tsflags & SOF_TIMESTAMPING_TX_ACK)
 438                        tcb->txstamp_ack = 1;
 439                if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
 440                        shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
 441        }
 442}
 443
 444/*
 445 *      Wait for a TCP event.
 446 *
 447 *      Note that we don't need to lock the socket, as the upper poll layers
 448 *      take care of normal races (between the test and the event) and we don't
 449 *      go look at any of the socket buffers directly.
 450 */
 451unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 452{
 453        unsigned int mask;
 454        struct sock *sk = sock->sk;
 455        const struct tcp_sock *tp = tcp_sk(sk);
 456        int state;
 457
 458        sock_rps_record_flow(sk);
 459
 460        sock_poll_wait(file, sk_sleep(sk), wait);
 461
 462        state = sk_state_load(sk);
 463        if (state == TCP_LISTEN)
 464                return inet_csk_listen_poll(sk);
 465
 466        /* Socket is not locked. We are protected from async events
 467         * by poll logic and correct handling of state changes
 468         * made by other threads is impossible in any case.
 469         */
 470
 471        mask = 0;
 472
 473        /*
 474         * POLLHUP is certainly not done right. But poll() doesn't
 475         * have a notion of HUP in just one direction, and for a
 476         * socket the read side is more interesting.
 477         *
 478         * Some poll() documentation says that POLLHUP is incompatible
 479         * with the POLLOUT/POLLWR flags, so somebody should check this
 480         * all. But careful, it tends to be safer to return too many
 481         * bits than too few, and you can easily break real applications
 482         * if you don't tell them that something has hung up!
 483         *
 484         * Check-me.
 485         *
 486         * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 487         * our fs/select.c). It means that after we received EOF,
 488         * poll always returns immediately, making impossible poll() on write()
 489         * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 490         * if and only if shutdown has been made in both directions.
 491         * Actually, it is interesting to look how Solaris and DUX
 492         * solve this dilemma. I would prefer, if POLLHUP were maskable,
 493         * then we could set it on SND_SHUTDOWN. BTW examples given
 494         * in Stevens' books assume exactly this behaviour, it explains
 495         * why POLLHUP is incompatible with POLLOUT.    --ANK
 496         *
 497         * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 498         * blocking on fresh not-connected or disconnected socket. --ANK
 499         */
 500        if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 501                mask |= POLLHUP;
 502        if (sk->sk_shutdown & RCV_SHUTDOWN)
 503                mask |= POLLIN | POLLRDNORM | POLLRDHUP;
 504
 505        /* Connected or passive Fast Open socket? */
 506        if (state != TCP_SYN_SENT &&
 507            (state != TCP_SYN_RECV || tp->fastopen_rsk)) {
 508                int target = sock_rcvlowat(sk, 0, INT_MAX);
 509
 510                if (tp->urg_seq == tp->copied_seq &&
 511                    !sock_flag(sk, SOCK_URGINLINE) &&
 512                    tp->urg_data)
 513                        target++;
 514
 515                if (tp->rcv_nxt - tp->copied_seq >= target)
 516                        mask |= POLLIN | POLLRDNORM;
 517
 518                if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 519                        if (sk_stream_is_writeable(sk)) {
 520                                mask |= POLLOUT | POLLWRNORM;
 521                        } else {  /* send SIGIO later */
 522                                sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 523                                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 524
 525                                /* Race breaker. If space is freed after
 526                                 * wspace test but before the flags are set,
 527                                 * IO signal will be lost. Memory barrier
 528                                 * pairs with the input side.
 529                                 */
 530                                smp_mb__after_atomic();
 531                                if (sk_stream_is_writeable(sk))
 532                                        mask |= POLLOUT | POLLWRNORM;
 533                        }
 534                } else
 535                        mask |= POLLOUT | POLLWRNORM;
 536
 537                if (tp->urg_data & TCP_URG_VALID)
 538                        mask |= POLLPRI;
 539        }
 540        /* This barrier is coupled with smp_wmb() in tcp_reset() */
 541        smp_rmb();
 542        if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
 543                mask |= POLLERR;
 544
 545        return mask;
 546}
 547EXPORT_SYMBOL(tcp_poll);
 548
 549int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 550{
 551        struct tcp_sock *tp = tcp_sk(sk);
 552        int answ;
 553        bool slow;
 554
 555        switch (cmd) {
 556        case SIOCINQ:
 557                if (sk->sk_state == TCP_LISTEN)
 558                        return -EINVAL;
 559
 560                slow = lock_sock_fast(sk);
 561                answ = tcp_inq(sk);
 562                unlock_sock_fast(sk, slow);
 563                break;
 564        case SIOCATMARK:
 565                answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 566                break;
 567        case SIOCOUTQ:
 568                if (sk->sk_state == TCP_LISTEN)
 569                        return -EINVAL;
 570
 571                if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 572                        answ = 0;
 573                else
 574                        answ = tp->write_seq - tp->snd_una;
 575                break;
 576        case SIOCOUTQNSD:
 577                if (sk->sk_state == TCP_LISTEN)
 578                        return -EINVAL;
 579
 580                if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 581                        answ = 0;
 582                else
 583                        answ = tp->write_seq - tp->snd_nxt;
 584                break;
 585        default:
 586                return -ENOIOCTLCMD;
 587        }
 588
 589        return put_user(answ, (int __user *)arg);
 590}
 591EXPORT_SYMBOL(tcp_ioctl);
 592
 593static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 594{
 595        TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
 596        tp->pushed_seq = tp->write_seq;
 597}
 598
 599static inline bool forced_push(const struct tcp_sock *tp)
 600{
 601        return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 602}
 603
 604static void skb_entail(struct sock *sk, struct sk_buff *skb)
 605{
 606        struct tcp_sock *tp = tcp_sk(sk);
 607        struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 608
 609        skb->csum    = 0;
 610        tcb->seq     = tcb->end_seq = tp->write_seq;
 611        tcb->tcp_flags = TCPHDR_ACK;
 612        tcb->sacked  = 0;
 613        __skb_header_release(skb);
 614        tcp_add_write_queue_tail(sk, skb);
 615        sk->sk_wmem_queued += skb->truesize;
 616        sk_mem_charge(sk, skb->truesize);
 617        if (tp->nonagle & TCP_NAGLE_PUSH)
 618                tp->nonagle &= ~TCP_NAGLE_PUSH;
 619
 620        tcp_slow_start_after_idle_check(sk);
 621}
 622
 623static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
 624{
 625        if (flags & MSG_OOB)
 626                tp->snd_up = tp->write_seq;
 627}
 628
 629/* If a not yet filled skb is pushed, do not send it if
 630 * we have data packets in Qdisc or NIC queues :
 631 * Because TX completion will happen shortly, it gives a chance
 632 * to coalesce future sendmsg() payload into this skb, without
 633 * need for a timer, and with no latency trade off.
 634 * As packets containing data payload have a bigger truesize
 635 * than pure acks (dataless) packets, the last checks prevent
 636 * autocorking if we only have an ACK in Qdisc/NIC queues,
 637 * or if TX completion was delayed after we processed ACK packet.
 638 */
 639static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
 640                                int size_goal)
 641{
 642        return skb->len < size_goal &&
 643               sysctl_tcp_autocorking &&
 644               skb != tcp_write_queue_head(sk) &&
 645               atomic_read(&sk->sk_wmem_alloc) > skb->truesize;
 646}
 647
 648static void tcp_push(struct sock *sk, int flags, int mss_now,
 649                     int nonagle, int size_goal)
 650{
 651        struct tcp_sock *tp = tcp_sk(sk);
 652        struct sk_buff *skb;
 653
 654        if (!tcp_send_head(sk))
 655                return;
 656
 657        skb = tcp_write_queue_tail(sk);
 658        if (!(flags & MSG_MORE) || forced_push(tp))
 659                tcp_mark_push(tp, skb);
 660
 661        tcp_mark_urg(tp, flags);
 662
 663        if (tcp_should_autocork(sk, skb, size_goal)) {
 664
 665                /* avoid atomic op if TSQ_THROTTLED bit is already set */
 666                if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
 667                        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
 668                        set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
 669                }
 670                /* It is possible TX completion already happened
 671                 * before we set TSQ_THROTTLED.
 672                 */
 673                if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
 674                        return;
 675        }
 676
 677        if (flags & MSG_MORE)
 678                nonagle = TCP_NAGLE_CORK;
 679
 680        __tcp_push_pending_frames(sk, mss_now, nonagle);
 681}
 682
 683static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 684                                unsigned int offset, size_t len)
 685{
 686        struct tcp_splice_state *tss = rd_desc->arg.data;
 687        int ret;
 688
 689        ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
 690                              min(rd_desc->count, len), tss->flags);
 691        if (ret > 0)
 692                rd_desc->count -= ret;
 693        return ret;
 694}
 695
 696static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 697{
 698        /* Store TCP splice context information in read_descriptor_t. */
 699        read_descriptor_t rd_desc = {
 700                .arg.data = tss,
 701                .count    = tss->len,
 702        };
 703
 704        return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 705}
 706
 707/**
 708 *  tcp_splice_read - splice data from TCP socket to a pipe
 709 * @sock:       socket to splice from
 710 * @ppos:       position (not valid)
 711 * @pipe:       pipe to splice to
 712 * @len:        number of bytes to splice
 713 * @flags:      splice modifier flags
 714 *
 715 * Description:
 716 *    Will read pages from given socket and fill them into a pipe.
 717 *
 718 **/
 719ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 720                        struct pipe_inode_info *pipe, size_t len,
 721                        unsigned int flags)
 722{
 723        struct sock *sk = sock->sk;
 724        struct tcp_splice_state tss = {
 725                .pipe = pipe,
 726                .len = len,
 727                .flags = flags,
 728        };
 729        long timeo;
 730        ssize_t spliced;
 731        int ret;
 732
 733        sock_rps_record_flow(sk);
 734        /*
 735         * We can't seek on a socket input
 736         */
 737        if (unlikely(*ppos))
 738                return -ESPIPE;
 739
 740        ret = spliced = 0;
 741
 742        lock_sock(sk);
 743
 744        timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
 745        while (tss.len) {
 746                ret = __tcp_splice_read(sk, &tss);
 747                if (ret < 0)
 748                        break;
 749                else if (!ret) {
 750                        if (spliced)
 751                                break;
 752                        if (sock_flag(sk, SOCK_DONE))
 753                                break;
 754                        if (sk->sk_err) {
 755                                ret = sock_error(sk);
 756                                break;
 757                        }
 758                        if (sk->sk_shutdown & RCV_SHUTDOWN)
 759                                break;
 760                        if (sk->sk_state == TCP_CLOSE) {
 761                                /*
 762                                 * This occurs when user tries to read
 763                                 * from never connected socket.
 764                                 */
 765                                if (!sock_flag(sk, SOCK_DONE))
 766                                        ret = -ENOTCONN;
 767                                break;
 768                        }
 769                        if (!timeo) {
 770                                ret = -EAGAIN;
 771                                break;
 772                        }
 773                        /* if __tcp_splice_read() got nothing while we have
 774                         * an skb in receive queue, we do not want to loop.
 775                         * This might happen with URG data.
 776                         */
 777                        if (!skb_queue_empty(&sk->sk_receive_queue))
 778                                break;
 779                        sk_wait_data(sk, &timeo, NULL);
 780                        if (signal_pending(current)) {
 781                                ret = sock_intr_errno(timeo);
 782                                break;
 783                        }
 784                        continue;
 785                }
 786                tss.len -= ret;
 787                spliced += ret;
 788
 789                if (!timeo)
 790                        break;
 791                release_sock(sk);
 792                lock_sock(sk);
 793
 794                if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 795                    (sk->sk_shutdown & RCV_SHUTDOWN) ||
 796                    signal_pending(current))
 797                        break;
 798        }
 799
 800        release_sock(sk);
 801
 802        if (spliced)
 803                return spliced;
 804
 805        return ret;
 806}
 807EXPORT_SYMBOL(tcp_splice_read);
 808
 809struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
 810                                    bool force_schedule)
 811{
 812        struct sk_buff *skb;
 813
 814        /* The TCP header must be at least 32-bit aligned.  */
 815        size = ALIGN(size, 4);
 816
 817        if (unlikely(tcp_under_memory_pressure(sk)))
 818                sk_mem_reclaim_partial(sk);
 819
 820        skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 821        if (likely(skb)) {
 822                bool mem_scheduled;
 823
 824                if (force_schedule) {
 825                        mem_scheduled = true;
 826                        sk_forced_mem_schedule(sk, skb->truesize);
 827                } else {
 828                        mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
 829                }
 830                if (likely(mem_scheduled)) {
 831                        skb_reserve(skb, sk->sk_prot->max_header);
 832                        /*
 833                         * Make sure that we have exactly size bytes
 834                         * available to the caller, no more, no less.
 835                         */
 836                        skb->reserved_tailroom = skb->end - skb->tail - size;
 837                        return skb;
 838                }
 839                __kfree_skb(skb);
 840        } else {
 841                sk->sk_prot->enter_memory_pressure(sk);
 842                sk_stream_moderate_sndbuf(sk);
 843        }
 844        return NULL;
 845}
 846
 847static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
 848                                       int large_allowed)
 849{
 850        struct tcp_sock *tp = tcp_sk(sk);
 851        u32 new_size_goal, size_goal;
 852
 853        if (!large_allowed || !sk_can_gso(sk))
 854                return mss_now;
 855
 856        /* Note : tcp_tso_autosize() will eventually split this later */
 857        new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
 858        new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
 859
 860        /* We try hard to avoid divides here */
 861        size_goal = tp->gso_segs * mss_now;
 862        if (unlikely(new_size_goal < size_goal ||
 863                     new_size_goal >= size_goal + mss_now)) {
 864                tp->gso_segs = min_t(u16, new_size_goal / mss_now,
 865                                     sk->sk_gso_max_segs);
 866                size_goal = tp->gso_segs * mss_now;
 867        }
 868
 869        return max(size_goal, mss_now);
 870}
 871
 872static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 873{
 874        int mss_now;
 875
 876        mss_now = tcp_current_mss(sk);
 877        *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
 878
 879        return mss_now;
 880}
 881
 882static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 883                                size_t size, int flags)
 884{
 885        struct tcp_sock *tp = tcp_sk(sk);
 886        int mss_now, size_goal;
 887        int err;
 888        ssize_t copied;
 889        long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 890
 891        /* Wait for a connection to finish. One exception is TCP Fast Open
 892         * (passive side) where data is allowed to be sent before a connection
 893         * is fully established.
 894         */
 895        if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
 896            !tcp_passive_fastopen(sk)) {
 897                err = sk_stream_wait_connect(sk, &timeo);
 898                if (err != 0)
 899                        goto out_err;
 900        }
 901
 902        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 903
 904        mss_now = tcp_send_mss(sk, &size_goal, flags);
 905        copied = 0;
 906
 907        err = -EPIPE;
 908        if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 909                goto out_err;
 910
 911        while (size > 0) {
 912                struct sk_buff *skb = tcp_write_queue_tail(sk);
 913                int copy, i;
 914                bool can_coalesce;
 915
 916                if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0 ||
 917                    !tcp_skb_can_collapse_to(skb)) {
 918new_segment:
 919                        if (!sk_stream_memory_free(sk))
 920                                goto wait_for_sndbuf;
 921
 922                        skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
 923                                                  skb_queue_empty(&sk->sk_write_queue));
 924                        if (!skb)
 925                                goto wait_for_memory;
 926
 927                        skb_entail(sk, skb);
 928                        copy = size_goal;
 929                }
 930
 931                if (copy > size)
 932                        copy = size;
 933
 934                i = skb_shinfo(skb)->nr_frags;
 935                can_coalesce = skb_can_coalesce(skb, i, page, offset);
 936                if (!can_coalesce && i >= sysctl_max_skb_frags) {
 937                        tcp_mark_push(tp, skb);
 938                        goto new_segment;
 939                }
 940                if (!sk_wmem_schedule(sk, copy))
 941                        goto wait_for_memory;
 942
 943                if (can_coalesce) {
 944                        skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
 945                } else {
 946                        get_page(page);
 947                        skb_fill_page_desc(skb, i, page, offset, copy);
 948                }
 949                skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
 950
 951                skb->len += copy;
 952                skb->data_len += copy;
 953                skb->truesize += copy;
 954                sk->sk_wmem_queued += copy;
 955                sk_mem_charge(sk, copy);
 956                skb->ip_summed = CHECKSUM_PARTIAL;
 957                tp->write_seq += copy;
 958                TCP_SKB_CB(skb)->end_seq += copy;
 959                tcp_skb_pcount_set(skb, 0);
 960
 961                if (!copied)
 962                        TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
 963
 964                copied += copy;
 965                offset += copy;
 966                size -= copy;
 967                if (!size) {
 968                        tcp_tx_timestamp(sk, sk->sk_tsflags, skb);
 969                        goto out;
 970                }
 971
 972                if (skb->len < size_goal || (flags & MSG_OOB))
 973                        continue;
 974
 975                if (forced_push(tp)) {
 976                        tcp_mark_push(tp, skb);
 977                        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 978                } else if (skb == tcp_send_head(sk))
 979                        tcp_push_one(sk, mss_now);
 980                continue;
 981
 982wait_for_sndbuf:
 983                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 984wait_for_memory:
 985                tcp_push(sk, flags & ~MSG_MORE, mss_now,
 986                         TCP_NAGLE_PUSH, size_goal);
 987
 988                err = sk_stream_wait_memory(sk, &timeo);
 989                if (err != 0)
 990                        goto do_error;
 991
 992                mss_now = tcp_send_mss(sk, &size_goal, flags);
 993        }
 994
 995out:
 996        if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
 997                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
 998        return copied;
 999
1000do_error:
1001        if (copied)
1002                goto out;
1003out_err:
1004        /* make sure we wake any epoll edge trigger waiter */
1005        if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 &&
1006                     err == -EAGAIN)) {
1007                sk->sk_write_space(sk);
1008                tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1009        }
1010        return sk_stream_error(sk, flags, err);
1011}
1012
1013int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1014                 size_t size, int flags)
1015{
1016        ssize_t res;
1017
1018        if (!(sk->sk_route_caps & NETIF_F_SG) ||
1019            !sk_check_csum_caps(sk))
1020                return sock_no_sendpage(sk->sk_socket, page, offset, size,
1021                                        flags);
1022
1023        lock_sock(sk);
1024
1025        tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1026
1027        res = do_tcp_sendpages(sk, page, offset, size, flags);
1028        release_sock(sk);
1029        return res;
1030}
1031EXPORT_SYMBOL(tcp_sendpage);
1032
1033/* Do not bother using a page frag for very small frames.
1034 * But use this heuristic only for the first skb in write queue.
1035 *
1036 * Having no payload in skb->head allows better SACK shifting
1037 * in tcp_shift_skb_data(), reducing sack/rack overhead, because
1038 * write queue has less skbs.
1039 * Each skb can hold up to MAX_SKB_FRAGS * 32Kbytes, or ~0.5 MB.
1040 * This also speeds up tso_fragment(), since it wont fallback
1041 * to tcp_fragment().
1042 */
1043static int linear_payload_sz(bool first_skb)
1044{
1045        if (first_skb)
1046                return SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
1047        return 0;
1048}
1049
1050static int select_size(const struct sock *sk, bool sg, bool first_skb)
1051{
1052        const struct tcp_sock *tp = tcp_sk(sk);
1053        int tmp = tp->mss_cache;
1054
1055        if (sg) {
1056                if (sk_can_gso(sk)) {
1057                        tmp = linear_payload_sz(first_skb);
1058                } else {
1059                        int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
1060
1061                        if (tmp >= pgbreak &&
1062                            tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
1063                                tmp = pgbreak;
1064                }
1065        }
1066
1067        return tmp;
1068}
1069
1070void tcp_free_fastopen_req(struct tcp_sock *tp)
1071{
1072        if (tp->fastopen_req) {
1073                kfree(tp->fastopen_req);
1074                tp->fastopen_req = NULL;
1075        }
1076}
1077
1078static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1079                                int *copied, size_t size)
1080{
1081        struct tcp_sock *tp = tcp_sk(sk);
1082        int err, flags;
1083
1084        if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE))
1085                return -EOPNOTSUPP;
1086        if (tp->fastopen_req)
1087                return -EALREADY; /* Another Fast Open is in progress */
1088
1089        tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1090                                   sk->sk_allocation);
1091        if (unlikely(!tp->fastopen_req))
1092                return -ENOBUFS;
1093        tp->fastopen_req->data = msg;
1094        tp->fastopen_req->size = size;
1095
1096        flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1097        err = __inet_stream_connect(sk->sk_socket, msg->msg_name,
1098                                    msg->msg_namelen, flags);
1099        *copied = tp->fastopen_req->copied;
1100        tcp_free_fastopen_req(tp);
1101        return err;
1102}
1103
1104int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1105{
1106        struct tcp_sock *tp = tcp_sk(sk);
1107        struct sk_buff *skb;
1108        struct sockcm_cookie sockc;
1109        int flags, err, copied = 0;
1110        int mss_now = 0, size_goal, copied_syn = 0;
1111        bool process_backlog = false;
1112        bool sg;
1113        long timeo;
1114
1115        lock_sock(sk);
1116
1117        flags = msg->msg_flags;
1118        if (flags & MSG_FASTOPEN) {
1119                err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
1120                if (err == -EINPROGRESS && copied_syn > 0)
1121                        goto out;
1122                else if (err)
1123                        goto out_err;
1124        }
1125
1126        timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1127
1128        tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1129
1130        /* Wait for a connection to finish. One exception is TCP Fast Open
1131         * (passive side) where data is allowed to be sent before a connection
1132         * is fully established.
1133         */
1134        if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1135            !tcp_passive_fastopen(sk)) {
1136                err = sk_stream_wait_connect(sk, &timeo);
1137                if (err != 0)
1138                        goto do_error;
1139        }
1140
1141        if (unlikely(tp->repair)) {
1142                if (tp->repair_queue == TCP_RECV_QUEUE) {
1143                        copied = tcp_send_rcvq(sk, msg, size);
1144                        goto out_nopush;
1145                }
1146
1147                err = -EINVAL;
1148                if (tp->repair_queue == TCP_NO_QUEUE)
1149                        goto out_err;
1150
1151                /* 'common' sending to sendq */
1152        }
1153
1154        sockc.tsflags = sk->sk_tsflags;
1155        if (msg->msg_controllen) {
1156                err = sock_cmsg_send(sk, msg, &sockc);
1157                if (unlikely(err)) {
1158                        err = -EINVAL;
1159                        goto out_err;
1160                }
1161        }
1162
1163        /* This should be in poll */
1164        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1165
1166        /* Ok commence sending. */
1167        copied = 0;
1168
1169restart:
1170        mss_now = tcp_send_mss(sk, &size_goal, flags);
1171
1172        err = -EPIPE;
1173        if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1174                goto do_error;
1175
1176        sg = !!(sk->sk_route_caps & NETIF_F_SG);
1177
1178        while (msg_data_left(msg)) {
1179                int copy = 0;
1180                int max = size_goal;
1181
1182                skb = tcp_write_queue_tail(sk);
1183                if (tcp_send_head(sk)) {
1184                        if (skb->ip_summed == CHECKSUM_NONE)
1185                                max = mss_now;
1186                        copy = max - skb->len;
1187                }
1188
1189                if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1190                        bool first_skb;
1191
1192new_segment:
1193                        /* Allocate new segment. If the interface is SG,
1194                         * allocate skb fitting to single page.
1195                         */
1196                        if (!sk_stream_memory_free(sk))
1197                                goto wait_for_sndbuf;
1198
1199                        if (process_backlog && sk_flush_backlog(sk)) {
1200                                process_backlog = false;
1201                                goto restart;
1202                        }
1203                        first_skb = skb_queue_empty(&sk->sk_write_queue);
1204                        skb = sk_stream_alloc_skb(sk,
1205                                                  select_size(sk, sg, first_skb),
1206                                                  sk->sk_allocation,
1207                                                  first_skb);
1208                        if (!skb)
1209                                goto wait_for_memory;
1210
1211                        process_backlog = true;
1212                        /*
1213                         * Check whether we can use HW checksum.
1214                         */
1215                        if (sk_check_csum_caps(sk))
1216                                skb->ip_summed = CHECKSUM_PARTIAL;
1217
1218                        skb_entail(sk, skb);
1219                        copy = size_goal;
1220                        max = size_goal;
1221
1222                        /* All packets are restored as if they have
1223                         * already been sent. skb_mstamp isn't set to
1224                         * avoid wrong rtt estimation.
1225                         */
1226                        if (tp->repair)
1227                                TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1228                }
1229
1230                /* Try to append data to the end of skb. */
1231                if (copy > msg_data_left(msg))
1232                        copy = msg_data_left(msg);
1233
1234                /* Where to copy to? */
1235                if (skb_availroom(skb) > 0) {
1236                        /* We have some space in skb head. Superb! */
1237                        copy = min_t(int, copy, skb_availroom(skb));
1238                        err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1239                        if (err)
1240                                goto do_fault;
1241                } else {
1242                        bool merge = true;
1243                        int i = skb_shinfo(skb)->nr_frags;
1244                        struct page_frag *pfrag = sk_page_frag(sk);
1245
1246                        if (!sk_page_frag_refill(sk, pfrag))
1247                                goto wait_for_memory;
1248
1249                        if (!skb_can_coalesce(skb, i, pfrag->page,
1250                                              pfrag->offset)) {
1251                                if (i >= sysctl_max_skb_frags || !sg) {
1252                                        tcp_mark_push(tp, skb);
1253                                        goto new_segment;
1254                                }
1255                                merge = false;
1256                        }
1257
1258                        copy = min_t(int, copy, pfrag->size - pfrag->offset);
1259
1260                        if (!sk_wmem_schedule(sk, copy))
1261                                goto wait_for_memory;
1262
1263                        err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1264                                                       pfrag->page,
1265                                                       pfrag->offset,
1266                                                       copy);
1267                        if (err)
1268                                goto do_error;
1269
1270                        /* Update the skb. */
1271                        if (merge) {
1272                                skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1273                        } else {
1274                                skb_fill_page_desc(skb, i, pfrag->page,
1275                                                   pfrag->offset, copy);
1276                                get_page(pfrag->page);
1277                        }
1278                        pfrag->offset += copy;
1279                }
1280
1281                if (!copied)
1282                        TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1283
1284                tp->write_seq += copy;
1285                TCP_SKB_CB(skb)->end_seq += copy;
1286                tcp_skb_pcount_set(skb, 0);
1287
1288                copied += copy;
1289                if (!msg_data_left(msg)) {
1290                        tcp_tx_timestamp(sk, sockc.tsflags, skb);
1291                        if (unlikely(flags & MSG_EOR))
1292                                TCP_SKB_CB(skb)->eor = 1;
1293                        goto out;
1294                }
1295
1296                if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))
1297                        continue;
1298
1299                if (forced_push(tp)) {
1300                        tcp_mark_push(tp, skb);
1301                        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1302                } else if (skb == tcp_send_head(sk))
1303                        tcp_push_one(sk, mss_now);
1304                continue;
1305
1306wait_for_sndbuf:
1307                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1308wait_for_memory:
1309                if (copied)
1310                        tcp_push(sk, flags & ~MSG_MORE, mss_now,
1311                                 TCP_NAGLE_PUSH, size_goal);
1312
1313                err = sk_stream_wait_memory(sk, &timeo);
1314                if (err != 0)
1315                        goto do_error;
1316
1317                mss_now = tcp_send_mss(sk, &size_goal, flags);
1318        }
1319
1320out:
1321        if (copied)
1322                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1323out_nopush:
1324        release_sock(sk);
1325        return copied + copied_syn;
1326
1327do_fault:
1328        if (!skb->len) {
1329                tcp_unlink_write_queue(skb, sk);
1330                /* It is the one place in all of TCP, except connection
1331                 * reset, where we can be unlinking the send_head.
1332                 */
1333                tcp_check_send_head(sk, skb);
1334                sk_wmem_free_skb(sk, skb);
1335        }
1336
1337do_error:
1338        if (copied + copied_syn)
1339                goto out;
1340out_err:
1341        err = sk_stream_error(sk, flags, err);
1342        /* make sure we wake any epoll edge trigger waiter */
1343        if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 &&
1344                     err == -EAGAIN)) {
1345                sk->sk_write_space(sk);
1346                tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1347        }
1348        release_sock(sk);
1349        return err;
1350}
1351EXPORT_SYMBOL(tcp_sendmsg);
1352
1353/*
1354 *      Handle reading urgent data. BSD has very simple semantics for
1355 *      this, no blocking and very strange errors 8)
1356 */
1357
1358static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1359{
1360        struct tcp_sock *tp = tcp_sk(sk);
1361
1362        /* No URG data to read. */
1363        if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1364            tp->urg_data == TCP_URG_READ)
1365                return -EINVAL; /* Yes this is right ! */
1366
1367        if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1368                return -ENOTCONN;
1369
1370        if (tp->urg_data & TCP_URG_VALID) {
1371                int err = 0;
1372                char c = tp->urg_data;
1373
1374                if (!(flags & MSG_PEEK))
1375                        tp->urg_data = TCP_URG_READ;
1376
1377                /* Read urgent data. */
1378                msg->msg_flags |= MSG_OOB;
1379
1380                if (len > 0) {
1381                        if (!(flags & MSG_TRUNC))
1382                                err = memcpy_to_msg(msg, &c, 1);
1383                        len = 1;
1384                } else
1385                        msg->msg_flags |= MSG_TRUNC;
1386
1387                return err ? -EFAULT : len;
1388        }
1389
1390        if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1391                return 0;
1392
1393        /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1394         * the available implementations agree in this case:
1395         * this call should never block, independent of the
1396         * blocking state of the socket.
1397         * Mike <pall@rz.uni-karlsruhe.de>
1398         */
1399        return -EAGAIN;
1400}
1401
1402static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1403{
1404        struct sk_buff *skb;
1405        int copied = 0, err = 0;
1406
1407        /* XXX -- need to support SO_PEEK_OFF */
1408
1409        skb_queue_walk(&sk->sk_write_queue, skb) {
1410                err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1411                if (err)
1412                        break;
1413
1414                copied += skb->len;
1415        }
1416
1417        return err ?: copied;
1418}
1419
1420/* Clean up the receive buffer for full frames taken by the user,
1421 * then send an ACK if necessary.  COPIED is the number of bytes
1422 * tcp_recvmsg has given to the user so far, it speeds up the
1423 * calculation of whether or not we must ACK for the sake of
1424 * a window update.
1425 */
1426static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1427{
1428        struct tcp_sock *tp = tcp_sk(sk);
1429        bool time_to_ack = false;
1430
1431        struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1432
1433        WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1434             "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1435             tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1436
1437        if (inet_csk_ack_scheduled(sk)) {
1438                const struct inet_connection_sock *icsk = inet_csk(sk);
1439                   /* Delayed ACKs frequently hit locked sockets during bulk
1440                    * receive. */
1441                if (icsk->icsk_ack.blocked ||
1442                    /* Once-per-two-segments ACK was not sent by tcp_input.c */
1443                    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1444                    /*
1445                     * If this read emptied read buffer, we send ACK, if
1446                     * connection is not bidirectional, user drained
1447                     * receive buffer and there was a small segment
1448                     * in queue.
1449                     */
1450                    (copied > 0 &&
1451                     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1452                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1453                       !icsk->icsk_ack.pingpong)) &&
1454                      !atomic_read(&sk->sk_rmem_alloc)))
1455                        time_to_ack = true;
1456        }
1457
1458        /* We send an ACK if we can now advertise a non-zero window
1459         * which has been raised "significantly".
1460         *
1461         * Even if window raised up to infinity, do not send window open ACK
1462         * in states, where we will not receive more. It is useless.
1463         */
1464        if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1465                __u32 rcv_window_now = tcp_receive_window(tp);
1466
1467                /* Optimize, __tcp_select_window() is not cheap. */
1468                if (2*rcv_window_now <= tp->window_clamp) {
1469                        __u32 new_window = __tcp_select_window(sk);
1470
1471                        /* Send ACK now, if this read freed lots of space
1472                         * in our buffer. Certainly, new_window is new window.
1473                         * We can advertise it now, if it is not less than current one.
1474                         * "Lots" means "at least twice" here.
1475                         */
1476                        if (new_window && new_window >= 2 * rcv_window_now)
1477                                time_to_ack = true;
1478                }
1479        }
1480        if (time_to_ack)
1481                tcp_send_ack(sk);
1482}
1483
1484static void tcp_prequeue_process(struct sock *sk)
1485{
1486        struct sk_buff *skb;
1487        struct tcp_sock *tp = tcp_sk(sk);
1488
1489        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1490
1491        while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1492                sk_backlog_rcv(sk, skb);
1493
1494        /* Clear memory counter. */
1495        tp->ucopy.memory = 0;
1496}
1497
1498static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1499{
1500        struct sk_buff *skb;
1501        u32 offset;
1502
1503        while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1504                offset = seq - TCP_SKB_CB(skb)->seq;
1505                if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1506                        pr_err_once("%s: found a SYN, please report !\n", __func__);
1507                        offset--;
1508                }
1509                if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1510                        *off = offset;
1511                        return skb;
1512                }
1513                /* This looks weird, but this can happen if TCP collapsing
1514                 * splitted a fat GRO packet, while we released socket lock
1515                 * in skb_splice_bits()
1516                 */
1517                sk_eat_skb(sk, skb);
1518        }
1519        return NULL;
1520}
1521
1522/*
1523 * This routine provides an alternative to tcp_recvmsg() for routines
1524 * that would like to handle copying from skbuffs directly in 'sendfile'
1525 * fashion.
1526 * Note:
1527 *      - It is assumed that the socket was locked by the caller.
1528 *      - The routine does not block.
1529 *      - At present, there is no support for reading OOB data
1530 *        or for 'peeking' the socket using this routine
1531 *        (although both would be easy to implement).
1532 */
1533int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1534                  sk_read_actor_t recv_actor)
1535{
1536        struct sk_buff *skb;
1537        struct tcp_sock *tp = tcp_sk(sk);
1538        u32 seq = tp->copied_seq;
1539        u32 offset;
1540        int copied = 0;
1541
1542        if (sk->sk_state == TCP_LISTEN)
1543                return -ENOTCONN;
1544        while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1545                if (offset < skb->len) {
1546                        int used;
1547                        size_t len;
1548
1549                        len = skb->len - offset;
1550                        /* Stop reading if we hit a patch of urgent data */
1551                        if (tp->urg_data) {
1552                                u32 urg_offset = tp->urg_seq - seq;
1553                                if (urg_offset < len)
1554                                        len = urg_offset;
1555                                if (!len)
1556                                        break;
1557                        }
1558                        used = recv_actor(desc, skb, offset, len);
1559                        if (used <= 0) {
1560                                if (!copied)
1561                                        copied = used;
1562                                break;
1563                        } else if (used <= len) {
1564                                seq += used;
1565                                copied += used;
1566                                offset += used;
1567                        }
1568                        /* If recv_actor drops the lock (e.g. TCP splice
1569                         * receive) the skb pointer might be invalid when
1570                         * getting here: tcp_collapse might have deleted it
1571                         * while aggregating skbs from the socket queue.
1572                         */
1573                        skb = tcp_recv_skb(sk, seq - 1, &offset);
1574                        if (!skb)
1575                                break;
1576                        /* TCP coalescing might have appended data to the skb.
1577                         * Try to splice more frags
1578                         */
1579                        if (offset + 1 != skb->len)
1580                                continue;
1581                }
1582                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1583                        sk_eat_skb(sk, skb);
1584                        ++seq;
1585                        break;
1586                }
1587                sk_eat_skb(sk, skb);
1588                if (!desc->count)
1589                        break;
1590                tp->copied_seq = seq;
1591        }
1592        tp->copied_seq = seq;
1593
1594        tcp_rcv_space_adjust(sk);
1595
1596        /* Clean up data we have read: This will do ACK frames. */
1597        if (copied > 0) {
1598                tcp_recv_skb(sk, seq, &offset);
1599                tcp_cleanup_rbuf(sk, copied);
1600        }
1601        return copied;
1602}
1603EXPORT_SYMBOL(tcp_read_sock);
1604
1605int tcp_peek_len(struct socket *sock)
1606{
1607        return tcp_inq(sock->sk);
1608}
1609EXPORT_SYMBOL(tcp_peek_len);
1610
1611/*
1612 *      This routine copies from a sock struct into the user buffer.
1613 *
1614 *      Technical note: in 2.3 we work on _locked_ socket, so that
1615 *      tricks with *seq access order and skb->users are not required.
1616 *      Probably, code can be easily improved even more.
1617 */
1618
1619int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
1620                int flags, int *addr_len)
1621{
1622        struct tcp_sock *tp = tcp_sk(sk);
1623        int copied = 0;
1624        u32 peek_seq;
1625        u32 *seq;
1626        unsigned long used;
1627        int err;
1628        int target;             /* Read at least this many bytes */
1629        long timeo;
1630        struct task_struct *user_recv = NULL;
1631        struct sk_buff *skb, *last;
1632        u32 urg_hole = 0;
1633
1634        if (unlikely(flags & MSG_ERRQUEUE))
1635                return inet_recv_error(sk, msg, len, addr_len);
1636
1637        if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1638            (sk->sk_state == TCP_ESTABLISHED))
1639                sk_busy_loop(sk, nonblock);
1640
1641        lock_sock(sk);
1642
1643        err = -ENOTCONN;
1644        if (sk->sk_state == TCP_LISTEN)
1645                goto out;
1646
1647        timeo = sock_rcvtimeo(sk, nonblock);
1648
1649        /* Urgent data needs to be handled specially. */
1650        if (flags & MSG_OOB)
1651                goto recv_urg;
1652
1653        if (unlikely(tp->repair)) {
1654                err = -EPERM;
1655                if (!(flags & MSG_PEEK))
1656                        goto out;
1657
1658                if (tp->repair_queue == TCP_SEND_QUEUE)
1659                        goto recv_sndq;
1660
1661                err = -EINVAL;
1662                if (tp->repair_queue == TCP_NO_QUEUE)
1663                        goto out;
1664
1665                /* 'common' recv queue MSG_PEEK-ing */
1666        }
1667
1668        seq = &tp->copied_seq;
1669        if (flags & MSG_PEEK) {
1670                peek_seq = tp->copied_seq;
1671                seq = &peek_seq;
1672        }
1673
1674        target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1675
1676        do {
1677                u32 offset;
1678
1679                /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1680                if (tp->urg_data && tp->urg_seq == *seq) {
1681                        if (copied)
1682                                break;
1683                        if (signal_pending(current)) {
1684                                copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1685                                break;
1686                        }
1687                }
1688
1689                /* Next get a buffer. */
1690
1691                last = skb_peek_tail(&sk->sk_receive_queue);
1692                skb_queue_walk(&sk->sk_receive_queue, skb) {
1693                        last = skb;
1694                        /* Now that we have two receive queues this
1695                         * shouldn't happen.
1696                         */
1697                        if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1698                                 "recvmsg bug: copied %X seq %X rcvnxt %X fl %X\n",
1699                                 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1700                                 flags))
1701                                break;
1702
1703                        offset = *seq - TCP_SKB_CB(skb)->seq;
1704                        if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1705                                pr_err_once("%s: found a SYN, please report !\n", __func__);
1706                                offset--;
1707                        }
1708                        if (offset < skb->len)
1709                                goto found_ok_skb;
1710                        if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1711                                goto found_fin_ok;
1712                        WARN(!(flags & MSG_PEEK),
1713                             "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
1714                             *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1715                }
1716
1717                /* Well, if we have backlog, try to process it now yet. */
1718
1719                if (copied >= target && !sk->sk_backlog.tail)
1720                        break;
1721
1722                if (copied) {
1723                        if (sk->sk_err ||
1724                            sk->sk_state == TCP_CLOSE ||
1725                            (sk->sk_shutdown & RCV_SHUTDOWN) ||
1726                            !timeo ||
1727                            signal_pending(current))
1728                                break;
1729                } else {
1730                        if (sock_flag(sk, SOCK_DONE))
1731                                break;
1732
1733                        if (sk->sk_err) {
1734                                copied = sock_error(sk);
1735                                break;
1736                        }
1737
1738                        if (sk->sk_shutdown & RCV_SHUTDOWN)
1739                                break;
1740
1741                        if (sk->sk_state == TCP_CLOSE) {
1742                                if (!sock_flag(sk, SOCK_DONE)) {
1743                                        /* This occurs when user tries to read
1744                                         * from never connected socket.
1745                                         */
1746                                        copied = -ENOTCONN;
1747                                        break;
1748                                }
1749                                break;
1750                        }
1751
1752                        if (!timeo) {
1753                                copied = -EAGAIN;
1754                                break;
1755                        }
1756
1757                        if (signal_pending(current)) {
1758                                copied = sock_intr_errno(timeo);
1759                                break;
1760                        }
1761                }
1762
1763                tcp_cleanup_rbuf(sk, copied);
1764
1765                if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1766                        /* Install new reader */
1767                        if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1768                                user_recv = current;
1769                                tp->ucopy.task = user_recv;
1770                                tp->ucopy.msg = msg;
1771                        }
1772
1773                        tp->ucopy.len = len;
1774
1775                        WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1776                                !(flags & (MSG_PEEK | MSG_TRUNC)));
1777
1778                        /* Ugly... If prequeue is not empty, we have to
1779                         * process it before releasing socket, otherwise
1780                         * order will be broken at second iteration.
1781                         * More elegant solution is required!!!
1782                         *
1783                         * Look: we have the following (pseudo)queues:
1784                         *
1785                         * 1. packets in flight
1786                         * 2. backlog
1787                         * 3. prequeue
1788                         * 4. receive_queue
1789                         *
1790                         * Each queue can be processed only if the next ones
1791                         * are empty. At this point we have empty receive_queue.
1792                         * But prequeue _can_ be not empty after 2nd iteration,
1793                         * when we jumped to start of loop because backlog
1794                         * processing added something to receive_queue.
1795                         * We cannot release_sock(), because backlog contains
1796                         * packets arrived _after_ prequeued ones.
1797                         *
1798                         * Shortly, algorithm is clear --- to process all
1799                         * the queues in order. We could make it more directly,
1800                         * requeueing packets from backlog to prequeue, if
1801                         * is not empty. It is more elegant, but eats cycles,
1802                         * unfortunately.
1803                         */
1804                        if (!skb_queue_empty(&tp->ucopy.prequeue))
1805                                goto do_prequeue;
1806
1807                        /* __ Set realtime policy in scheduler __ */
1808                }
1809
1810                if (copied >= target) {
1811                        /* Do not sleep, just process backlog. */
1812                        release_sock(sk);
1813                        lock_sock(sk);
1814                } else {
1815                        sk_wait_data(sk, &timeo, last);
1816                }
1817
1818                if (user_recv) {
1819                        int chunk;
1820
1821                        /* __ Restore normal policy in scheduler __ */
1822
1823                        chunk = len - tp->ucopy.len;
1824                        if (chunk != 0) {
1825                                NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1826                                len -= chunk;
1827                                copied += chunk;
1828                        }
1829
1830                        if (tp->rcv_nxt == tp->copied_seq &&
1831                            !skb_queue_empty(&tp->ucopy.prequeue)) {
1832do_prequeue:
1833                                tcp_prequeue_process(sk);
1834
1835                                chunk = len - tp->ucopy.len;
1836                                if (chunk != 0) {
1837                                        NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1838                                        len -= chunk;
1839                                        copied += chunk;
1840                                }
1841                        }
1842                }
1843                if ((flags & MSG_PEEK) &&
1844                    (peek_seq - copied - urg_hole != tp->copied_seq)) {
1845                        net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
1846                                            current->comm,
1847                                            task_pid_nr(current));
1848                        peek_seq = tp->copied_seq;
1849                }
1850                continue;
1851
1852        found_ok_skb:
1853                /* Ok so how much can we use? */
1854                used = skb->len - offset;
1855                if (len < used)
1856                        used = len;
1857
1858                /* Do we have urgent data here? */
1859                if (tp->urg_data) {
1860                        u32 urg_offset = tp->urg_seq - *seq;
1861                        if (urg_offset < used) {
1862                                if (!urg_offset) {
1863                                        if (!sock_flag(sk, SOCK_URGINLINE)) {
1864                                                ++*seq;
1865                                                urg_hole++;
1866                                                offset++;
1867                                                used--;
1868                                                if (!used)
1869                                                        goto skip_copy;
1870                                        }
1871                                } else
1872                                        used = urg_offset;
1873                        }
1874                }
1875
1876                if (!(flags & MSG_TRUNC)) {
1877                        err = skb_copy_datagram_msg(skb, offset, msg, used);
1878                        if (err) {
1879                                /* Exception. Bailout! */
1880                                if (!copied)
1881                                        copied = -EFAULT;
1882                                break;
1883                        }
1884                }
1885
1886                *seq += used;
1887                copied += used;
1888                len -= used;
1889
1890                tcp_rcv_space_adjust(sk);
1891
1892skip_copy:
1893                if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1894                        tp->urg_data = 0;
1895                        tcp_fast_path_check(sk);
1896                }
1897                if (used + offset < skb->len)
1898                        continue;
1899
1900                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1901                        goto found_fin_ok;
1902                if (!(flags & MSG_PEEK))
1903                        sk_eat_skb(sk, skb);
1904                continue;
1905
1906        found_fin_ok:
1907                /* Process the FIN. */
1908                ++*seq;
1909                if (!(flags & MSG_PEEK))
1910                        sk_eat_skb(sk, skb);
1911                break;
1912        } while (len > 0);
1913
1914        if (user_recv) {
1915                if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1916                        int chunk;
1917
1918                        tp->ucopy.len = copied > 0 ? len : 0;
1919
1920                        tcp_prequeue_process(sk);
1921
1922                        if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1923                                NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1924                                len -= chunk;
1925                                copied += chunk;
1926                        }
1927                }
1928
1929                tp->ucopy.task = NULL;
1930                tp->ucopy.len = 0;
1931        }
1932
1933        /* According to UNIX98, msg_name/msg_namelen are ignored
1934         * on connected socket. I was just happy when found this 8) --ANK
1935         */
1936
1937        /* Clean up data we have read: This will do ACK frames. */
1938        tcp_cleanup_rbuf(sk, copied);
1939
1940        release_sock(sk);
1941        return copied;
1942
1943out:
1944        release_sock(sk);
1945        return err;
1946
1947recv_urg:
1948        err = tcp_recv_urg(sk, msg, len, flags);
1949        goto out;
1950
1951recv_sndq:
1952        err = tcp_peek_sndq(sk, msg, len);
1953        goto out;
1954}
1955EXPORT_SYMBOL(tcp_recvmsg);
1956
1957void tcp_set_state(struct sock *sk, int state)
1958{
1959        int oldstate = sk->sk_state;
1960
1961        switch (state) {
1962        case TCP_ESTABLISHED:
1963                if (oldstate != TCP_ESTABLISHED)
1964                        TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1965                break;
1966
1967        case TCP_CLOSE:
1968                if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1969                        TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1970
1971                sk->sk_prot->unhash(sk);
1972                if (inet_csk(sk)->icsk_bind_hash &&
1973                    !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1974                        inet_put_port(sk);
1975                /* fall through */
1976        default:
1977                if (oldstate == TCP_ESTABLISHED)
1978                        TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1979        }
1980
1981        /* Change state AFTER socket is unhashed to avoid closed
1982         * socket sitting in hash tables.
1983         */
1984        sk_state_store(sk, state);
1985
1986#ifdef STATE_TRACE
1987        SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1988#endif
1989}
1990EXPORT_SYMBOL_GPL(tcp_set_state);
1991
1992/*
1993 *      State processing on a close. This implements the state shift for
1994 *      sending our FIN frame. Note that we only send a FIN for some
1995 *      states. A shutdown() may have already sent the FIN, or we may be
1996 *      closed.
1997 */
1998
1999static const unsigned char new_state[16] = {
2000  /* current state:        new state:      action:      */
2001  [0 /* (Invalid) */]   = TCP_CLOSE,
2002  [TCP_ESTABLISHED]     = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2003  [TCP_SYN_SENT]        = TCP_CLOSE,
2004  [TCP_SYN_RECV]        = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2005  [TCP_FIN_WAIT1]       = TCP_FIN_WAIT1,
2006  [TCP_FIN_WAIT2]       = TCP_FIN_WAIT2,
2007  [TCP_TIME_WAIT]       = TCP_CLOSE,
2008  [TCP_CLOSE]           = TCP_CLOSE,
2009  [TCP_CLOSE_WAIT]      = TCP_LAST_ACK  | TCP_ACTION_FIN,
2010  [TCP_LAST_ACK]        = TCP_LAST_ACK,
2011  [TCP_LISTEN]          = TCP_CLOSE,
2012  [TCP_CLOSING]         = TCP_CLOSING,
2013  [TCP_NEW_SYN_RECV]    = TCP_CLOSE,    /* should not happen ! */
2014};
2015
2016static int tcp_close_state(struct sock *sk)
2017{
2018        int next = (int)new_state[sk->sk_state];
2019        int ns = next & TCP_STATE_MASK;
2020
2021        tcp_set_state(sk, ns);
2022
2023        return next & TCP_ACTION_FIN;
2024}
2025
2026/*
2027 *      Shutdown the sending side of a connection. Much like close except
2028 *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
2029 */
2030
2031void tcp_shutdown(struct sock *sk, int how)
2032{
2033        /*      We need to grab some memory, and put together a FIN,
2034         *      and then put it into the queue to be sent.
2035         *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2036         */
2037        if (!(how & SEND_SHUTDOWN))
2038                return;
2039
2040        /* If we've already sent a FIN, or it's a closed state, skip this. */
2041        if ((1 << sk->sk_state) &
2042            (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2043             TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2044                /* Clear out any half completed packets.  FIN if needed. */
2045                if (tcp_close_state(sk))
2046                        tcp_send_fin(sk);
2047        }
2048}
2049EXPORT_SYMBOL(tcp_shutdown);
2050
2051bool tcp_check_oom(struct sock *sk, int shift)
2052{
2053        bool too_many_orphans, out_of_socket_memory;
2054
2055        too_many_orphans = tcp_too_many_orphans(sk, shift);
2056        out_of_socket_memory = tcp_out_of_memory(sk);
2057
2058        if (too_many_orphans)
2059                net_info_ratelimited("too many orphaned sockets\n");
2060        if (out_of_socket_memory)
2061                net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2062        return too_many_orphans || out_of_socket_memory;
2063}
2064
2065void tcp_close(struct sock *sk, long timeout)
2066{
2067        struct sk_buff *skb;
2068        int data_was_unread = 0;
2069        int state;
2070
2071        lock_sock(sk);
2072        sk->sk_shutdown = SHUTDOWN_MASK;
2073
2074        if (sk->sk_state == TCP_LISTEN) {
2075                tcp_set_state(sk, TCP_CLOSE);
2076
2077                /* Special case. */
2078                inet_csk_listen_stop(sk);
2079
2080                goto adjudge_to_death;
2081        }
2082
2083        /*  We need to flush the recv. buffs.  We do this only on the
2084         *  descriptor close, not protocol-sourced closes, because the
2085         *  reader process may not have drained the data yet!
2086         */
2087        while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2088                u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2089
2090                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2091                        len--;
2092                data_was_unread += len;
2093                __kfree_skb(skb);
2094        }
2095
2096        sk_mem_reclaim(sk);
2097
2098        /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2099        if (sk->sk_state == TCP_CLOSE)
2100                goto adjudge_to_death;
2101
2102        /* As outlined in RFC 2525, section 2.17, we send a RST here because
2103         * data was lost. To witness the awful effects of the old behavior of
2104         * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2105         * GET in an FTP client, suspend the process, wait for the client to
2106         * advertise a zero window, then kill -9 the FTP client, wheee...
2107         * Note: timeout is always zero in such a case.
2108         */
2109        if (unlikely(tcp_sk(sk)->repair)) {
2110                sk->sk_prot->disconnect(sk, 0);
2111        } else if (data_was_unread) {
2112                /* Unread data was tossed, zap the connection. */
2113                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2114                tcp_set_state(sk, TCP_CLOSE);
2115                tcp_send_active_reset(sk, sk->sk_allocation);
2116        } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2117                /* Check zero linger _after_ checking for unread data. */
2118                sk->sk_prot->disconnect(sk, 0);
2119                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2120        } else if (tcp_close_state(sk)) {
2121                /* We FIN if the application ate all the data before
2122                 * zapping the connection.
2123                 */
2124
2125                /* RED-PEN. Formally speaking, we have broken TCP state
2126                 * machine. State transitions:
2127                 *
2128                 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2129                 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
2130                 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2131                 *
2132                 * are legal only when FIN has been sent (i.e. in window),
2133                 * rather than queued out of window. Purists blame.
2134                 *
2135                 * F.e. "RFC state" is ESTABLISHED,
2136                 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2137                 *
2138                 * The visible declinations are that sometimes
2139                 * we enter time-wait state, when it is not required really
2140                 * (harmless), do not send active resets, when they are
2141                 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2142                 * they look as CLOSING or LAST_ACK for Linux)
2143                 * Probably, I missed some more holelets.
2144                 *                                              --ANK
2145                 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2146                 * in a single packet! (May consider it later but will
2147                 * probably need API support or TCP_CORK SYN-ACK until
2148                 * data is written and socket is closed.)
2149                 */
2150                tcp_send_fin(sk);
2151        }
2152
2153        sk_stream_wait_close(sk, timeout);
2154
2155adjudge_to_death:
2156        state = sk->sk_state;
2157        sock_hold(sk);
2158        sock_orphan(sk);
2159
2160        /* It is the last release_sock in its life. It will remove backlog. */
2161        release_sock(sk);
2162
2163
2164        /* Now socket is owned by kernel and we acquire BH lock
2165           to finish close. No need to check for user refs.
2166         */
2167        local_bh_disable();
2168        bh_lock_sock(sk);
2169        WARN_ON(sock_owned_by_user(sk));
2170
2171        percpu_counter_inc(sk->sk_prot->orphan_count);
2172
2173        /* Have we already been destroyed by a softirq or backlog? */
2174        if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2175                goto out;
2176
2177        /*      This is a (useful) BSD violating of the RFC. There is a
2178         *      problem with TCP as specified in that the other end could
2179         *      keep a socket open forever with no application left this end.
2180         *      We use a 1 minute timeout (about the same as BSD) then kill
2181         *      our end. If they send after that then tough - BUT: long enough
2182         *      that we won't make the old 4*rto = almost no time - whoops
2183         *      reset mistake.
2184         *
2185         *      Nope, it was not mistake. It is really desired behaviour
2186         *      f.e. on http servers, when such sockets are useless, but
2187         *      consume significant resources. Let's do it with special
2188         *      linger2 option.                                 --ANK
2189         */
2190
2191        if (sk->sk_state == TCP_FIN_WAIT2) {
2192                struct tcp_sock *tp = tcp_sk(sk);
2193                if (tp->linger2 < 0) {
2194                        tcp_set_state(sk, TCP_CLOSE);
2195                        tcp_send_active_reset(sk, GFP_ATOMIC);
2196                        __NET_INC_STATS(sock_net(sk),
2197                                        LINUX_MIB_TCPABORTONLINGER);
2198                } else {
2199                        const int tmo = tcp_fin_time(sk);
2200
2201                        if (tmo > TCP_TIMEWAIT_LEN) {
2202                                inet_csk_reset_keepalive_timer(sk,
2203                                                tmo - TCP_TIMEWAIT_LEN);
2204                        } else {
2205                                tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2206                                goto out;
2207                        }
2208                }
2209        }
2210        if (sk->sk_state != TCP_CLOSE) {
2211                sk_mem_reclaim(sk);
2212                if (tcp_check_oom(sk, 0)) {
2213                        tcp_set_state(sk, TCP_CLOSE);
2214                        tcp_send_active_reset(sk, GFP_ATOMIC);
2215                        __NET_INC_STATS(sock_net(sk),
2216                                        LINUX_MIB_TCPABORTONMEMORY);
2217                }
2218        }
2219
2220        if (sk->sk_state == TCP_CLOSE) {
2221                struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
2222                /* We could get here with a non-NULL req if the socket is
2223                 * aborted (e.g., closed with unread data) before 3WHS
2224                 * finishes.
2225                 */
2226                if (req)
2227                        reqsk_fastopen_remove(sk, req, false);
2228                inet_csk_destroy_sock(sk);
2229        }
2230        /* Otherwise, socket is reprieved until protocol close. */
2231
2232out:
2233        bh_unlock_sock(sk);
2234        local_bh_enable();
2235        sock_put(sk);
2236}
2237EXPORT_SYMBOL(tcp_close);
2238
2239/* These states need RST on ABORT according to RFC793 */
2240
2241static inline bool tcp_need_reset(int state)
2242{
2243        return (1 << state) &
2244               (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2245                TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2246}
2247
2248int tcp_disconnect(struct sock *sk, int flags)
2249{
2250        struct inet_sock *inet = inet_sk(sk);
2251        struct inet_connection_sock *icsk = inet_csk(sk);
2252        struct tcp_sock *tp = tcp_sk(sk);
2253        int err = 0;
2254        int old_state = sk->sk_state;
2255
2256        if (old_state != TCP_CLOSE)
2257                tcp_set_state(sk, TCP_CLOSE);
2258
2259        /* ABORT function of RFC793 */
2260        if (old_state == TCP_LISTEN) {
2261                inet_csk_listen_stop(sk);
2262        } else if (unlikely(tp->repair)) {
2263                sk->sk_err = ECONNABORTED;
2264        } else if (tcp_need_reset(old_state) ||
2265                   (tp->snd_nxt != tp->write_seq &&
2266                    (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2267                /* The last check adjusts for discrepancy of Linux wrt. RFC
2268                 * states
2269                 */
2270                tcp_send_active_reset(sk, gfp_any());
2271                sk->sk_err = ECONNRESET;
2272        } else if (old_state == TCP_SYN_SENT)
2273                sk->sk_err = ECONNRESET;
2274
2275        tcp_clear_xmit_timers(sk);
2276        __skb_queue_purge(&sk->sk_receive_queue);
2277        tcp_write_queue_purge(sk);
2278        skb_rbtree_purge(&tp->out_of_order_queue);
2279
2280        inet->inet_dport = 0;
2281
2282        if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2283                inet_reset_saddr(sk);
2284
2285        sk->sk_shutdown = 0;
2286        sock_reset_flag(sk, SOCK_DONE);
2287        tp->srtt_us = 0;
2288        tp->write_seq += tp->max_window + 2;
2289        if (tp->write_seq == 0)
2290                tp->write_seq = 1;
2291        icsk->icsk_backoff = 0;
2292        tp->snd_cwnd = 2;
2293        icsk->icsk_probes_out = 0;
2294        tp->packets_out = 0;
2295        tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2296        tp->snd_cwnd_cnt = 0;
2297        tp->window_clamp = 0;
2298        tcp_set_ca_state(sk, TCP_CA_Open);
2299        tcp_clear_retrans(tp);
2300        inet_csk_delack_init(sk);
2301        tcp_init_send_head(sk);
2302        memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2303        __sk_dst_reset(sk);
2304
2305        WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2306
2307        sk->sk_error_report(sk);
2308        return err;
2309}
2310EXPORT_SYMBOL(tcp_disconnect);
2311
2312static inline bool tcp_can_repair_sock(const struct sock *sk)
2313{
2314        return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2315                (sk->sk_state != TCP_LISTEN);
2316}
2317
2318static int tcp_repair_set_window(struct tcp_sock *tp, char __user *optbuf, int len)
2319{
2320        struct tcp_repair_window opt;
2321
2322        if (!tp->repair)
2323                return -EPERM;
2324
2325        if (len != sizeof(opt))
2326                return -EINVAL;
2327
2328        if (copy_from_user(&opt, optbuf, sizeof(opt)))
2329                return -EFAULT;
2330
2331        if (opt.max_window < opt.snd_wnd)
2332                return -EINVAL;
2333
2334        if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
2335                return -EINVAL;
2336
2337        if (after(opt.rcv_wup, tp->rcv_nxt))
2338                return -EINVAL;
2339
2340        tp->snd_wl1     = opt.snd_wl1;
2341        tp->snd_wnd     = opt.snd_wnd;
2342        tp->max_window  = opt.max_window;
2343
2344        tp->rcv_wnd     = opt.rcv_wnd;
2345        tp->rcv_wup     = opt.rcv_wup;
2346
2347        return 0;
2348}
2349
2350static int tcp_repair_options_est(struct tcp_sock *tp,
2351                struct tcp_repair_opt __user *optbuf, unsigned int len)
2352{
2353        struct tcp_repair_opt opt;
2354
2355        while (len >= sizeof(opt)) {
2356                if (copy_from_user(&opt, optbuf, sizeof(opt)))
2357                        return -EFAULT;
2358
2359                optbuf++;
2360                len -= sizeof(opt);
2361
2362                switch (opt.opt_code) {
2363                case TCPOPT_MSS:
2364                        tp->rx_opt.mss_clamp = opt.opt_val;
2365                        break;
2366                case TCPOPT_WINDOW:
2367                        {
2368                                u16 snd_wscale = opt.opt_val & 0xFFFF;
2369                                u16 rcv_wscale = opt.opt_val >> 16;
2370
2371                                if (snd_wscale > 14 || rcv_wscale > 14)
2372                                        return -EFBIG;
2373
2374                                tp->rx_opt.snd_wscale = snd_wscale;
2375                                tp->rx_opt.rcv_wscale = rcv_wscale;
2376                                tp->rx_opt.wscale_ok = 1;
2377                        }
2378                        break;
2379                case TCPOPT_SACK_PERM:
2380                        if (opt.opt_val != 0)
2381                                return -EINVAL;
2382
2383                        tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2384                        if (sysctl_tcp_fack)
2385                                tcp_enable_fack(tp);
2386                        break;
2387                case TCPOPT_TIMESTAMP:
2388                        if (opt.opt_val != 0)
2389                                return -EINVAL;
2390
2391                        tp->rx_opt.tstamp_ok = 1;
2392                        break;
2393                }
2394        }
2395
2396        return 0;
2397}
2398
2399/*
2400 *      Socket option code for TCP.
2401 */
2402static int do_tcp_setsockopt(struct sock *sk, int level,
2403                int optname, char __user *optval, unsigned int optlen)
2404{
2405        struct tcp_sock *tp = tcp_sk(sk);
2406        struct inet_connection_sock *icsk = inet_csk(sk);
2407        struct net *net = sock_net(sk);
2408        int val;
2409        int err = 0;
2410
2411        /* These are data/string values, all the others are ints */
2412        switch (optname) {
2413        case TCP_CONGESTION: {
2414                char name[TCP_CA_NAME_MAX];
2415
2416                if (optlen < 1)
2417                        return -EINVAL;
2418
2419                val = strncpy_from_user(name, optval,
2420                                        min_t(long, TCP_CA_NAME_MAX-1, optlen));
2421                if (val < 0)
2422                        return -EFAULT;
2423                name[val] = 0;
2424
2425                lock_sock(sk);
2426                err = tcp_set_congestion_control(sk, name);
2427                release_sock(sk);
2428                return err;
2429        }
2430        default:
2431                /* fallthru */
2432                break;
2433        }
2434
2435        if (optlen < sizeof(int))
2436                return -EINVAL;
2437
2438        if (get_user(val, (int __user *)optval))
2439                return -EFAULT;
2440
2441        lock_sock(sk);
2442
2443        switch (optname) {
2444        case TCP_MAXSEG:
2445                /* Values greater than interface MTU won't take effect. However
2446                 * at the point when this call is done we typically don't yet
2447                 * know which interface is going to be used */
2448                if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2449                        err = -EINVAL;
2450                        break;
2451                }
2452                tp->rx_opt.user_mss = val;
2453                break;
2454
2455        case TCP_NODELAY:
2456                if (val) {
2457                        /* TCP_NODELAY is weaker than TCP_CORK, so that
2458                         * this option on corked socket is remembered, but
2459                         * it is not activated until cork is cleared.
2460                         *
2461                         * However, when TCP_NODELAY is set we make
2462                         * an explicit push, which overrides even TCP_CORK
2463                         * for currently queued segments.
2464                         */
2465                        tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2466                        tcp_push_pending_frames(sk);
2467                } else {
2468                        tp->nonagle &= ~TCP_NAGLE_OFF;
2469                }
2470                break;
2471
2472        case TCP_THIN_LINEAR_TIMEOUTS:
2473                if (val < 0 || val > 1)
2474                        err = -EINVAL;
2475                else
2476                        tp->thin_lto = val;
2477                break;
2478
2479        case TCP_THIN_DUPACK:
2480                if (val < 0 || val > 1)
2481                        err = -EINVAL;
2482                else {
2483                        tp->thin_dupack = val;
2484                        if (tp->thin_dupack)
2485                                tcp_disable_early_retrans(tp);
2486                }
2487                break;
2488
2489        case TCP_REPAIR:
2490                if (!tcp_can_repair_sock(sk))
2491                        err = -EPERM;
2492                else if (val == 1) {
2493                        tp->repair = 1;
2494                        sk->sk_reuse = SK_FORCE_REUSE;
2495                        tp->repair_queue = TCP_NO_QUEUE;
2496                } else if (val == 0) {
2497                        tp->repair = 0;
2498                        sk->sk_reuse = SK_NO_REUSE;
2499                        tcp_send_window_probe(sk);
2500                } else
2501                        err = -EINVAL;
2502
2503                break;
2504
2505        case TCP_REPAIR_QUEUE:
2506                if (!tp->repair)
2507                        err = -EPERM;
2508                else if (val < TCP_QUEUES_NR)
2509                        tp->repair_queue = val;
2510                else
2511                        err = -EINVAL;
2512                break;
2513
2514        case TCP_QUEUE_SEQ:
2515                if (sk->sk_state != TCP_CLOSE)
2516                        err = -EPERM;
2517                else if (tp->repair_queue == TCP_SEND_QUEUE)
2518                        tp->write_seq = val;
2519                else if (tp->repair_queue == TCP_RECV_QUEUE)
2520                        tp->rcv_nxt = val;
2521                else
2522                        err = -EINVAL;
2523                break;
2524
2525        case TCP_REPAIR_OPTIONS:
2526                if (!tp->repair)
2527                        err = -EINVAL;
2528                else if (sk->sk_state == TCP_ESTABLISHED)
2529                        err = tcp_repair_options_est(tp,
2530                                        (struct tcp_repair_opt __user *)optval,
2531                                        optlen);
2532                else
2533                        err = -EPERM;
2534                break;
2535
2536        case TCP_CORK:
2537                /* When set indicates to always queue non-full frames.
2538                 * Later the user clears this option and we transmit
2539                 * any pending partial frames in the queue.  This is
2540                 * meant to be used alongside sendfile() to get properly
2541                 * filled frames when the user (for example) must write
2542                 * out headers with a write() call first and then use
2543                 * sendfile to send out the data parts.
2544                 *
2545                 * TCP_CORK can be set together with TCP_NODELAY and it is
2546                 * stronger than TCP_NODELAY.
2547                 */
2548                if (val) {
2549                        tp->nonagle |= TCP_NAGLE_CORK;
2550                } else {
2551                        tp->nonagle &= ~TCP_NAGLE_CORK;
2552                        if (tp->nonagle&TCP_NAGLE_OFF)
2553                                tp->nonagle |= TCP_NAGLE_PUSH;
2554                        tcp_push_pending_frames(sk);
2555                }
2556                break;
2557
2558        case TCP_KEEPIDLE:
2559                if (val < 1 || val > MAX_TCP_KEEPIDLE)
2560                        err = -EINVAL;
2561                else {
2562                        tp->keepalive_time = val * HZ;
2563                        if (sock_flag(sk, SOCK_KEEPOPEN) &&
2564                            !((1 << sk->sk_state) &
2565                              (TCPF_CLOSE | TCPF_LISTEN))) {
2566                                u32 elapsed = keepalive_time_elapsed(tp);
2567                                if (tp->keepalive_time > elapsed)
2568                                        elapsed = tp->keepalive_time - elapsed;
2569                                else
2570                                        elapsed = 0;
2571                                inet_csk_reset_keepalive_timer(sk, elapsed);
2572                        }
2573                }
2574                break;
2575        case TCP_KEEPINTVL:
2576                if (val < 1 || val > MAX_TCP_KEEPINTVL)
2577                        err = -EINVAL;
2578                else
2579                        tp->keepalive_intvl = val * HZ;
2580                break;
2581        case TCP_KEEPCNT:
2582                if (val < 1 || val > MAX_TCP_KEEPCNT)
2583                        err = -EINVAL;
2584                else
2585                        tp->keepalive_probes = val;
2586                break;
2587        case TCP_SYNCNT:
2588                if (val < 1 || val > MAX_TCP_SYNCNT)
2589                        err = -EINVAL;
2590                else
2591                        icsk->icsk_syn_retries = val;
2592                break;
2593
2594        case TCP_SAVE_SYN:
2595                if (val < 0 || val > 1)
2596                        err = -EINVAL;
2597                else
2598                        tp->save_syn = val;
2599                break;
2600
2601        case TCP_LINGER2:
2602                if (val < 0)
2603                        tp->linger2 = -1;
2604                else if (val > net->ipv4.sysctl_tcp_fin_timeout / HZ)
2605                        tp->linger2 = 0;
2606                else
2607                        tp->linger2 = val * HZ;
2608                break;
2609
2610        case TCP_DEFER_ACCEPT:
2611                /* Translate value in seconds to number of retransmits */
2612                icsk->icsk_accept_queue.rskq_defer_accept =
2613                        secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2614                                        TCP_RTO_MAX / HZ);
2615                break;
2616
2617        case TCP_WINDOW_CLAMP:
2618                if (!val) {
2619                        if (sk->sk_state != TCP_CLOSE) {
2620                                err = -EINVAL;
2621                                break;
2622                        }
2623                        tp->window_clamp = 0;
2624                } else
2625                        tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2626                                                SOCK_MIN_RCVBUF / 2 : val;
2627                break;
2628
2629        case TCP_QUICKACK:
2630                if (!val) {
2631                        icsk->icsk_ack.pingpong = 1;
2632                } else {
2633                        icsk->icsk_ack.pingpong = 0;
2634                        if ((1 << sk->sk_state) &
2635                            (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2636                            inet_csk_ack_scheduled(sk)) {
2637                                icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2638                                tcp_cleanup_rbuf(sk, 1);
2639                                if (!(val & 1))
2640                                        icsk->icsk_ack.pingpong = 1;
2641                        }
2642                }
2643                break;
2644
2645#ifdef CONFIG_TCP_MD5SIG
2646        case TCP_MD5SIG:
2647                /* Read the IP->Key mappings from userspace */
2648                err = tp->af_specific->md5_parse(sk, optval, optlen);
2649                break;
2650#endif
2651        case TCP_USER_TIMEOUT:
2652                /* Cap the max time in ms TCP will retry or probe the window
2653                 * before giving up and aborting (ETIMEDOUT) a connection.
2654                 */
2655                if (val < 0)
2656                        err = -EINVAL;
2657                else
2658                        icsk->icsk_user_timeout = msecs_to_jiffies(val);
2659                break;
2660
2661        case TCP_FASTOPEN:
2662                if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
2663                    TCPF_LISTEN))) {
2664                        tcp_fastopen_init_key_once(true);
2665
2666                        fastopen_queue_tune(sk, val);
2667                } else {
2668                        err = -EINVAL;
2669                }
2670                break;
2671        case TCP_TIMESTAMP:
2672                if (!tp->repair)
2673                        err = -EPERM;
2674                else
2675                        tp->tsoffset = val - tcp_time_stamp;
2676                break;
2677        case TCP_REPAIR_WINDOW:
2678                err = tcp_repair_set_window(tp, optval, optlen);
2679                break;
2680        case TCP_NOTSENT_LOWAT:
2681                tp->notsent_lowat = val;
2682                sk->sk_write_space(sk);
2683                break;
2684        default:
2685                err = -ENOPROTOOPT;
2686                break;
2687        }
2688
2689        release_sock(sk);
2690        return err;
2691}
2692
2693int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2694                   unsigned int optlen)
2695{
2696        const struct inet_connection_sock *icsk = inet_csk(sk);
2697
2698        if (level != SOL_TCP)
2699                return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2700                                                     optval, optlen);
2701        return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2702}
2703EXPORT_SYMBOL(tcp_setsockopt);
2704
2705#ifdef CONFIG_COMPAT
2706int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2707                          char __user *optval, unsigned int optlen)
2708{
2709        if (level != SOL_TCP)
2710                return inet_csk_compat_setsockopt(sk, level, optname,
2711                                                  optval, optlen);
2712        return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2713}
2714EXPORT_SYMBOL(compat_tcp_setsockopt);
2715#endif
2716
2717static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
2718                                      struct tcp_info *info)
2719{
2720        u64 stats[__TCP_CHRONO_MAX], total = 0;
2721        enum tcp_chrono i;
2722
2723        for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
2724                stats[i] = tp->chrono_stat[i - 1];
2725                if (i == tp->chrono_type)
2726                        stats[i] += tcp_time_stamp - tp->chrono_start;
2727                stats[i] *= USEC_PER_SEC / HZ;
2728                total += stats[i];
2729        }
2730
2731        info->tcpi_busy_time = total;
2732        info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
2733        info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
2734}
2735
2736/* Return information about state of tcp endpoint in API format. */
2737void tcp_get_info(struct sock *sk, struct tcp_info *info)
2738{
2739        const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
2740        const struct inet_connection_sock *icsk = inet_csk(sk);
2741        u32 now = tcp_time_stamp, intv;
2742        u64 rate64;
2743        bool slow;
2744        u32 rate;
2745
2746        memset(info, 0, sizeof(*info));
2747        if (sk->sk_type != SOCK_STREAM)
2748                return;
2749
2750        info->tcpi_state = sk_state_load(sk);
2751
2752        /* Report meaningful fields for all TCP states, including listeners */
2753        rate = READ_ONCE(sk->sk_pacing_rate);
2754        rate64 = rate != ~0U ? rate : ~0ULL;
2755        info->tcpi_pacing_rate = rate64;
2756
2757        rate = READ_ONCE(sk->sk_max_pacing_rate);
2758        rate64 = rate != ~0U ? rate : ~0ULL;
2759        info->tcpi_max_pacing_rate = rate64;
2760
2761        info->tcpi_reordering = tp->reordering;
2762        info->tcpi_snd_cwnd = tp->snd_cwnd;
2763
2764        if (info->tcpi_state == TCP_LISTEN) {
2765                /* listeners aliased fields :
2766                 * tcpi_unacked -> Number of children ready for accept()
2767                 * tcpi_sacked  -> max backlog
2768                 */
2769                info->tcpi_unacked = sk->sk_ack_backlog;
2770                info->tcpi_sacked = sk->sk_max_ack_backlog;
2771                return;
2772        }
2773        info->tcpi_ca_state = icsk->icsk_ca_state;
2774        info->tcpi_retransmits = icsk->icsk_retransmits;
2775        info->tcpi_probes = icsk->icsk_probes_out;
2776        info->tcpi_backoff = icsk->icsk_backoff;
2777
2778        if (tp->rx_opt.tstamp_ok)
2779                info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2780        if (tcp_is_sack(tp))
2781                info->tcpi_options |= TCPI_OPT_SACK;
2782        if (tp->rx_opt.wscale_ok) {
2783                info->tcpi_options |= TCPI_OPT_WSCALE;
2784                info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2785                info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2786        }
2787
2788        if (tp->ecn_flags & TCP_ECN_OK)
2789                info->tcpi_options |= TCPI_OPT_ECN;
2790        if (tp->ecn_flags & TCP_ECN_SEEN)
2791                info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2792        if (tp->syn_data_acked)
2793                info->tcpi_options |= TCPI_OPT_SYN_DATA;
2794
2795        info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2796        info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2797        info->tcpi_snd_mss = tp->mss_cache;
2798        info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2799
2800        info->tcpi_unacked = tp->packets_out;
2801        info->tcpi_sacked = tp->sacked_out;
2802
2803        info->tcpi_lost = tp->lost_out;
2804        info->tcpi_retrans = tp->retrans_out;
2805        info->tcpi_fackets = tp->fackets_out;
2806
2807        info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2808        info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2809        info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2810
2811        info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2812        info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2813        info->tcpi_rtt = tp->srtt_us >> 3;
2814        info->tcpi_rttvar = tp->mdev_us >> 2;
2815        info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2816        info->tcpi_advmss = tp->advmss;
2817
2818        info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2819        info->tcpi_rcv_space = tp->rcvq_space.space;
2820
2821        info->tcpi_total_retrans = tp->total_retrans;
2822
2823        slow = lock_sock_fast(sk);
2824
2825        info->tcpi_bytes_acked = tp->bytes_acked;
2826        info->tcpi_bytes_received = tp->bytes_received;
2827        info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
2828        tcp_get_info_chrono_stats(tp, info);
2829
2830        unlock_sock_fast(sk, slow);
2831
2832        info->tcpi_segs_out = tp->segs_out;
2833        info->tcpi_segs_in = tp->segs_in;
2834
2835        info->tcpi_min_rtt = tcp_min_rtt(tp);
2836        info->tcpi_data_segs_in = tp->data_segs_in;
2837        info->tcpi_data_segs_out = tp->data_segs_out;
2838
2839        info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
2840        rate = READ_ONCE(tp->rate_delivered);
2841        intv = READ_ONCE(tp->rate_interval_us);
2842        if (rate && intv) {
2843                rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
2844                do_div(rate64, intv);
2845                info->tcpi_delivery_rate = rate64;
2846        }
2847}
2848EXPORT_SYMBOL_GPL(tcp_get_info);
2849
2850struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk)
2851{
2852        const struct tcp_sock *tp = tcp_sk(sk);
2853        struct sk_buff *stats;
2854        struct tcp_info info;
2855
2856        stats = alloc_skb(3 * nla_total_size_64bit(sizeof(u64)), GFP_ATOMIC);
2857        if (!stats)
2858                return NULL;
2859
2860        tcp_get_info_chrono_stats(tp, &info);
2861        nla_put_u64_64bit(stats, TCP_NLA_BUSY,
2862                          info.tcpi_busy_time, TCP_NLA_PAD);
2863        nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
2864                          info.tcpi_rwnd_limited, TCP_NLA_PAD);
2865        nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
2866                          info.tcpi_sndbuf_limited, TCP_NLA_PAD);
2867        return stats;
2868}
2869
2870static int do_tcp_getsockopt(struct sock *sk, int level,
2871                int optname, char __user *optval, int __user *optlen)
2872{
2873        struct inet_connection_sock *icsk = inet_csk(sk);
2874        struct tcp_sock *tp = tcp_sk(sk);
2875        struct net *net = sock_net(sk);
2876        int val, len;
2877
2878        if (get_user(len, optlen))
2879                return -EFAULT;
2880
2881        len = min_t(unsigned int, len, sizeof(int));
2882
2883        if (len < 0)
2884                return -EINVAL;
2885
2886        switch (optname) {
2887        case TCP_MAXSEG:
2888                val = tp->mss_cache;
2889                if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2890                        val = tp->rx_opt.user_mss;
2891                if (tp->repair)
2892                        val = tp->rx_opt.mss_clamp;
2893                break;
2894        case TCP_NODELAY:
2895                val = !!(tp->nonagle&TCP_NAGLE_OFF);
2896                break;
2897        case TCP_CORK:
2898                val = !!(tp->nonagle&TCP_NAGLE_CORK);
2899                break;
2900        case TCP_KEEPIDLE:
2901                val = keepalive_time_when(tp) / HZ;
2902                break;
2903        case TCP_KEEPINTVL:
2904                val = keepalive_intvl_when(tp) / HZ;
2905                break;
2906        case TCP_KEEPCNT:
2907                val = keepalive_probes(tp);
2908                break;
2909        case TCP_SYNCNT:
2910                val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries;
2911                break;
2912        case TCP_LINGER2:
2913                val = tp->linger2;
2914                if (val >= 0)
2915                        val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
2916                break;
2917        case TCP_DEFER_ACCEPT:
2918                val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2919                                      TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2920                break;
2921        case TCP_WINDOW_CLAMP:
2922                val = tp->window_clamp;
2923                break;
2924        case TCP_INFO: {
2925                struct tcp_info info;
2926
2927                if (get_user(len, optlen))
2928                        return -EFAULT;
2929
2930                tcp_get_info(sk, &info);
2931
2932                len = min_t(unsigned int, len, sizeof(info));
2933                if (put_user(len, optlen))
2934                        return -EFAULT;
2935                if (copy_to_user(optval, &info, len))
2936                        return -EFAULT;
2937                return 0;
2938        }
2939        case TCP_CC_INFO: {
2940                const struct tcp_congestion_ops *ca_ops;
2941                union tcp_cc_info info;
2942                size_t sz = 0;
2943                int attr;
2944
2945                if (get_user(len, optlen))
2946                        return -EFAULT;
2947
2948                ca_ops = icsk->icsk_ca_ops;
2949                if (ca_ops && ca_ops->get_info)
2950                        sz = ca_ops->get_info(sk, ~0U, &attr, &info);
2951
2952                len = min_t(unsigned int, len, sz);
2953                if (put_user(len, optlen))
2954                        return -EFAULT;
2955                if (copy_to_user(optval, &info, len))
2956                        return -EFAULT;
2957                return 0;
2958        }
2959        case TCP_QUICKACK:
2960                val = !icsk->icsk_ack.pingpong;
2961                break;
2962
2963        case TCP_CONGESTION:
2964                if (get_user(len, optlen))
2965                        return -EFAULT;
2966                len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2967                if (put_user(len, optlen))
2968                        return -EFAULT;
2969                if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2970                        return -EFAULT;
2971                return 0;
2972
2973        case TCP_THIN_LINEAR_TIMEOUTS:
2974                val = tp->thin_lto;
2975                break;
2976        case TCP_THIN_DUPACK:
2977                val = tp->thin_dupack;
2978                break;
2979
2980        case TCP_REPAIR:
2981                val = tp->repair;
2982                break;
2983
2984        case TCP_REPAIR_QUEUE:
2985                if (tp->repair)
2986                        val = tp->repair_queue;
2987                else
2988                        return -EINVAL;
2989                break;
2990
2991        case TCP_REPAIR_WINDOW: {
2992                struct tcp_repair_window opt;
2993
2994                if (get_user(len, optlen))
2995                        return -EFAULT;
2996
2997                if (len != sizeof(opt))
2998                        return -EINVAL;
2999
3000                if (!tp->repair)
3001                        return -EPERM;
3002
3003                opt.snd_wl1     = tp->snd_wl1;
3004                opt.snd_wnd     = tp->snd_wnd;
3005                opt.max_window  = tp->max_window;
3006                opt.rcv_wnd     = tp->rcv_wnd;
3007                opt.rcv_wup     = tp->rcv_wup;
3008
3009                if (copy_to_user(optval, &opt, len))
3010                        return -EFAULT;
3011                return 0;
3012        }
3013        case TCP_QUEUE_SEQ:
3014                if (tp->repair_queue == TCP_SEND_QUEUE)
3015                        val = tp->write_seq;
3016                else if (tp->repair_queue == TCP_RECV_QUEUE)
3017                        val = tp->rcv_nxt;
3018                else
3019                        return -EINVAL;
3020                break;
3021
3022        case TCP_USER_TIMEOUT:
3023                val = jiffies_to_msecs(icsk->icsk_user_timeout);
3024                break;
3025
3026        case TCP_FASTOPEN:
3027                val = icsk->icsk_accept_queue.fastopenq.max_qlen;
3028                break;
3029
3030        case TCP_TIMESTAMP:
3031                val = tcp_time_stamp + tp->tsoffset;
3032                break;
3033        case TCP_NOTSENT_LOWAT:
3034                val = tp->notsent_lowat;
3035                break;
3036        case TCP_SAVE_SYN:
3037                val = tp->save_syn;
3038                break;
3039        case TCP_SAVED_SYN: {
3040                if (get_user(len, optlen))
3041                        return -EFAULT;
3042
3043                lock_sock(sk);
3044                if (tp->saved_syn) {
3045                        if (len < tp->saved_syn[0]) {
3046                                if (put_user(tp->saved_syn[0], optlen)) {
3047                                        release_sock(sk);
3048                                        return -EFAULT;
3049                                }
3050                                release_sock(sk);
3051                                return -EINVAL;
3052                        }
3053                        len = tp->saved_syn[0];
3054                        if (put_user(len, optlen)) {
3055                                release_sock(sk);
3056                                return -EFAULT;
3057                        }
3058                        if (copy_to_user(optval, tp->saved_syn + 1, len)) {
3059                                release_sock(sk);
3060                                return -EFAULT;
3061                        }
3062                        tcp_saved_syn_free(tp);
3063                        release_sock(sk);
3064                } else {
3065                        release_sock(sk);
3066                        len = 0;
3067                        if (put_user(len, optlen))
3068                                return -EFAULT;
3069                }
3070                return 0;
3071        }
3072        default:
3073                return -ENOPROTOOPT;
3074        }
3075
3076        if (put_user(len, optlen))
3077                return -EFAULT;
3078        if (copy_to_user(optval, &val, len))
3079                return -EFAULT;
3080        return 0;
3081}
3082
3083int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
3084                   int __user *optlen)
3085{
3086        struct inet_connection_sock *icsk = inet_csk(sk);
3087
3088        if (level != SOL_TCP)
3089                return icsk->icsk_af_ops->getsockopt(sk, level, optname,
3090                                                     optval, optlen);
3091        return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3092}
3093EXPORT_SYMBOL(tcp_getsockopt);
3094
3095#ifdef CONFIG_COMPAT
3096int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
3097                          char __user *optval, int __user *optlen)
3098{
3099        if (level != SOL_TCP)
3100                return inet_csk_compat_getsockopt(sk, level, optname,
3101                                                  optval, optlen);
3102        return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3103}
3104EXPORT_SYMBOL(compat_tcp_getsockopt);
3105#endif
3106
3107#ifdef CONFIG_TCP_MD5SIG
3108static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
3109static DEFINE_MUTEX(tcp_md5sig_mutex);
3110static bool tcp_md5sig_pool_populated = false;
3111
3112static void __tcp_alloc_md5sig_pool(void)
3113{
3114        struct crypto_ahash *hash;
3115        int cpu;
3116
3117        hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
3118        if (IS_ERR(hash))
3119                return;
3120
3121        for_each_possible_cpu(cpu) {
3122                void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch;
3123                struct ahash_request *req;
3124
3125                if (!scratch) {
3126                        scratch = kmalloc_node(sizeof(union tcp_md5sum_block) +
3127                                               sizeof(struct tcphdr),
3128                                               GFP_KERNEL,
3129                                               cpu_to_node(cpu));
3130                        if (!scratch)
3131                                return;
3132                        per_cpu(tcp_md5sig_pool, cpu).scratch = scratch;
3133                }
3134                if (per_cpu(tcp_md5sig_pool, cpu).md5_req)
3135                        continue;
3136
3137                req = ahash_request_alloc(hash, GFP_KERNEL);
3138                if (!req)
3139                        return;
3140
3141                ahash_request_set_callback(req, 0, NULL, NULL);
3142
3143                per_cpu(tcp_md5sig_pool, cpu).md5_req = req;
3144        }
3145        /* before setting tcp_md5sig_pool_populated, we must commit all writes
3146         * to memory. See smp_rmb() in tcp_get_md5sig_pool()
3147         */
3148        smp_wmb();
3149        tcp_md5sig_pool_populated = true;
3150}
3151
3152bool tcp_alloc_md5sig_pool(void)
3153{
3154        if (unlikely(!tcp_md5sig_pool_populated)) {
3155                mutex_lock(&tcp_md5sig_mutex);
3156
3157                if (!tcp_md5sig_pool_populated)
3158                        __tcp_alloc_md5sig_pool();
3159
3160                mutex_unlock(&tcp_md5sig_mutex);
3161        }
3162        return tcp_md5sig_pool_populated;
3163}
3164EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3165
3166
3167/**
3168 *      tcp_get_md5sig_pool - get md5sig_pool for this user
3169 *
3170 *      We use percpu structure, so if we succeed, we exit with preemption
3171 *      and BH disabled, to make sure another thread or softirq handling
3172 *      wont try to get same context.
3173 */
3174struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3175{
3176        local_bh_disable();
3177
3178        if (tcp_md5sig_pool_populated) {
3179                /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3180                smp_rmb();
3181                return this_cpu_ptr(&tcp_md5sig_pool);
3182        }
3183        local_bh_enable();
3184        return NULL;
3185}
3186EXPORT_SYMBOL(tcp_get_md5sig_pool);
3187
3188int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3189                          const struct sk_buff *skb, unsigned int header_len)
3190{
3191        struct scatterlist sg;
3192        const struct tcphdr *tp = tcp_hdr(skb);
3193        struct ahash_request *req = hp->md5_req;
3194        unsigned int i;
3195        const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3196                                           skb_headlen(skb) - header_len : 0;
3197        const struct skb_shared_info *shi = skb_shinfo(skb);
3198        struct sk_buff *frag_iter;
3199
3200        sg_init_table(&sg, 1);
3201
3202        sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3203        ahash_request_set_crypt(req, &sg, NULL, head_data_len);
3204        if (crypto_ahash_update(req))
3205                return 1;
3206
3207        for (i = 0; i < shi->nr_frags; ++i) {
3208                const struct skb_frag_struct *f = &shi->frags[i];
3209                unsigned int offset = f->page_offset;
3210                struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3211
3212                sg_set_page(&sg, page, skb_frag_size(f),
3213                            offset_in_page(offset));
3214                ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
3215                if (crypto_ahash_update(req))
3216                        return 1;
3217        }
3218
3219        skb_walk_frags(skb, frag_iter)
3220                if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3221                        return 1;
3222
3223        return 0;
3224}
3225EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3226
3227int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3228{
3229        struct scatterlist sg;
3230
3231        sg_init_one(&sg, key->key, key->keylen);
3232        ahash_request_set_crypt(hp->md5_req, &sg, NULL, key->keylen);
3233        return crypto_ahash_update(hp->md5_req);
3234}
3235EXPORT_SYMBOL(tcp_md5_hash_key);
3236
3237#endif
3238
3239void tcp_done(struct sock *sk)
3240{
3241        struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
3242
3243        if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3244                TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3245
3246        tcp_set_state(sk, TCP_CLOSE);
3247        tcp_clear_xmit_timers(sk);
3248        if (req)
3249                reqsk_fastopen_remove(sk, req, false);
3250
3251        sk->sk_shutdown = SHUTDOWN_MASK;
3252
3253        if (!sock_flag(sk, SOCK_DEAD))
3254                sk->sk_state_change(sk);
3255        else
3256                inet_csk_destroy_sock(sk);
3257}
3258EXPORT_SYMBOL_GPL(tcp_done);
3259
3260int tcp_abort(struct sock *sk, int err)
3261{
3262        if (!sk_fullsock(sk)) {
3263                if (sk->sk_state == TCP_NEW_SYN_RECV) {
3264                        struct request_sock *req = inet_reqsk(sk);
3265
3266                        local_bh_disable();
3267                        inet_csk_reqsk_queue_drop_and_put(req->rsk_listener,
3268                                                          req);
3269                        local_bh_enable();
3270                        return 0;
3271                }
3272                return -EOPNOTSUPP;
3273        }
3274
3275        /* Don't race with userspace socket closes such as tcp_close. */
3276        lock_sock(sk);
3277
3278        if (sk->sk_state == TCP_LISTEN) {
3279                tcp_set_state(sk, TCP_CLOSE);
3280                inet_csk_listen_stop(sk);
3281        }
3282
3283        /* Don't race with BH socket closes such as inet_csk_listen_stop. */
3284        local_bh_disable();
3285        bh_lock_sock(sk);
3286
3287        if (!sock_flag(sk, SOCK_DEAD)) {
3288                sk->sk_err = err;
3289                /* This barrier is coupled with smp_rmb() in tcp_poll() */
3290                smp_wmb();
3291                sk->sk_error_report(sk);
3292                if (tcp_need_reset(sk->sk_state))
3293                        tcp_send_active_reset(sk, GFP_ATOMIC);
3294                tcp_done(sk);
3295        }
3296
3297        bh_unlock_sock(sk);
3298        local_bh_enable();
3299        release_sock(sk);
3300        return 0;
3301}
3302EXPORT_SYMBOL_GPL(tcp_abort);
3303
3304extern struct tcp_congestion_ops tcp_reno;
3305
3306static __initdata unsigned long thash_entries;
3307static int __init set_thash_entries(char *str)
3308{
3309        ssize_t ret;
3310
3311        if (!str)
3312                return 0;
3313
3314        ret = kstrtoul(str, 0, &thash_entries);
3315        if (ret)
3316                return 0;
3317
3318        return 1;
3319}
3320__setup("thash_entries=", set_thash_entries);
3321
3322static void __init tcp_init_mem(void)
3323{
3324        unsigned long limit = nr_free_buffer_pages() / 16;
3325
3326        limit = max(limit, 128UL);
3327        sysctl_tcp_mem[0] = limit / 4 * 3;              /* 4.68 % */
3328        sysctl_tcp_mem[1] = limit;                      /* 6.25 % */
3329        sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;      /* 9.37 % */
3330}
3331
3332void __init tcp_init(void)
3333{
3334        int max_rshare, max_wshare, cnt;
3335        unsigned long limit;
3336        unsigned int i;
3337
3338        BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
3339                     FIELD_SIZEOF(struct sk_buff, cb));
3340
3341        percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3342        percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
3343        tcp_hashinfo.bind_bucket_cachep =
3344                kmem_cache_create("tcp_bind_bucket",
3345                                  sizeof(struct inet_bind_bucket), 0,
3346                                  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3347
3348        /* Size and allocate the main established and bind bucket
3349         * hash tables.
3350         *
3351         * The methodology is similar to that of the buffer cache.
3352         */
3353        tcp_hashinfo.ehash =
3354                alloc_large_system_hash("TCP established",
3355                                        sizeof(struct inet_ehash_bucket),
3356                                        thash_entries,
3357                                        17, /* one slot per 128 KB of memory */
3358                                        0,
3359                                        NULL,
3360                                        &tcp_hashinfo.ehash_mask,
3361                                        0,
3362                                        thash_entries ? 0 : 512 * 1024);
3363        for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3364                INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3365
3366        if (inet_ehash_locks_alloc(&tcp_hashinfo))
3367                panic("TCP: failed to alloc ehash_locks");
3368        tcp_hashinfo.bhash =
3369                alloc_large_system_hash("TCP bind",
3370                                        sizeof(struct inet_bind_hashbucket),
3371                                        tcp_hashinfo.ehash_mask + 1,
3372                                        17, /* one slot per 128 KB of memory */
3373                                        0,
3374                                        &tcp_hashinfo.bhash_size,
3375                                        NULL,
3376                                        0,
3377                                        64 * 1024);
3378        tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3379        for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3380                spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3381                INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3382        }
3383
3384
3385        cnt = tcp_hashinfo.ehash_mask + 1;
3386
3387        tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3388        sysctl_tcp_max_orphans = cnt / 2;
3389        sysctl_max_syn_backlog = max(128, cnt / 256);
3390
3391        tcp_init_mem();
3392        /* Set per-socket limits to no more than 1/128 the pressure threshold */
3393        limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3394        max_wshare = min(4UL*1024*1024, limit);
3395        max_rshare = min(6UL*1024*1024, limit);
3396
3397        sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
3398        sysctl_tcp_wmem[1] = 16*1024;
3399        sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
3400
3401        sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
3402        sysctl_tcp_rmem[1] = 87380;
3403        sysctl_tcp_rmem[2] = max(87380, max_rshare);
3404
3405        pr_info("Hash tables configured (established %u bind %u)\n",
3406                tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
3407
3408        tcp_metrics_init();
3409        BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
3410        tcp_tasklet_init();
3411}
3412
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.