linux/net/smc/af_smc.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4 *
   5 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6 *  applies to SOCK_STREAM sockets only
   7 *  offers an alternative communication option for TCP-protocol sockets
   8 *  applicable with RoCE-cards only
   9 *
  10 *  Initial restrictions:
  11 *    - support for alternate links postponed
  12 *
  13 *  Copyright IBM Corp. 2016, 2018
  14 *
  15 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16 *              based on prototype from Frank Blaschka
  17 */
  18
  19#define KMSG_COMPONENT "smc"
  20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22#include <linux/module.h>
  23#include <linux/socket.h>
  24#include <linux/workqueue.h>
  25#include <linux/in.h>
  26#include <linux/sched/signal.h>
  27#include <linux/if_vlan.h>
  28#include <linux/rcupdate_wait.h>
  29#include <linux/ctype.h>
  30
  31#include <net/sock.h>
  32#include <net/tcp.h>
  33#include <net/smc.h>
  34#include <asm/ioctls.h>
  35
  36#include <net/net_namespace.h>
  37#include <net/netns/generic.h>
  38#include "smc_netns.h"
  39
  40#include "smc.h"
  41#include "smc_clc.h"
  42#include "smc_llc.h"
  43#include "smc_cdc.h"
  44#include "smc_core.h"
  45#include "smc_ib.h"
  46#include "smc_ism.h"
  47#include "smc_pnet.h"
  48#include "smc_netlink.h"
  49#include "smc_tx.h"
  50#include "smc_rx.h"
  51#include "smc_close.h"
  52#include "smc_stats.h"
  53
  54static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  55                                                 * creation on server
  56                                                 */
  57static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  58                                                 * creation on client
  59                                                 */
  60
  61struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  62struct workqueue_struct *smc_close_wq;  /* wq for close work */
  63
  64static void smc_tcp_listen_work(struct work_struct *);
  65static void smc_connect_work(struct work_struct *);
  66
  67static void smc_set_keepalive(struct sock *sk, int val)
  68{
  69        struct smc_sock *smc = smc_sk(sk);
  70
  71        smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  72}
  73
  74static struct smc_hashinfo smc_v4_hashinfo = {
  75        .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  76};
  77
  78static struct smc_hashinfo smc_v6_hashinfo = {
  79        .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  80};
  81
  82int smc_hash_sk(struct sock *sk)
  83{
  84        struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  85        struct hlist_head *head;
  86
  87        head = &h->ht;
  88
  89        write_lock_bh(&h->lock);
  90        sk_add_node(sk, head);
  91        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  92        write_unlock_bh(&h->lock);
  93
  94        return 0;
  95}
  96EXPORT_SYMBOL_GPL(smc_hash_sk);
  97
  98void smc_unhash_sk(struct sock *sk)
  99{
 100        struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 101
 102        write_lock_bh(&h->lock);
 103        if (sk_del_node_init(sk))
 104                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 105        write_unlock_bh(&h->lock);
 106}
 107EXPORT_SYMBOL_GPL(smc_unhash_sk);
 108
 109struct proto smc_proto = {
 110        .name           = "SMC",
 111        .owner          = THIS_MODULE,
 112        .keepalive      = smc_set_keepalive,
 113        .hash           = smc_hash_sk,
 114        .unhash         = smc_unhash_sk,
 115        .obj_size       = sizeof(struct smc_sock),
 116        .h.smc_hash     = &smc_v4_hashinfo,
 117        .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 118};
 119EXPORT_SYMBOL_GPL(smc_proto);
 120
 121struct proto smc_proto6 = {
 122        .name           = "SMC6",
 123        .owner          = THIS_MODULE,
 124        .keepalive      = smc_set_keepalive,
 125        .hash           = smc_hash_sk,
 126        .unhash         = smc_unhash_sk,
 127        .obj_size       = sizeof(struct smc_sock),
 128        .h.smc_hash     = &smc_v6_hashinfo,
 129        .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 130};
 131EXPORT_SYMBOL_GPL(smc_proto6);
 132
 133static void smc_restore_fallback_changes(struct smc_sock *smc)
 134{
 135        if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 136                smc->clcsock->file->private_data = smc->sk.sk_socket;
 137                smc->clcsock->file = NULL;
 138        }
 139}
 140
 141static int __smc_release(struct smc_sock *smc)
 142{
 143        struct sock *sk = &smc->sk;
 144        int rc = 0;
 145
 146        if (!smc->use_fallback) {
 147                rc = smc_close_active(smc);
 148                sock_set_flag(sk, SOCK_DEAD);
 149                sk->sk_shutdown |= SHUTDOWN_MASK;
 150        } else {
 151                if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 152                        sock_put(sk); /* passive closing */
 153                if (sk->sk_state == SMC_LISTEN) {
 154                        /* wake up clcsock accept */
 155                        rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 156                }
 157                sk->sk_state = SMC_CLOSED;
 158                sk->sk_state_change(sk);
 159                smc_restore_fallback_changes(smc);
 160        }
 161
 162        sk->sk_prot->unhash(sk);
 163
 164        if (sk->sk_state == SMC_CLOSED) {
 165                if (smc->clcsock) {
 166                        release_sock(sk);
 167                        smc_clcsock_release(smc);
 168                        lock_sock(sk);
 169                }
 170                if (!smc->use_fallback)
 171                        smc_conn_free(&smc->conn);
 172        }
 173
 174        return rc;
 175}
 176
 177static int smc_release(struct socket *sock)
 178{
 179        struct sock *sk = sock->sk;
 180        struct smc_sock *smc;
 181        int rc = 0;
 182
 183        if (!sk)
 184                goto out;
 185
 186        sock_hold(sk); /* sock_put below */
 187        smc = smc_sk(sk);
 188
 189        /* cleanup for a dangling non-blocking connect */
 190        if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 191                tcp_abort(smc->clcsock->sk, ECONNABORTED);
 192        flush_work(&smc->connect_work);
 193
 194        if (sk->sk_state == SMC_LISTEN)
 195                /* smc_close_non_accepted() is called and acquires
 196                 * sock lock for child sockets again
 197                 */
 198                lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 199        else
 200                lock_sock(sk);
 201
 202        rc = __smc_release(smc);
 203
 204        /* detach socket */
 205        sock_orphan(sk);
 206        sock->sk = NULL;
 207        release_sock(sk);
 208
 209        sock_put(sk); /* sock_hold above */
 210        sock_put(sk); /* final sock_put */
 211out:
 212        return rc;
 213}
 214
 215static void smc_destruct(struct sock *sk)
 216{
 217        if (sk->sk_state != SMC_CLOSED)
 218                return;
 219        if (!sock_flag(sk, SOCK_DEAD))
 220                return;
 221
 222        sk_refcnt_debug_dec(sk);
 223}
 224
 225static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 226                                   int protocol)
 227{
 228        struct smc_sock *smc;
 229        struct proto *prot;
 230        struct sock *sk;
 231
 232        prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 233        sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 234        if (!sk)
 235                return NULL;
 236
 237        sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 238        sk->sk_state = SMC_INIT;
 239        sk->sk_destruct = smc_destruct;
 240        sk->sk_protocol = protocol;
 241        smc = smc_sk(sk);
 242        INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 243        INIT_WORK(&smc->connect_work, smc_connect_work);
 244        INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 245        INIT_LIST_HEAD(&smc->accept_q);
 246        spin_lock_init(&smc->accept_q_lock);
 247        spin_lock_init(&smc->conn.send_lock);
 248        sk->sk_prot->hash(sk);
 249        sk_refcnt_debug_inc(sk);
 250        mutex_init(&smc->clcsock_release_lock);
 251
 252        return sk;
 253}
 254
 255static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 256                    int addr_len)
 257{
 258        struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 259        struct sock *sk = sock->sk;
 260        struct smc_sock *smc;
 261        int rc;
 262
 263        smc = smc_sk(sk);
 264
 265        /* replicate tests from inet_bind(), to be safe wrt. future changes */
 266        rc = -EINVAL;
 267        if (addr_len < sizeof(struct sockaddr_in))
 268                goto out;
 269
 270        rc = -EAFNOSUPPORT;
 271        if (addr->sin_family != AF_INET &&
 272            addr->sin_family != AF_INET6 &&
 273            addr->sin_family != AF_UNSPEC)
 274                goto out;
 275        /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 276        if (addr->sin_family == AF_UNSPEC &&
 277            addr->sin_addr.s_addr != htonl(INADDR_ANY))
 278                goto out;
 279
 280        lock_sock(sk);
 281
 282        /* Check if socket is already active */
 283        rc = -EINVAL;
 284        if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 285                goto out_rel;
 286
 287        smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 288        rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 289
 290out_rel:
 291        release_sock(sk);
 292out:
 293        return rc;
 294}
 295
 296static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 297                                   unsigned long mask)
 298{
 299        /* options we don't get control via setsockopt for */
 300        nsk->sk_type = osk->sk_type;
 301        nsk->sk_sndbuf = osk->sk_sndbuf;
 302        nsk->sk_rcvbuf = osk->sk_rcvbuf;
 303        nsk->sk_sndtimeo = osk->sk_sndtimeo;
 304        nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 305        nsk->sk_mark = osk->sk_mark;
 306        nsk->sk_priority = osk->sk_priority;
 307        nsk->sk_rcvlowat = osk->sk_rcvlowat;
 308        nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 309        nsk->sk_err = osk->sk_err;
 310
 311        nsk->sk_flags &= ~mask;
 312        nsk->sk_flags |= osk->sk_flags & mask;
 313}
 314
 315#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 316                             (1UL << SOCK_KEEPOPEN) | \
 317                             (1UL << SOCK_LINGER) | \
 318                             (1UL << SOCK_BROADCAST) | \
 319                             (1UL << SOCK_TIMESTAMP) | \
 320                             (1UL << SOCK_DBG) | \
 321                             (1UL << SOCK_RCVTSTAMP) | \
 322                             (1UL << SOCK_RCVTSTAMPNS) | \
 323                             (1UL << SOCK_LOCALROUTE) | \
 324                             (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 325                             (1UL << SOCK_RXQ_OVFL) | \
 326                             (1UL << SOCK_WIFI_STATUS) | \
 327                             (1UL << SOCK_NOFCS) | \
 328                             (1UL << SOCK_FILTER_LOCKED) | \
 329                             (1UL << SOCK_TSTAMP_NEW))
 330/* copy only relevant settings and flags of SOL_SOCKET level from smc to
 331 * clc socket (since smc is not called for these options from net/core)
 332 */
 333static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 334{
 335        smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 336}
 337
 338#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 339                             (1UL << SOCK_KEEPOPEN) | \
 340                             (1UL << SOCK_LINGER) | \
 341                             (1UL << SOCK_DBG))
 342/* copy only settings and flags relevant for smc from clc to smc socket */
 343static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 344{
 345        smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 346}
 347
 348/* register the new rmb on all links */
 349static int smcr_lgr_reg_rmbs(struct smc_link *link,
 350                             struct smc_buf_desc *rmb_desc)
 351{
 352        struct smc_link_group *lgr = link->lgr;
 353        int i, rc = 0;
 354
 355        rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 356        if (rc)
 357                return rc;
 358        /* protect against parallel smc_llc_cli_rkey_exchange() and
 359         * parallel smcr_link_reg_rmb()
 360         */
 361        mutex_lock(&lgr->llc_conf_mutex);
 362        for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 363                if (!smc_link_active(&lgr->lnk[i]))
 364                        continue;
 365                rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 366                if (rc)
 367                        goto out;
 368        }
 369
 370        /* exchange confirm_rkey msg with peer */
 371        rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 372        if (rc) {
 373                rc = -EFAULT;
 374                goto out;
 375        }
 376        rmb_desc->is_conf_rkey = true;
 377out:
 378        mutex_unlock(&lgr->llc_conf_mutex);
 379        smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 380        return rc;
 381}
 382
 383static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 384{
 385        struct smc_link *link = smc->conn.lnk;
 386        struct smc_llc_qentry *qentry;
 387        int rc;
 388
 389        /* receive CONFIRM LINK request from server over RoCE fabric */
 390        qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 391                              SMC_LLC_CONFIRM_LINK);
 392        if (!qentry) {
 393                struct smc_clc_msg_decline dclc;
 394
 395                rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 396                                      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 397                return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 398        }
 399        smc_llc_save_peer_uid(qentry);
 400        rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 401        smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 402        if (rc)
 403                return SMC_CLC_DECL_RMBE_EC;
 404
 405        rc = smc_ib_modify_qp_rts(link);
 406        if (rc)
 407                return SMC_CLC_DECL_ERR_RDYLNK;
 408
 409        smc_wr_remember_qp_attr(link);
 410
 411        if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 412                return SMC_CLC_DECL_ERR_REGRMB;
 413
 414        /* confirm_rkey is implicit on 1st contact */
 415        smc->conn.rmb_desc->is_conf_rkey = true;
 416
 417        /* send CONFIRM LINK response over RoCE fabric */
 418        rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 419        if (rc < 0)
 420                return SMC_CLC_DECL_TIMEOUT_CL;
 421
 422        smc_llc_link_active(link);
 423        smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 424
 425        /* optional 2nd link, receive ADD LINK request from server */
 426        qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 427                              SMC_LLC_ADD_LINK);
 428        if (!qentry) {
 429                struct smc_clc_msg_decline dclc;
 430
 431                rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 432                                      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 433                if (rc == -EAGAIN)
 434                        rc = 0; /* no DECLINE received, go with one link */
 435                return rc;
 436        }
 437        smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 438        smc_llc_cli_add_link(link, qentry);
 439        return 0;
 440}
 441
 442static void smcr_conn_save_peer_info(struct smc_sock *smc,
 443                                     struct smc_clc_msg_accept_confirm *clc)
 444{
 445        int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 446
 447        smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 448        smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 449        smc->conn.peer_rmbe_size = bufsize;
 450        atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 451        smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 452}
 453
 454static bool smc_isascii(char *hostname)
 455{
 456        int i;
 457
 458        for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 459                if (!isascii(hostname[i]))
 460                        return false;
 461        return true;
 462}
 463
 464static void smcd_conn_save_peer_info(struct smc_sock *smc,
 465                                     struct smc_clc_msg_accept_confirm *clc)
 466{
 467        int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 468
 469        smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 470        smc->conn.peer_token = clc->d0.token;
 471        /* msg header takes up space in the buffer */
 472        smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 473        atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 474        smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 475        if (clc->hdr.version > SMC_V1 &&
 476            (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) {
 477                struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 478                        (struct smc_clc_msg_accept_confirm_v2 *)clc;
 479                struct smc_clc_first_contact_ext *fce =
 480                        (struct smc_clc_first_contact_ext *)
 481                                (((u8 *)clc_v2) + sizeof(*clc_v2));
 482
 483                memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid,
 484                       SMC_MAX_EID_LEN);
 485                smc->conn.lgr->peer_os = fce->os_type;
 486                smc->conn.lgr->peer_smc_release = fce->release;
 487                if (smc_isascii(fce->hostname))
 488                        memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 489                               SMC_MAX_HOSTNAME_LEN);
 490        }
 491}
 492
 493static void smc_conn_save_peer_info(struct smc_sock *smc,
 494                                    struct smc_clc_msg_accept_confirm *clc)
 495{
 496        if (smc->conn.lgr->is_smcd)
 497                smcd_conn_save_peer_info(smc, clc);
 498        else
 499                smcr_conn_save_peer_info(smc, clc);
 500}
 501
 502static void smc_link_save_peer_info(struct smc_link *link,
 503                                    struct smc_clc_msg_accept_confirm *clc)
 504{
 505        link->peer_qpn = ntoh24(clc->r0.qpn);
 506        memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE);
 507        memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac));
 508        link->peer_psn = ntoh24(clc->r0.psn);
 509        link->peer_mtu = clc->r0.qp_mtu;
 510}
 511
 512static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 513                                       struct smc_stats_fback *fback_arr)
 514{
 515        int cnt;
 516
 517        for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 518                if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 519                        fback_arr[cnt].count++;
 520                        break;
 521                }
 522                if (!fback_arr[cnt].fback_code) {
 523                        fback_arr[cnt].fback_code = smc->fallback_rsn;
 524                        fback_arr[cnt].count++;
 525                        break;
 526                }
 527        }
 528}
 529
 530static void smc_stat_fallback(struct smc_sock *smc)
 531{
 532        struct net *net = sock_net(&smc->sk);
 533
 534        mutex_lock(&net->smc.mutex_fback_rsn);
 535        if (smc->listen_smc) {
 536                smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 537                net->smc.fback_rsn->srv_fback_cnt++;
 538        } else {
 539                smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 540                net->smc.fback_rsn->clnt_fback_cnt++;
 541        }
 542        mutex_unlock(&net->smc.mutex_fback_rsn);
 543}
 544
 545static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 546{
 547        smc->use_fallback = true;
 548        smc->fallback_rsn = reason_code;
 549        smc_stat_fallback(smc);
 550        if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 551                smc->clcsock->file = smc->sk.sk_socket->file;
 552                smc->clcsock->file->private_data = smc->clcsock;
 553                smc->clcsock->wq.fasync_list =
 554                        smc->sk.sk_socket->wq.fasync_list;
 555        }
 556}
 557
 558/* fall back during connect */
 559static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 560{
 561        smc_switch_to_fallback(smc, reason_code);
 562        smc_copy_sock_settings_to_clc(smc);
 563        smc->connect_nonblock = 0;
 564        if (smc->sk.sk_state == SMC_INIT)
 565                smc->sk.sk_state = SMC_ACTIVE;
 566        return 0;
 567}
 568
 569/* decline and fall back during connect */
 570static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 571                                        u8 version)
 572{
 573        struct net *net = sock_net(&smc->sk);
 574        int rc;
 575
 576        if (reason_code < 0) { /* error, fallback is not possible */
 577                this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 578                if (smc->sk.sk_state == SMC_INIT)
 579                        sock_put(&smc->sk); /* passive closing */
 580                return reason_code;
 581        }
 582        if (reason_code != SMC_CLC_DECL_PEERDECL) {
 583                rc = smc_clc_send_decline(smc, reason_code, version);
 584                if (rc < 0) {
 585                        this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 586                        if (smc->sk.sk_state == SMC_INIT)
 587                                sock_put(&smc->sk); /* passive closing */
 588                        return rc;
 589                }
 590        }
 591        return smc_connect_fallback(smc, reason_code);
 592}
 593
 594static void smc_conn_abort(struct smc_sock *smc, int local_first)
 595{
 596        if (local_first)
 597                smc_lgr_cleanup_early(&smc->conn);
 598        else
 599                smc_conn_free(&smc->conn);
 600}
 601
 602/* check if there is a rdma device available for this connection. */
 603/* called for connect and listen */
 604static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 605{
 606        /* PNET table look up: search active ib_device and port
 607         * within same PNETID that also contains the ethernet device
 608         * used for the internal TCP socket
 609         */
 610        smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 611        if (!ini->ib_dev)
 612                return SMC_CLC_DECL_NOSMCRDEV;
 613        return 0;
 614}
 615
 616/* check if there is an ISM device available for this connection. */
 617/* called for connect and listen */
 618static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 619{
 620        /* Find ISM device with same PNETID as connecting interface  */
 621        smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 622        if (!ini->ism_dev[0])
 623                return SMC_CLC_DECL_NOSMCDDEV;
 624        else
 625                ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 626        return 0;
 627}
 628
 629/* is chid unique for the ism devices that are already determined? */
 630static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 631                                           int cnt)
 632{
 633        int i = (!ini->ism_dev[0]) ? 1 : 0;
 634
 635        for (; i < cnt; i++)
 636                if (ini->ism_chid[i] == chid)
 637                        return false;
 638        return true;
 639}
 640
 641/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 642 * PNETID matching net_device)
 643 */
 644static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 645                                       struct smc_init_info *ini)
 646{
 647        int rc = SMC_CLC_DECL_NOSMCDDEV;
 648        struct smcd_dev *smcd;
 649        int i = 1;
 650        u16 chid;
 651
 652        if (smcd_indicated(ini->smc_type_v1))
 653                rc = 0;         /* already initialized for V1 */
 654        mutex_lock(&smcd_dev_list.mutex);
 655        list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 656                if (smcd->going_away || smcd == ini->ism_dev[0])
 657                        continue;
 658                chid = smc_ism_get_chid(smcd);
 659                if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 660                        continue;
 661                if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 662                    smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 663                        ini->ism_dev[i] = smcd;
 664                        ini->ism_chid[i] = chid;
 665                        ini->is_smcd = true;
 666                        rc = 0;
 667                        i++;
 668                        if (i > SMC_MAX_ISM_DEVS)
 669                                break;
 670                }
 671        }
 672        mutex_unlock(&smcd_dev_list.mutex);
 673        ini->ism_offered_cnt = i - 1;
 674        if (!ini->ism_dev[0] && !ini->ism_dev[1])
 675                ini->smcd_version = 0;
 676
 677        return rc;
 678}
 679
 680/* Check for VLAN ID and register it on ISM device just for CLC handshake */
 681static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 682                                      struct smc_init_info *ini)
 683{
 684        if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 685                return SMC_CLC_DECL_ISMVLANERR;
 686        return 0;
 687}
 688
 689static int smc_find_proposal_devices(struct smc_sock *smc,
 690                                     struct smc_init_info *ini)
 691{
 692        int rc = 0;
 693
 694        /* check if there is an ism device available */
 695        if (ini->smcd_version & SMC_V1) {
 696                if (smc_find_ism_device(smc, ini) ||
 697                    smc_connect_ism_vlan_setup(smc, ini)) {
 698                        if (ini->smc_type_v1 == SMC_TYPE_B)
 699                                ini->smc_type_v1 = SMC_TYPE_R;
 700                        else
 701                                ini->smc_type_v1 = SMC_TYPE_N;
 702                } /* else ISM V1 is supported for this connection */
 703                if (smc_find_rdma_device(smc, ini)) {
 704                        if (ini->smc_type_v1 == SMC_TYPE_B)
 705                                ini->smc_type_v1 = SMC_TYPE_D;
 706                        else
 707                                ini->smc_type_v1 = SMC_TYPE_N;
 708                } /* else RDMA is supported for this connection */
 709        }
 710        if (smc_ism_is_v2_capable() && smc_find_ism_v2_device_clnt(smc, ini))
 711                ini->smc_type_v2 = SMC_TYPE_N;
 712
 713        /* if neither ISM nor RDMA are supported, fallback */
 714        if (!smcr_indicated(ini->smc_type_v1) &&
 715            ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
 716                rc = SMC_CLC_DECL_NOSMCDEV;
 717
 718        return rc;
 719}
 720
 721/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 722 * used, the VLAN ID will be registered again during the connection setup.
 723 */
 724static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
 725                                        struct smc_init_info *ini)
 726{
 727        if (!smcd_indicated(ini->smc_type_v1))
 728                return 0;
 729        if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
 730                return SMC_CLC_DECL_CNFERR;
 731        return 0;
 732}
 733
 734#define SMC_CLC_MAX_ACCEPT_LEN \
 735        (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
 736         sizeof(struct smc_clc_first_contact_ext) + \
 737         sizeof(struct smc_clc_msg_trail))
 738
 739/* CLC handshake during connect */
 740static int smc_connect_clc(struct smc_sock *smc,
 741                           struct smc_clc_msg_accept_confirm_v2 *aclc2,
 742                           struct smc_init_info *ini)
 743{
 744        int rc = 0;
 745
 746        /* do inband token exchange */
 747        rc = smc_clc_send_proposal(smc, ini);
 748        if (rc)
 749                return rc;
 750        /* receive SMC Accept CLC message */
 751        return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
 752                                SMC_CLC_ACCEPT, CLC_WAIT_TIME);
 753}
 754
 755/* setup for RDMA connection of client */
 756static int smc_connect_rdma(struct smc_sock *smc,
 757                            struct smc_clc_msg_accept_confirm *aclc,
 758                            struct smc_init_info *ini)
 759{
 760        int i, reason_code = 0;
 761        struct smc_link *link;
 762
 763        ini->is_smcd = false;
 764        ini->ib_lcl = &aclc->r0.lcl;
 765        ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
 766        ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 767
 768        mutex_lock(&smc_client_lgr_pending);
 769        reason_code = smc_conn_create(smc, ini);
 770        if (reason_code) {
 771                mutex_unlock(&smc_client_lgr_pending);
 772                return reason_code;
 773        }
 774
 775        smc_conn_save_peer_info(smc, aclc);
 776
 777        if (ini->first_contact_local) {
 778                link = smc->conn.lnk;
 779        } else {
 780                /* set link that was assigned by server */
 781                link = NULL;
 782                for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 783                        struct smc_link *l = &smc->conn.lgr->lnk[i];
 784
 785                        if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
 786                            !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
 787                                    SMC_GID_SIZE) &&
 788                            !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
 789                                    sizeof(l->peer_mac))) {
 790                                link = l;
 791                                break;
 792                        }
 793                }
 794                if (!link) {
 795                        reason_code = SMC_CLC_DECL_NOSRVLINK;
 796                        goto connect_abort;
 797                }
 798                smc_switch_link_and_count(&smc->conn, link);
 799        }
 800
 801        /* create send buffer and rmb */
 802        if (smc_buf_create(smc, false)) {
 803                reason_code = SMC_CLC_DECL_MEM;
 804                goto connect_abort;
 805        }
 806
 807        if (ini->first_contact_local)
 808                smc_link_save_peer_info(link, aclc);
 809
 810        if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
 811                reason_code = SMC_CLC_DECL_ERR_RTOK;
 812                goto connect_abort;
 813        }
 814
 815        smc_close_init(smc);
 816        smc_rx_init(smc);
 817
 818        if (ini->first_contact_local) {
 819                if (smc_ib_ready_link(link)) {
 820                        reason_code = SMC_CLC_DECL_ERR_RDYLNK;
 821                        goto connect_abort;
 822                }
 823        } else {
 824                if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
 825                        reason_code = SMC_CLC_DECL_ERR_REGRMB;
 826                        goto connect_abort;
 827                }
 828        }
 829        smc_rmb_sync_sg_for_device(&smc->conn);
 830
 831        reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
 832                                           SMC_V1);
 833        if (reason_code)
 834                goto connect_abort;
 835
 836        smc_tx_init(smc);
 837
 838        if (ini->first_contact_local) {
 839                /* QP confirmation over RoCE fabric */
 840                smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
 841                reason_code = smcr_clnt_conf_first_link(smc);
 842                smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
 843                if (reason_code)
 844                        goto connect_abort;
 845        }
 846        mutex_unlock(&smc_client_lgr_pending);
 847
 848        smc_copy_sock_settings_to_clc(smc);
 849        smc->connect_nonblock = 0;
 850        if (smc->sk.sk_state == SMC_INIT)
 851                smc->sk.sk_state = SMC_ACTIVE;
 852
 853        return 0;
 854connect_abort:
 855        smc_conn_abort(smc, ini->first_contact_local);
 856        mutex_unlock(&smc_client_lgr_pending);
 857        smc->connect_nonblock = 0;
 858
 859        return reason_code;
 860}
 861
 862/* The server has chosen one of the proposed ISM devices for the communication.
 863 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
 864 */
 865static int
 866smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
 867                               struct smc_init_info *ini)
 868{
 869        int i;
 870
 871        for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
 872                if (ini->ism_chid[i] == ntohs(aclc->chid)) {
 873                        ini->ism_selected = i;
 874                        return 0;
 875                }
 876        }
 877
 878        return -EPROTO;
 879}
 880
 881/* setup for ISM connection of client */
 882static int smc_connect_ism(struct smc_sock *smc,
 883                           struct smc_clc_msg_accept_confirm *aclc,
 884                           struct smc_init_info *ini)
 885{
 886        int rc = 0;
 887
 888        ini->is_smcd = true;
 889        ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 890
 891        if (aclc->hdr.version == SMC_V2) {
 892                struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
 893                        (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 894
 895                rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
 896                if (rc)
 897                        return rc;
 898        }
 899        ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
 900
 901        /* there is only one lgr role for SMC-D; use server lock */
 902        mutex_lock(&smc_server_lgr_pending);
 903        rc = smc_conn_create(smc, ini);
 904        if (rc) {
 905                mutex_unlock(&smc_server_lgr_pending);
 906                return rc;
 907        }
 908
 909        /* Create send and receive buffers */
 910        rc = smc_buf_create(smc, true);
 911        if (rc) {
 912                rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
 913                goto connect_abort;
 914        }
 915
 916        smc_conn_save_peer_info(smc, aclc);
 917        smc_close_init(smc);
 918        smc_rx_init(smc);
 919        smc_tx_init(smc);
 920
 921        rc = smc_clc_send_confirm(smc, ini->first_contact_local,
 922                                  aclc->hdr.version);
 923        if (rc)
 924                goto connect_abort;
 925        mutex_unlock(&smc_server_lgr_pending);
 926
 927        smc_copy_sock_settings_to_clc(smc);
 928        smc->connect_nonblock = 0;
 929        if (smc->sk.sk_state == SMC_INIT)
 930                smc->sk.sk_state = SMC_ACTIVE;
 931
 932        return 0;
 933connect_abort:
 934        smc_conn_abort(smc, ini->first_contact_local);
 935        mutex_unlock(&smc_server_lgr_pending);
 936        smc->connect_nonblock = 0;
 937
 938        return rc;
 939}
 940
 941/* check if received accept type and version matches a proposed one */
 942static int smc_connect_check_aclc(struct smc_init_info *ini,
 943                                  struct smc_clc_msg_accept_confirm *aclc)
 944{
 945        if ((aclc->hdr.typev1 == SMC_TYPE_R &&
 946             !smcr_indicated(ini->smc_type_v1)) ||
 947            (aclc->hdr.typev1 == SMC_TYPE_D &&
 948             ((!smcd_indicated(ini->smc_type_v1) &&
 949               !smcd_indicated(ini->smc_type_v2)) ||
 950              (aclc->hdr.version == SMC_V1 &&
 951               !smcd_indicated(ini->smc_type_v1)) ||
 952              (aclc->hdr.version == SMC_V2 &&
 953               !smcd_indicated(ini->smc_type_v2)))))
 954                return SMC_CLC_DECL_MODEUNSUPP;
 955
 956        return 0;
 957}
 958
 959/* perform steps before actually connecting */
 960static int __smc_connect(struct smc_sock *smc)
 961{
 962        u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
 963        struct smc_clc_msg_accept_confirm_v2 *aclc2;
 964        struct smc_clc_msg_accept_confirm *aclc;
 965        struct smc_init_info *ini = NULL;
 966        u8 *buf = NULL;
 967        int rc = 0;
 968
 969        if (smc->use_fallback)
 970                return smc_connect_fallback(smc, smc->fallback_rsn);
 971
 972        /* if peer has not signalled SMC-capability, fall back */
 973        if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 974                return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 975
 976        /* IPSec connections opt out of SMC optimizations */
 977        if (using_ipsec(smc))
 978                return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
 979                                                    version);
 980
 981        ini = kzalloc(sizeof(*ini), GFP_KERNEL);
 982        if (!ini)
 983                return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
 984                                                    version);
 985
 986        ini->smcd_version = SMC_V1;
 987        ini->smcd_version |= smc_ism_is_v2_capable() ? SMC_V2 : 0;
 988        ini->smc_type_v1 = SMC_TYPE_B;
 989        ini->smc_type_v2 = smc_ism_is_v2_capable() ? SMC_TYPE_D : SMC_TYPE_N;
 990
 991        /* get vlan id from IP device */
 992        if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
 993                ini->smcd_version &= ~SMC_V1;
 994                ini->smc_type_v1 = SMC_TYPE_N;
 995                if (!ini->smcd_version) {
 996                        rc = SMC_CLC_DECL_GETVLANERR;
 997                        goto fallback;
 998                }
 999        }
1000
1001        rc = smc_find_proposal_devices(smc, ini);
1002        if (rc)
1003                goto fallback;
1004
1005        buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1006        if (!buf) {
1007                rc = SMC_CLC_DECL_MEM;
1008                goto fallback;
1009        }
1010        aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1011        aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1012
1013        /* perform CLC handshake */
1014        rc = smc_connect_clc(smc, aclc2, ini);
1015        if (rc)
1016                goto vlan_cleanup;
1017
1018        /* check if smc modes and versions of CLC proposal and accept match */
1019        rc = smc_connect_check_aclc(ini, aclc);
1020        version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1021        ini->smcd_version = version;
1022        if (rc)
1023                goto vlan_cleanup;
1024
1025        /* depending on previous steps, connect using rdma or ism */
1026        if (aclc->hdr.typev1 == SMC_TYPE_R)
1027                rc = smc_connect_rdma(smc, aclc, ini);
1028        else if (aclc->hdr.typev1 == SMC_TYPE_D)
1029                rc = smc_connect_ism(smc, aclc, ini);
1030        if (rc)
1031                goto vlan_cleanup;
1032
1033        SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1034        smc_connect_ism_vlan_cleanup(smc, ini);
1035        kfree(buf);
1036        kfree(ini);
1037        return 0;
1038
1039vlan_cleanup:
1040        smc_connect_ism_vlan_cleanup(smc, ini);
1041        kfree(buf);
1042fallback:
1043        kfree(ini);
1044        return smc_connect_decline_fallback(smc, rc, version);
1045}
1046
1047static void smc_connect_work(struct work_struct *work)
1048{
1049        struct smc_sock *smc = container_of(work, struct smc_sock,
1050                                            connect_work);
1051        long timeo = smc->sk.sk_sndtimeo;
1052        int rc = 0;
1053
1054        if (!timeo)
1055                timeo = MAX_SCHEDULE_TIMEOUT;
1056        lock_sock(smc->clcsock->sk);
1057        if (smc->clcsock->sk->sk_err) {
1058                smc->sk.sk_err = smc->clcsock->sk->sk_err;
1059        } else if ((1 << smc->clcsock->sk->sk_state) &
1060                                        (TCPF_SYN_SENT | TCP_SYN_RECV)) {
1061                rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1062                if ((rc == -EPIPE) &&
1063                    ((1 << smc->clcsock->sk->sk_state) &
1064                                        (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1065                        rc = 0;
1066        }
1067        release_sock(smc->clcsock->sk);
1068        lock_sock(&smc->sk);
1069        if (rc != 0 || smc->sk.sk_err) {
1070                smc->sk.sk_state = SMC_CLOSED;
1071                if (rc == -EPIPE || rc == -EAGAIN)
1072                        smc->sk.sk_err = EPIPE;
1073                else if (signal_pending(current))
1074                        smc->sk.sk_err = -sock_intr_errno(timeo);
1075                sock_put(&smc->sk); /* passive closing */
1076                goto out;
1077        }
1078
1079        rc = __smc_connect(smc);
1080        if (rc < 0)
1081                smc->sk.sk_err = -rc;
1082
1083out:
1084        if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1085                if (smc->sk.sk_err) {
1086                        smc->sk.sk_state_change(&smc->sk);
1087                } else { /* allow polling before and after fallback decision */
1088                        smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1089                        smc->sk.sk_write_space(&smc->sk);
1090                }
1091        }
1092        release_sock(&smc->sk);
1093}
1094
1095static int smc_connect(struct socket *sock, struct sockaddr *addr,
1096                       int alen, int flags)
1097{
1098        struct sock *sk = sock->sk;
1099        struct smc_sock *smc;
1100        int rc = -EINVAL;
1101
1102        smc = smc_sk(sk);
1103
1104        /* separate smc parameter checking to be safe */
1105        if (alen < sizeof(addr->sa_family))
1106                goto out_err;
1107        if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1108                goto out_err;
1109
1110        lock_sock(sk);
1111        switch (sk->sk_state) {
1112        default:
1113                goto out;
1114        case SMC_ACTIVE:
1115                rc = -EISCONN;
1116                goto out;
1117        case SMC_INIT:
1118                break;
1119        }
1120
1121        smc_copy_sock_settings_to_clc(smc);
1122        tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1123        if (smc->connect_nonblock) {
1124                rc = -EALREADY;
1125                goto out;
1126        }
1127        rc = kernel_connect(smc->clcsock, addr, alen, flags);
1128        if (rc && rc != -EINPROGRESS)
1129                goto out;
1130
1131        sock_hold(&smc->sk); /* sock put in passive closing */
1132        if (smc->use_fallback)
1133                goto out;
1134        if (flags & O_NONBLOCK) {
1135                if (queue_work(smc_hs_wq, &smc->connect_work))
1136                        smc->connect_nonblock = 1;
1137                rc = -EINPROGRESS;
1138        } else {
1139                rc = __smc_connect(smc);
1140                if (rc < 0)
1141                        goto out;
1142                else
1143                        rc = 0; /* success cases including fallback */
1144        }
1145
1146out:
1147        release_sock(sk);
1148out_err:
1149        return rc;
1150}
1151
1152static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1153{
1154        struct socket *new_clcsock = NULL;
1155        struct sock *lsk = &lsmc->sk;
1156        struct sock *new_sk;
1157        int rc = -EINVAL;
1158
1159        release_sock(lsk);
1160        new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1161        if (!new_sk) {
1162                rc = -ENOMEM;
1163                lsk->sk_err = ENOMEM;
1164                *new_smc = NULL;
1165                lock_sock(lsk);
1166                goto out;
1167        }
1168        *new_smc = smc_sk(new_sk);
1169
1170        mutex_lock(&lsmc->clcsock_release_lock);
1171        if (lsmc->clcsock)
1172                rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1173        mutex_unlock(&lsmc->clcsock_release_lock);
1174        lock_sock(lsk);
1175        if  (rc < 0 && rc != -EAGAIN)
1176                lsk->sk_err = -rc;
1177        if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1178                new_sk->sk_prot->unhash(new_sk);
1179                if (new_clcsock)
1180                        sock_release(new_clcsock);
1181                new_sk->sk_state = SMC_CLOSED;
1182                sock_set_flag(new_sk, SOCK_DEAD);
1183                sock_put(new_sk); /* final */
1184                *new_smc = NULL;
1185                goto out;
1186        }
1187
1188        /* new clcsock has inherited the smc listen-specific sk_data_ready
1189         * function; switch it back to the original sk_data_ready function
1190         */
1191        new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1192        (*new_smc)->clcsock = new_clcsock;
1193out:
1194        return rc;
1195}
1196
1197/* add a just created sock to the accept queue of the listen sock as
1198 * candidate for a following socket accept call from user space
1199 */
1200static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1201{
1202        struct smc_sock *par = smc_sk(parent);
1203
1204        sock_hold(sk); /* sock_put in smc_accept_unlink () */
1205        spin_lock(&par->accept_q_lock);
1206        list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1207        spin_unlock(&par->accept_q_lock);
1208        sk_acceptq_added(parent);
1209}
1210
1211/* remove a socket from the accept queue of its parental listening socket */
1212static void smc_accept_unlink(struct sock *sk)
1213{
1214        struct smc_sock *par = smc_sk(sk)->listen_smc;
1215
1216        spin_lock(&par->accept_q_lock);
1217        list_del_init(&smc_sk(sk)->accept_q);
1218        spin_unlock(&par->accept_q_lock);
1219        sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1220        sock_put(sk); /* sock_hold in smc_accept_enqueue */
1221}
1222
1223/* remove a sock from the accept queue to bind it to a new socket created
1224 * for a socket accept call from user space
1225 */
1226struct sock *smc_accept_dequeue(struct sock *parent,
1227                                struct socket *new_sock)
1228{
1229        struct smc_sock *isk, *n;
1230        struct sock *new_sk;
1231
1232        list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1233                new_sk = (struct sock *)isk;
1234
1235                smc_accept_unlink(new_sk);
1236                if (new_sk->sk_state == SMC_CLOSED) {
1237                        new_sk->sk_prot->unhash(new_sk);
1238                        if (isk->clcsock) {
1239                                sock_release(isk->clcsock);
1240                                isk->clcsock = NULL;
1241                        }
1242                        sock_put(new_sk); /* final */
1243                        continue;
1244                }
1245                if (new_sock) {
1246                        sock_graft(new_sk, new_sock);
1247                        if (isk->use_fallback) {
1248                                smc_sk(new_sk)->clcsock->file = new_sock->file;
1249                                isk->clcsock->file->private_data = isk->clcsock;
1250                        }
1251                }
1252                return new_sk;
1253        }
1254        return NULL;
1255}
1256
1257/* clean up for a created but never accepted sock */
1258void smc_close_non_accepted(struct sock *sk)
1259{
1260        struct smc_sock *smc = smc_sk(sk);
1261
1262        sock_hold(sk); /* sock_put below */
1263        lock_sock(sk);
1264        if (!sk->sk_lingertime)
1265                /* wait for peer closing */
1266                sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1267        __smc_release(smc);
1268        release_sock(sk);
1269        sock_put(sk); /* sock_hold above */
1270        sock_put(sk); /* final sock_put */
1271}
1272
1273static int smcr_serv_conf_first_link(struct smc_sock *smc)
1274{
1275        struct smc_link *link = smc->conn.lnk;
1276        struct smc_llc_qentry *qentry;
1277        int rc;
1278
1279        if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1280                return SMC_CLC_DECL_ERR_REGRMB;
1281
1282        /* send CONFIRM LINK request to client over the RoCE fabric */
1283        rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1284        if (rc < 0)
1285                return SMC_CLC_DECL_TIMEOUT_CL;
1286
1287        /* receive CONFIRM LINK response from client over the RoCE fabric */
1288        qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1289                              SMC_LLC_CONFIRM_LINK);
1290        if (!qentry) {
1291                struct smc_clc_msg_decline dclc;
1292
1293                rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1294                                      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1295                return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1296        }
1297        smc_llc_save_peer_uid(qentry);
1298        rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1299        smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1300        if (rc)
1301                return SMC_CLC_DECL_RMBE_EC;
1302
1303        /* confirm_rkey is implicit on 1st contact */
1304        smc->conn.rmb_desc->is_conf_rkey = true;
1305
1306        smc_llc_link_active(link);
1307        smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1308
1309        /* initial contact - try to establish second link */
1310        smc_llc_srv_add_link(link);
1311        return 0;
1312}
1313
1314/* listen worker: finish */
1315static void smc_listen_out(struct smc_sock *new_smc)
1316{
1317        struct smc_sock *lsmc = new_smc->listen_smc;
1318        struct sock *newsmcsk = &new_smc->sk;
1319
1320        if (lsmc->sk.sk_state == SMC_LISTEN) {
1321                lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1322                smc_accept_enqueue(&lsmc->sk, newsmcsk);
1323                release_sock(&lsmc->sk);
1324        } else { /* no longer listening */
1325                smc_close_non_accepted(newsmcsk);
1326        }
1327
1328        /* Wake up accept */
1329        lsmc->sk.sk_data_ready(&lsmc->sk);
1330        sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1331}
1332
1333/* listen worker: finish in state connected */
1334static void smc_listen_out_connected(struct smc_sock *new_smc)
1335{
1336        struct sock *newsmcsk = &new_smc->sk;
1337
1338        sk_refcnt_debug_inc(newsmcsk);
1339        if (newsmcsk->sk_state == SMC_INIT)
1340                newsmcsk->sk_state = SMC_ACTIVE;
1341
1342        smc_listen_out(new_smc);
1343}
1344
1345/* listen worker: finish in error state */
1346static void smc_listen_out_err(struct smc_sock *new_smc)
1347{
1348        struct sock *newsmcsk = &new_smc->sk;
1349        struct net *net = sock_net(newsmcsk);
1350
1351        this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1352        if (newsmcsk->sk_state == SMC_INIT)
1353                sock_put(&new_smc->sk); /* passive closing */
1354        newsmcsk->sk_state = SMC_CLOSED;
1355
1356        smc_listen_out(new_smc);
1357}
1358
1359/* listen worker: decline and fall back if possible */
1360static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1361                               int local_first, u8 version)
1362{
1363        /* RDMA setup failed, switch back to TCP */
1364        smc_conn_abort(new_smc, local_first);
1365        if (reason_code < 0) { /* error, no fallback possible */
1366                smc_listen_out_err(new_smc);
1367                return;
1368        }
1369        smc_switch_to_fallback(new_smc, reason_code);
1370        if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1371                if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1372                        smc_listen_out_err(new_smc);
1373                        return;
1374                }
1375        }
1376        smc_listen_out_connected(new_smc);
1377}
1378
1379/* listen worker: version checking */
1380static int smc_listen_v2_check(struct smc_sock *new_smc,
1381                               struct smc_clc_msg_proposal *pclc,
1382                               struct smc_init_info *ini)
1383{
1384        struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1385        struct smc_clc_v2_extension *pclc_v2_ext;
1386        int rc = SMC_CLC_DECL_PEERNOSMC;
1387
1388        ini->smc_type_v1 = pclc->hdr.typev1;
1389        ini->smc_type_v2 = pclc->hdr.typev2;
1390        ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0;
1391        if (pclc->hdr.version > SMC_V1)
1392                ini->smcd_version |=
1393                                ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0;
1394        if (!(ini->smcd_version & SMC_V2)) {
1395                rc = SMC_CLC_DECL_PEERNOSMC;
1396                goto out;
1397        }
1398        if (!smc_ism_is_v2_capable()) {
1399                ini->smcd_version &= ~SMC_V2;
1400                rc = SMC_CLC_DECL_NOISM2SUPP;
1401                goto out;
1402        }
1403        pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1404        if (!pclc_v2_ext) {
1405                ini->smcd_version &= ~SMC_V2;
1406                rc = SMC_CLC_DECL_NOV2EXT;
1407                goto out;
1408        }
1409        pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1410        if (!pclc_smcd_v2_ext) {
1411                ini->smcd_version &= ~SMC_V2;
1412                rc = SMC_CLC_DECL_NOV2DEXT;
1413        }
1414
1415out:
1416        if (!ini->smcd_version)
1417                return rc;
1418
1419        return 0;
1420}
1421
1422/* listen worker: check prefixes */
1423static int smc_listen_prfx_check(struct smc_sock *new_smc,
1424                                 struct smc_clc_msg_proposal *pclc)
1425{
1426        struct smc_clc_msg_proposal_prefix *pclc_prfx;
1427        struct socket *newclcsock = new_smc->clcsock;
1428
1429        if (pclc->hdr.typev1 == SMC_TYPE_N)
1430                return 0;
1431        pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1432        if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1433                return SMC_CLC_DECL_DIFFPREFIX;
1434
1435        return 0;
1436}
1437
1438/* listen worker: initialize connection and buffers */
1439static int smc_listen_rdma_init(struct smc_sock *new_smc,
1440                                struct smc_init_info *ini)
1441{
1442        int rc;
1443
1444        /* allocate connection / link group */
1445        rc = smc_conn_create(new_smc, ini);
1446        if (rc)
1447                return rc;
1448
1449        /* create send buffer and rmb */
1450        if (smc_buf_create(new_smc, false))
1451                return SMC_CLC_DECL_MEM;
1452
1453        return 0;
1454}
1455
1456/* listen worker: initialize connection and buffers for SMC-D */
1457static int smc_listen_ism_init(struct smc_sock *new_smc,
1458                               struct smc_init_info *ini)
1459{
1460        int rc;
1461
1462        rc = smc_conn_create(new_smc, ini);
1463        if (rc)
1464                return rc;
1465
1466        /* Create send and receive buffers */
1467        rc = smc_buf_create(new_smc, true);
1468        if (rc) {
1469                smc_conn_abort(new_smc, ini->first_contact_local);
1470                return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1471                                         SMC_CLC_DECL_MEM;
1472        }
1473
1474        return 0;
1475}
1476
1477static bool smc_is_already_selected(struct smcd_dev *smcd,
1478                                    struct smc_init_info *ini,
1479                                    int matches)
1480{
1481        int i;
1482
1483        for (i = 0; i < matches; i++)
1484                if (smcd == ini->ism_dev[i])
1485                        return true;
1486
1487        return false;
1488}
1489
1490/* check for ISM devices matching proposed ISM devices */
1491static void smc_check_ism_v2_match(struct smc_init_info *ini,
1492                                   u16 proposed_chid, u64 proposed_gid,
1493                                   unsigned int *matches)
1494{
1495        struct smcd_dev *smcd;
1496
1497        list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1498                if (smcd->going_away)
1499                        continue;
1500                if (smc_is_already_selected(smcd, ini, *matches))
1501                        continue;
1502                if (smc_ism_get_chid(smcd) == proposed_chid &&
1503                    !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1504                        ini->ism_peer_gid[*matches] = proposed_gid;
1505                        ini->ism_dev[*matches] = smcd;
1506                        (*matches)++;
1507                        break;
1508                }
1509        }
1510}
1511
1512static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1513{
1514        if (!ini->rc)
1515                ini->rc = rc;
1516}
1517
1518static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1519                                        struct smc_clc_msg_proposal *pclc,
1520                                        struct smc_init_info *ini)
1521{
1522        struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1523        struct smc_clc_v2_extension *smc_v2_ext;
1524        struct smc_clc_msg_smcd *pclc_smcd;
1525        unsigned int matches = 0;
1526        u8 smcd_version;
1527        u8 *eid = NULL;
1528        int i, rc;
1529
1530        if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1531                goto not_found;
1532
1533        pclc_smcd = smc_get_clc_msg_smcd(pclc);
1534        smc_v2_ext = smc_get_clc_v2_ext(pclc);
1535        smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1536        if (!smcd_v2_ext ||
1537            !smc_v2_ext->hdr.flag.seid) { /* no system EID support for SMCD */
1538                smc_find_ism_store_rc(SMC_CLC_DECL_NOSEID, ini);
1539                goto not_found;
1540        }
1541
1542        mutex_lock(&smcd_dev_list.mutex);
1543        if (pclc_smcd->ism.chid)
1544                /* check for ISM device matching proposed native ISM device */
1545                smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1546                                       ntohll(pclc_smcd->ism.gid), &matches);
1547        for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1548                /* check for ISM devices matching proposed non-native ISM
1549                 * devices
1550                 */
1551                smc_check_ism_v2_match(ini,
1552                                       ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1553                                       ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1554                                       &matches);
1555        }
1556        mutex_unlock(&smcd_dev_list.mutex);
1557
1558        if (ini->ism_dev[0]) {
1559                smc_ism_get_system_eid(ini->ism_dev[0], &eid);
1560                if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN))
1561                        goto not_found;
1562        } else {
1563                goto not_found;
1564        }
1565
1566        /* separate - outside the smcd_dev_list.lock */
1567        smcd_version = ini->smcd_version;
1568        for (i = 0; i < matches; i++) {
1569                ini->smcd_version = SMC_V2;
1570                ini->is_smcd = true;
1571                ini->ism_selected = i;
1572                rc = smc_listen_ism_init(new_smc, ini);
1573                if (rc) {
1574                        smc_find_ism_store_rc(rc, ini);
1575                        /* try next active ISM device */
1576                        continue;
1577                }
1578                return; /* matching and usable V2 ISM device found */
1579        }
1580        /* no V2 ISM device could be initialized */
1581        ini->smcd_version = smcd_version;       /* restore original value */
1582
1583not_found:
1584        ini->smcd_version &= ~SMC_V2;
1585        ini->ism_dev[0] = NULL;
1586        ini->is_smcd = false;
1587}
1588
1589static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1590                                        struct smc_clc_msg_proposal *pclc,
1591                                        struct smc_init_info *ini)
1592{
1593        struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
1594        int rc = 0;
1595
1596        /* check if ISM V1 is available */
1597        if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1598                goto not_found;
1599        ini->is_smcd = true; /* prepare ISM check */
1600        ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1601        rc = smc_find_ism_device(new_smc, ini);
1602        if (rc)
1603                goto not_found;
1604        ini->ism_selected = 0;
1605        rc = smc_listen_ism_init(new_smc, ini);
1606        if (!rc)
1607                return;         /* V1 ISM device found */
1608
1609not_found:
1610        smc_find_ism_store_rc(rc, ini);
1611        ini->ism_dev[0] = NULL;
1612        ini->is_smcd = false;
1613}
1614
1615/* listen worker: register buffers */
1616static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1617{
1618        struct smc_connection *conn = &new_smc->conn;
1619
1620        if (!local_first) {
1621                if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1622                        return SMC_CLC_DECL_ERR_REGRMB;
1623        }
1624        smc_rmb_sync_sg_for_device(&new_smc->conn);
1625
1626        return 0;
1627}
1628
1629static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1630                                        struct smc_clc_msg_proposal *pclc,
1631                                        struct smc_init_info *ini)
1632{
1633        int rc;
1634
1635        if (!smcr_indicated(ini->smc_type_v1))
1636                return SMC_CLC_DECL_NOSMCDEV;
1637
1638        /* prepare RDMA check */
1639        ini->ib_lcl = &pclc->lcl;
1640        rc = smc_find_rdma_device(new_smc, ini);
1641        if (rc) {
1642                /* no RDMA device found */
1643                if (ini->smc_type_v1 == SMC_TYPE_B)
1644                        /* neither ISM nor RDMA device found */
1645                        rc = SMC_CLC_DECL_NOSMCDEV;
1646                return rc;
1647        }
1648        rc = smc_listen_rdma_init(new_smc, ini);
1649        if (rc)
1650                return rc;
1651        return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1652}
1653
1654/* determine the local device matching to proposal */
1655static int smc_listen_find_device(struct smc_sock *new_smc,
1656                                  struct smc_clc_msg_proposal *pclc,
1657                                  struct smc_init_info *ini)
1658{
1659        int rc;
1660
1661        /* check for ISM device matching V2 proposed device */
1662        smc_find_ism_v2_device_serv(new_smc, pclc, ini);
1663        if (ini->ism_dev[0])
1664                return 0;
1665
1666        if (!(ini->smcd_version & SMC_V1))
1667                return ini->rc ?: SMC_CLC_DECL_NOSMCD2DEV;
1668
1669        /* check for matching IP prefix and subnet length */
1670        rc = smc_listen_prfx_check(new_smc, pclc);
1671        if (rc)
1672                return ini->rc ?: rc;
1673
1674        /* get vlan id from IP device */
1675        if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
1676                return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
1677
1678        /* check for ISM device matching V1 proposed device */
1679        smc_find_ism_v1_device_serv(new_smc, pclc, ini);
1680        if (ini->ism_dev[0])
1681                return 0;
1682
1683        if (pclc->hdr.typev1 == SMC_TYPE_D)
1684                /* skip RDMA and decline */
1685                return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
1686
1687        /* check if RDMA is available */
1688        rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1689        smc_find_ism_store_rc(rc, ini);
1690
1691        return (!rc) ? 0 : ini->rc;
1692}
1693
1694/* listen worker: finish RDMA setup */
1695static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1696                                  struct smc_clc_msg_accept_confirm *cclc,
1697                                  bool local_first)
1698{
1699        struct smc_link *link = new_smc->conn.lnk;
1700        int reason_code = 0;
1701
1702        if (local_first)
1703                smc_link_save_peer_info(link, cclc);
1704
1705        if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
1706                return SMC_CLC_DECL_ERR_RTOK;
1707
1708        if (local_first) {
1709                if (smc_ib_ready_link(link))
1710                        return SMC_CLC_DECL_ERR_RDYLNK;
1711                /* QP confirmation over RoCE fabric */
1712                smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1713                reason_code = smcr_serv_conf_first_link(new_smc);
1714                smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1715        }
1716        return reason_code;
1717}
1718
1719/* setup for connection of server */
1720static void smc_listen_work(struct work_struct *work)
1721{
1722        struct smc_sock *new_smc = container_of(work, struct smc_sock,
1723                                                smc_listen_work);
1724        u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1725        struct socket *newclcsock = new_smc->clcsock;
1726        struct smc_clc_msg_accept_confirm *cclc;
1727        struct smc_clc_msg_proposal_area *buf;
1728        struct smc_clc_msg_proposal *pclc;
1729        struct smc_init_info *ini = NULL;
1730        int rc = 0;
1731
1732        if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1733                return smc_listen_out_err(new_smc);
1734
1735        if (new_smc->use_fallback) {
1736                smc_listen_out_connected(new_smc);
1737                return;
1738        }
1739
1740        /* check if peer is smc capable */
1741        if (!tcp_sk(newclcsock->sk)->syn_smc) {
1742                smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
1743                smc_listen_out_connected(new_smc);
1744                return;
1745        }
1746
1747        /* do inband token exchange -
1748         * wait for and receive SMC Proposal CLC message
1749         */
1750        buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1751        if (!buf) {
1752                rc = SMC_CLC_DECL_MEM;
1753                goto out_decl;
1754        }
1755        pclc = (struct smc_clc_msg_proposal *)buf;
1756        rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
1757                              SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1758        if (rc)
1759                goto out_decl;
1760        version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version;
1761
1762        /* IPSec connections opt out of SMC optimizations */
1763        if (using_ipsec(new_smc)) {
1764                rc = SMC_CLC_DECL_IPSEC;
1765                goto out_decl;
1766        }
1767
1768        ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1769        if (!ini) {
1770                rc = SMC_CLC_DECL_MEM;
1771                goto out_decl;
1772        }
1773
1774        /* initial version checking */
1775        rc = smc_listen_v2_check(new_smc, pclc, ini);
1776        if (rc)
1777                goto out_decl;
1778
1779        mutex_lock(&smc_server_lgr_pending);
1780        smc_close_init(new_smc);
1781        smc_rx_init(new_smc);
1782        smc_tx_init(new_smc);
1783
1784        /* determine ISM or RoCE device used for connection */
1785        rc = smc_listen_find_device(new_smc, pclc, ini);
1786        if (rc)
1787                goto out_unlock;
1788
1789        /* send SMC Accept CLC message */
1790        rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
1791                                 ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1);
1792        if (rc)
1793                goto out_unlock;
1794
1795        /* SMC-D does not need this lock any more */
1796        if (ini->is_smcd)
1797                mutex_unlock(&smc_server_lgr_pending);
1798
1799        /* receive SMC Confirm CLC message */
1800        memset(buf, 0, sizeof(*buf));
1801        cclc = (struct smc_clc_msg_accept_confirm *)buf;
1802        rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
1803                              SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1804        if (rc) {
1805                if (!ini->is_smcd)
1806                        goto out_unlock;
1807                goto out_decl;
1808        }
1809
1810        /* finish worker */
1811        if (!ini->is_smcd) {
1812                rc = smc_listen_rdma_finish(new_smc, cclc,
1813                                            ini->first_contact_local);
1814                if (rc)
1815                        goto out_unlock;
1816                mutex_unlock(&smc_server_lgr_pending);
1817        }
1818        smc_conn_save_peer_info(new_smc, cclc);
1819        smc_listen_out_connected(new_smc);
1820        SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
1821        goto out_free;
1822
1823out_unlock:
1824        mutex_unlock(&smc_server_lgr_pending);
1825out_decl:
1826        smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
1827                           version);
1828out_free:
1829        kfree(ini);
1830        kfree(buf);
1831}
1832
1833static void smc_tcp_listen_work(struct work_struct *work)
1834{
1835        struct smc_sock *lsmc = container_of(work, struct smc_sock,
1836                                             tcp_listen_work);
1837        struct sock *lsk = &lsmc->sk;
1838        struct smc_sock *new_smc;
1839        int rc = 0;
1840
1841        lock_sock(lsk);
1842        while (lsk->sk_state == SMC_LISTEN) {
1843                rc = smc_clcsock_accept(lsmc, &new_smc);
1844                if (rc) /* clcsock accept queue empty or error */
1845                        goto out;
1846                if (!new_smc)
1847                        continue;
1848
1849                new_smc->listen_smc = lsmc;
1850                new_smc->use_fallback = lsmc->use_fallback;
1851                new_smc->fallback_rsn = lsmc->fallback_rsn;
1852                sock_hold(lsk); /* sock_put in smc_listen_work */
1853                INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1854                smc_copy_sock_settings_to_smc(new_smc);
1855                new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1856                new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1857                sock_hold(&new_smc->sk); /* sock_put in passive closing */
1858                if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
1859                        sock_put(&new_smc->sk);
1860        }
1861
1862out:
1863        release_sock(lsk);
1864        sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
1865}
1866
1867static void smc_clcsock_data_ready(struct sock *listen_clcsock)
1868{
1869        struct smc_sock *lsmc;
1870
1871        lsmc = (struct smc_sock *)
1872               ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
1873        if (!lsmc)
1874                return;
1875        lsmc->clcsk_data_ready(listen_clcsock);
1876        if (lsmc->sk.sk_state == SMC_LISTEN) {
1877                sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
1878                if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
1879                        sock_put(&lsmc->sk);
1880        }
1881}
1882
1883static int smc_listen(struct socket *sock, int backlog)
1884{
1885        struct sock *sk = sock->sk;
1886        struct smc_sock *smc;
1887        int rc;
1888
1889        smc = smc_sk(sk);
1890        lock_sock(sk);
1891
1892        rc = -EINVAL;
1893        if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1894            smc->connect_nonblock)
1895                goto out;
1896
1897        rc = 0;
1898        if (sk->sk_state == SMC_LISTEN) {
1899                sk->sk_max_ack_backlog = backlog;
1900                goto out;
1901        }
1902        /* some socket options are handled in core, so we could not apply
1903         * them to the clc socket -- copy smc socket options to clc socket
1904         */
1905        smc_copy_sock_settings_to_clc(smc);
1906        if (!smc->use_fallback)
1907                tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1908
1909        /* save original sk_data_ready function and establish
1910         * smc-specific sk_data_ready function
1911         */
1912        smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
1913        smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
1914        smc->clcsock->sk->sk_user_data =
1915                (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
1916        rc = kernel_listen(smc->clcsock, backlog);
1917        if (rc)
1918                goto out;
1919        sk->sk_max_ack_backlog = backlog;
1920        sk->sk_ack_backlog = 0;
1921        sk->sk_state = SMC_LISTEN;
1922
1923out:
1924        release_sock(sk);
1925        return rc;
1926}
1927
1928static int smc_accept(struct socket *sock, struct socket *new_sock,
1929                      int flags, bool kern)
1930{
1931        struct sock *sk = sock->sk, *nsk;
1932        DECLARE_WAITQUEUE(wait, current);
1933        struct smc_sock *lsmc;
1934        long timeo;
1935        int rc = 0;
1936
1937        lsmc = smc_sk(sk);
1938        sock_hold(sk); /* sock_put below */
1939        lock_sock(sk);
1940
1941        if (lsmc->sk.sk_state != SMC_LISTEN) {
1942                rc = -EINVAL;
1943                release_sock(sk);
1944                goto out;
1945        }
1946
1947        /* Wait for an incoming connection */
1948        timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1949        add_wait_queue_exclusive(sk_sleep(sk), &wait);
1950        while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1951                set_current_state(TASK_INTERRUPTIBLE);
1952                if (!timeo) {
1953                        rc = -EAGAIN;
1954                        break;
1955                }
1956                release_sock(sk);
1957                timeo = schedule_timeout(timeo);
1958                /* wakeup by sk_data_ready in smc_listen_work() */
1959                sched_annotate_sleep();
1960                lock_sock(sk);
1961                if (signal_pending(current)) {
1962                        rc = sock_intr_errno(timeo);
1963                        break;
1964                }
1965        }
1966        set_current_state(TASK_RUNNING);
1967        remove_wait_queue(sk_sleep(sk), &wait);
1968
1969        if (!rc)
1970                rc = sock_error(nsk);
1971        release_sock(sk);
1972        if (rc)
1973                goto out;
1974
1975        if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1976                /* wait till data arrives on the socket */
1977                timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1978                                                                MSEC_PER_SEC);
1979                if (smc_sk(nsk)->use_fallback) {
1980                        struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1981
1982                        lock_sock(clcsk);
1983                        if (skb_queue_empty(&clcsk->sk_receive_queue))
1984                                sk_wait_data(clcsk, &timeo, NULL);
1985                        release_sock(clcsk);
1986                } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1987                        lock_sock(nsk);
1988                        smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1989                        release_sock(nsk);
1990                }
1991        }
1992
1993out:
1994        sock_put(sk); /* sock_hold above */
1995        return rc;
1996}
1997
1998static int smc_getname(struct socket *sock, struct sockaddr *addr,
1999                       int peer)
2000{
2001        struct smc_sock *smc;
2002
2003        if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2004            (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2005                return -ENOTCONN;
2006
2007        smc = smc_sk(sock->sk);
2008
2009        return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2010}
2011
2012static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2013{
2014        struct sock *sk = sock->sk;
2015        struct smc_sock *smc;
2016        int rc = -EPIPE;
2017
2018        smc = smc_sk(sk);
2019        lock_sock(sk);
2020        if ((sk->sk_state != SMC_ACTIVE) &&
2021            (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2022            (sk->sk_state != SMC_INIT))
2023                goto out;
2024
2025        if (msg->msg_flags & MSG_FASTOPEN) {
2026                if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2027                        smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2028                } else {
2029                        rc = -EINVAL;
2030                        goto out;
2031                }
2032        }
2033
2034        if (smc->use_fallback) {
2035                rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2036        } else {
2037                rc = smc_tx_sendmsg(smc, msg, len);
2038                SMC_STAT_TX_PAYLOAD(smc, len, rc);
2039        }
2040out:
2041        release_sock(sk);
2042        return rc;
2043}
2044
2045static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2046                       int flags)
2047{
2048        struct sock *sk = sock->sk;
2049        struct smc_sock *smc;
2050        int rc = -ENOTCONN;
2051
2052        smc = smc_sk(sk);
2053        lock_sock(sk);
2054        if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2055                /* socket was connected before, no more data to read */
2056                rc = 0;
2057                goto out;
2058        }
2059        if ((sk->sk_state == SMC_INIT) ||
2060            (sk->sk_state == SMC_LISTEN) ||
2061            (sk->sk_state == SMC_CLOSED))
2062                goto out;
2063
2064        if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2065                rc = 0;
2066                goto out;
2067        }
2068
2069        if (smc->use_fallback) {
2070                rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2071        } else {
2072                msg->msg_namelen = 0;
2073                rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2074                SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2075        }
2076
2077out:
2078        release_sock(sk);
2079        return rc;
2080}
2081
2082static __poll_t smc_accept_poll(struct sock *parent)
2083{
2084        struct smc_sock *isk = smc_sk(parent);
2085        __poll_t mask = 0;
2086
2087        spin_lock(&isk->accept_q_lock);
2088        if (!list_empty(&isk->accept_q))
2089                mask = EPOLLIN | EPOLLRDNORM;
2090        spin_unlock(&isk->accept_q_lock);
2091
2092        return mask;
2093}
2094
2095static __poll_t smc_poll(struct file *file, struct socket *sock,
2096                             poll_table *wait)
2097{
2098        struct sock *sk = sock->sk;
2099        struct smc_sock *smc;
2100        __poll_t mask = 0;
2101
2102        if (!sk)
2103                return EPOLLNVAL;
2104
2105        smc = smc_sk(sock->sk);
2106        if (smc->use_fallback) {
2107                /* delegate to CLC child sock */
2108                mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2109                sk->sk_err = smc->clcsock->sk->sk_err;
2110        } else {
2111                if (sk->sk_state != SMC_CLOSED)
2112                        sock_poll_wait(file, sock, wait);
2113                if (sk->sk_err)
2114                        mask |= EPOLLERR;
2115                if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2116                    (sk->sk_state == SMC_CLOSED))
2117                        mask |= EPOLLHUP;
2118                if (sk->sk_state == SMC_LISTEN) {
2119                        /* woken up by sk_data_ready in smc_listen_work() */
2120                        mask |= smc_accept_poll(sk);
2121                } else if (smc->use_fallback) { /* as result of connect_work()*/
2122                        mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2123                                                           wait);
2124                        sk->sk_err = smc->clcsock->sk->sk_err;
2125                } else {
2126                        if ((sk->sk_state != SMC_INIT &&
2127                             atomic_read(&smc->conn.sndbuf_space)) ||
2128                            sk->sk_shutdown & SEND_SHUTDOWN) {
2129                                mask |= EPOLLOUT | EPOLLWRNORM;
2130                        } else {
2131                                sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2132                                set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2133                        }
2134                        if (atomic_read(&smc->conn.bytes_to_rcv))
2135                                mask |= EPOLLIN | EPOLLRDNORM;
2136                        if (sk->sk_shutdown & RCV_SHUTDOWN)
2137                                mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2138                        if (sk->sk_state == SMC_APPCLOSEWAIT1)
2139                                mask |= EPOLLIN;
2140                        if (smc->conn.urg_state == SMC_URG_VALID)
2141                                mask |= EPOLLPRI;
2142                }
2143        }
2144
2145        return mask;
2146}
2147
2148static int smc_shutdown(struct socket *sock, int how)
2149{
2150        struct sock *sk = sock->sk;
2151        struct smc_sock *smc;
2152        int rc = -EINVAL;
2153        int rc1 = 0;
2154
2155        smc = smc_sk(sk);
2156
2157        if ((how < SHUT_RD) || (how > SHUT_RDWR))
2158                return rc;
2159
2160        lock_sock(sk);
2161
2162        rc = -ENOTCONN;
2163        if ((sk->sk_state != SMC_ACTIVE) &&
2164            (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2165            (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2166            (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2167            (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2168            (sk->sk_state != SMC_APPFINCLOSEWAIT))
2169                goto out;
2170        if (smc->use_fallback) {
2171                rc = kernel_sock_shutdown(smc->clcsock, how);
2172                sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2173                if (sk->sk_shutdown == SHUTDOWN_MASK)
2174                        sk->sk_state = SMC_CLOSED;
2175                goto out;
2176        }
2177        switch (how) {
2178        case SHUT_RDWR:         /* shutdown in both directions */
2179                rc = smc_close_active(smc);
2180                break;
2181        case SHUT_WR:
2182                rc = smc_close_shutdown_write(smc);
2183                break;
2184        case SHUT_RD:
2185                rc = 0;
2186                /* nothing more to do because peer is not involved */
2187                break;
2188        }
2189        if (smc->clcsock)
2190                rc1 = kernel_sock_shutdown(smc->clcsock, how);
2191        /* map sock_shutdown_cmd constants to sk_shutdown value range */
2192        sk->sk_shutdown |= how + 1;
2193
2194out:
2195        release_sock(sk);
2196        return rc ? rc : rc1;
2197}
2198
2199static int smc_setsockopt(struct socket *sock, int level, int optname,
2200                          sockptr_t optval, unsigned int optlen)
2201{
2202        struct sock *sk = sock->sk;
2203        struct smc_sock *smc;
2204        int val, rc;
2205
2206        if (level == SOL_TCP && optname == TCP_ULP)
2207                return -EOPNOTSUPP;
2208
2209        smc = smc_sk(sk);
2210
2211        /* generic setsockopts reaching us here always apply to the
2212         * CLC socket
2213         */
2214        if (unlikely(!smc->clcsock->ops->setsockopt))
2215                rc = -EOPNOTSUPP;
2216        else
2217                rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2218                                                   optval, optlen);
2219        if (smc->clcsock->sk->sk_err) {
2220                sk->sk_err = smc->clcsock->sk->sk_err;
2221                sk_error_report(sk);
2222        }
2223
2224        if (optlen < sizeof(int))
2225                return -EINVAL;
2226        if (copy_from_sockptr(&val, optval, sizeof(int)))
2227                return -EFAULT;
2228
2229        lock_sock(sk);
2230        if (rc || smc->use_fallback)
2231                goto out;
2232        switch (optname) {
2233        case TCP_FASTOPEN:
2234        case TCP_FASTOPEN_CONNECT:
2235        case TCP_FASTOPEN_KEY:
2236        case TCP_FASTOPEN_NO_COOKIE:
2237                /* option not supported by SMC */
2238                if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2239                        smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2240                } else {
2241                        rc = -EINVAL;
2242                }
2243                break;
2244        case TCP_NODELAY:
2245                if (sk->sk_state != SMC_INIT &&
2246                    sk->sk_state != SMC_LISTEN &&
2247                    sk->sk_state != SMC_CLOSED) {
2248                        if (val) {
2249                                SMC_STAT_INC(smc, ndly_cnt);
2250                                mod_delayed_work(smc->conn.lgr->tx_wq,
2251                                                 &smc->conn.tx_work, 0);
2252                        }
2253                }
2254                break;
2255        case TCP_CORK:
2256                if (sk->sk_state != SMC_INIT &&
2257                    sk->sk_state != SMC_LISTEN &&
2258                    sk->sk_state != SMC_CLOSED) {
2259                        if (!val) {
2260                                SMC_STAT_INC(smc, cork_cnt);
2261                                mod_delayed_work(smc->conn.lgr->tx_wq,
2262                                                 &smc->conn.tx_work, 0);
2263                        }
2264                }
2265                break;
2266        case TCP_DEFER_ACCEPT:
2267                smc->sockopt_defer_accept = val;
2268                break;
2269        default:
2270                break;
2271        }
2272out:
2273        release_sock(sk);
2274
2275        return rc;
2276}
2277
2278static int smc_getsockopt(struct socket *sock, int level, int optname,
2279                          char __user *optval, int __user *optlen)
2280{
2281        struct smc_sock *smc;
2282
2283        smc = smc_sk(sock->sk);
2284        /* socket options apply to the CLC socket */
2285        if (unlikely(!smc->clcsock->ops->getsockopt))
2286                return -EOPNOTSUPP;
2287        return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2288                                             optval, optlen);
2289}
2290
2291static int smc_ioctl(struct socket *sock, unsigned int cmd,
2292                     unsigned long arg)
2293{
2294        union smc_host_cursor cons, urg;
2295        struct smc_connection *conn;
2296        struct smc_sock *smc;
2297        int answ;
2298
2299        smc = smc_sk(sock->sk);
2300        conn = &smc->conn;
2301        lock_sock(&smc->sk);
2302        if (smc->use_fallback) {
2303                if (!smc->clcsock) {
2304                        release_sock(&smc->sk);
2305                        return -EBADF;
2306                }
2307                answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2308                release_sock(&smc->sk);
2309                return answ;
2310        }
2311        switch (cmd) {
2312        case SIOCINQ: /* same as FIONREAD */
2313                if (smc->sk.sk_state == SMC_LISTEN) {
2314                        release_sock(&smc->sk);
2315                        return -EINVAL;
2316                }
2317                if (smc->sk.sk_state == SMC_INIT ||
2318                    smc->sk.sk_state == SMC_CLOSED)
2319                        answ = 0;
2320                else
2321                        answ = atomic_read(&smc->conn.bytes_to_rcv);
2322                break;
2323        case SIOCOUTQ:
2324                /* output queue size (not send + not acked) */
2325                if (smc->sk.sk_state == SMC_LISTEN) {
2326                        release_sock(&smc->sk);
2327                        return -EINVAL;
2328                }
2329                if (smc->sk.sk_state == SMC_INIT ||
2330                    smc->sk.sk_state == SMC_CLOSED)
2331                        answ = 0;
2332                else
2333                        answ = smc->conn.sndbuf_desc->len -
2334                                        atomic_read(&smc->conn.sndbuf_space);
2335                break;
2336        case SIOCOUTQNSD:
2337                /* output queue size (not send only) */
2338                if (smc->sk.sk_state == SMC_LISTEN) {
2339                        release_sock(&smc->sk);
2340                        return -EINVAL;
2341                }
2342                if (smc->sk.sk_state == SMC_INIT ||
2343                    smc->sk.sk_state == SMC_CLOSED)
2344                        answ = 0;
2345                else
2346                        answ = smc_tx_prepared_sends(&smc->conn);
2347                break;
2348        case SIOCATMARK:
2349                if (smc->sk.sk_state == SMC_LISTEN) {
2350                        release_sock(&smc->sk);
2351                        return -EINVAL;
2352                }
2353                if (smc->sk.sk_state == SMC_INIT ||
2354                    smc->sk.sk_state == SMC_CLOSED) {
2355                        answ = 0;
2356                } else {
2357                        smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2358                        smc_curs_copy(&urg, &conn->urg_curs, conn);
2359                        answ = smc_curs_diff(conn->rmb_desc->len,
2360                                             &cons, &urg) == 1;
2361                }
2362                break;
2363        default:
2364                release_sock(&smc->sk);
2365                return -ENOIOCTLCMD;
2366        }
2367        release_sock(&smc->sk);
2368
2369        return put_user(answ, (int __user *)arg);
2370}
2371
2372static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2373                            int offset, size_t size, int flags)
2374{
2375        struct sock *sk = sock->sk;
2376        struct smc_sock *smc;
2377        int rc = -EPIPE;
2378
2379        smc = smc_sk(sk);
2380        lock_sock(sk);
2381        if (sk->sk_state != SMC_ACTIVE) {
2382                release_sock(sk);
2383                goto out;
2384        }
2385        release_sock(sk);
2386        if (smc->use_fallback) {
2387                rc = kernel_sendpage(smc->clcsock, page, offset,
2388                                     size, flags);
2389        } else {
2390                SMC_STAT_INC(smc, sendpage_cnt);
2391                rc = sock_no_sendpage(sock, page, offset, size, flags);
2392        }
2393
2394out:
2395        return rc;
2396}
2397
2398/* Map the affected portions of the rmbe into an spd, note the number of bytes
2399 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2400 * updates till whenever a respective page has been fully processed.
2401 * Note that subsequent recv() calls have to wait till all splice() processing
2402 * completed.
2403 */
2404static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2405                               struct pipe_inode_info *pipe, size_t len,
2406                               unsigned int flags)
2407{
2408        struct sock *sk = sock->sk;
2409        struct smc_sock *smc;
2410        int rc = -ENOTCONN;
2411
2412        smc = smc_sk(sk);
2413        lock_sock(sk);
2414        if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2415                /* socket was connected before, no more data to read */
2416                rc = 0;
2417                goto out;
2418        }
2419        if (sk->sk_state == SMC_INIT ||
2420            sk->sk_state == SMC_LISTEN ||
2421            sk->sk_state == SMC_CLOSED)
2422                goto out;
2423
2424        if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2425                rc = 0;
2426                goto out;
2427        }
2428
2429        if (smc->use_fallback) {
2430                rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2431                                                    pipe, len, flags);
2432        } else {
2433                if (*ppos) {
2434                        rc = -ESPIPE;
2435                        goto out;
2436                }
2437                if (flags & SPLICE_F_NONBLOCK)
2438                        flags = MSG_DONTWAIT;
2439                else
2440                        flags = 0;
2441                SMC_STAT_INC(smc, splice_cnt);
2442                rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
2443        }
2444out:
2445        release_sock(sk);
2446
2447        return rc;
2448}
2449
2450/* must look like tcp */
2451static const struct proto_ops smc_sock_ops = {
2452        .family         = PF_SMC,
2453        .owner          = THIS_MODULE,
2454        .release        = smc_release,
2455        .bind           = smc_bind,
2456        .connect        = smc_connect,
2457        .socketpair     = sock_no_socketpair,
2458        .accept         = smc_accept,
2459        .getname        = smc_getname,
2460        .poll           = smc_poll,
2461        .ioctl          = smc_ioctl,
2462        .listen         = smc_listen,
2463        .shutdown       = smc_shutdown,
2464        .setsockopt     = smc_setsockopt,
2465        .getsockopt     = smc_getsockopt,
2466        .sendmsg        = smc_sendmsg,
2467        .recvmsg        = smc_recvmsg,
2468        .mmap           = sock_no_mmap,
2469        .sendpage       = smc_sendpage,
2470        .splice_read    = smc_splice_read,
2471};
2472
2473static int smc_create(struct net *net, struct socket *sock, int protocol,
2474                      int kern)
2475{
2476        int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
2477        struct smc_sock *smc;
2478        struct sock *sk;
2479        int rc;
2480
2481        rc = -ESOCKTNOSUPPORT;
2482        if (sock->type != SOCK_STREAM)
2483                goto out;
2484
2485        rc = -EPROTONOSUPPORT;
2486        if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2487                goto out;
2488
2489        rc = -ENOBUFS;
2490        sock->ops = &smc_sock_ops;
2491        sk = smc_sock_alloc(net, sock, protocol);
2492        if (!sk)
2493                goto out;
2494
2495        /* create internal TCP socket for CLC handshake and fallback */
2496        smc = smc_sk(sk);
2497        smc->use_fallback = false; /* assume rdma capability first */
2498        smc->fallback_rsn = 0;
2499        rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2500                              &smc->clcsock);
2501        if (rc) {
2502                sk_common_release(sk);
2503                goto out;
2504        }
2505        smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2506        smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2507
2508out:
2509        return rc;
2510}
2511
2512static const struct net_proto_family smc_sock_family_ops = {
2513        .family = PF_SMC,
2514        .owner  = THIS_MODULE,
2515        .create = smc_create,
2516};
2517
2518unsigned int smc_net_id;
2519
2520static __net_init int smc_net_init(struct net *net)
2521{
2522        return smc_pnet_net_init(net);
2523}
2524
2525static void __net_exit smc_net_exit(struct net *net)
2526{
2527        smc_pnet_net_exit(net);
2528}
2529
2530static __net_init int smc_net_stat_init(struct net *net)
2531{
2532        return smc_stats_init(net);
2533}
2534
2535static void __net_exit smc_net_stat_exit(struct net *net)
2536{
2537        smc_stats_exit(net);
2538}
2539
2540static struct pernet_operations smc_net_ops = {
2541        .init = smc_net_init,
2542        .exit = smc_net_exit,
2543        .id   = &smc_net_id,
2544        .size = sizeof(struct smc_net),
2545};
2546
2547static struct pernet_operations smc_net_stat_ops = {
2548        .init = smc_net_stat_init,
2549        .exit = smc_net_stat_exit,
2550};
2551
2552static int __init smc_init(void)
2553{
2554        int rc;
2555
2556        rc = register_pernet_subsys(&smc_net_ops);
2557        if (rc)
2558                return rc;
2559
2560        rc = register_pernet_subsys(&smc_net_stat_ops);
2561        if (rc)
2562                return rc;
2563
2564        smc_ism_init();
2565        smc_clc_init();
2566
2567        rc = smc_nl_init();
2568        if (rc)
2569                goto out_pernet_subsys;
2570
2571        rc = smc_pnet_init();
2572        if (rc)
2573                goto out_nl;
2574
2575        rc = -ENOMEM;
2576        smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2577        if (!smc_hs_wq)
2578                goto out_pnet;
2579
2580        smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2581        if (!smc_close_wq)
2582                goto out_alloc_hs_wq;
2583
2584        rc = smc_core_init();
2585        if (rc) {
2586                pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2587                goto out_alloc_wqs;
2588        }
2589
2590        rc = smc_llc_init();
2591        if (rc) {
2592                pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2593                goto out_core;
2594        }
2595
2596        rc = smc_cdc_init();
2597        if (rc) {
2598                pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2599                goto out_core;
2600        }
2601
2602        rc = proto_register(&smc_proto, 1);
2603        if (rc) {
2604                pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2605                goto out_core;
2606        }
2607
2608        rc = proto_register(&smc_proto6, 1);
2609        if (rc) {
2610                pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2611                goto out_proto;
2612        }
2613
2614        rc = sock_register(&smc_sock_family_ops);
2615        if (rc) {
2616                pr_err("%s: sock_register fails with %d\n", __func__, rc);
2617                goto out_proto6;
2618        }
2619        INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2620        INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2621
2622        rc = smc_ib_register_client();
2623        if (rc) {
2624                pr_err("%s: ib_register fails with %d\n", __func__, rc);
2625                goto out_sock;
2626        }
2627
2628        static_branch_enable(&tcp_have_smc);
2629        return 0;
2630
2631out_sock:
2632        sock_unregister(PF_SMC);
2633out_proto6:
2634        proto_unregister(&smc_proto6);
2635out_proto:
2636        proto_unregister(&smc_proto);
2637out_core:
2638        smc_core_exit();
2639out_alloc_wqs:
2640        destroy_workqueue(smc_close_wq);
2641out_alloc_hs_wq:
2642        destroy_workqueue(smc_hs_wq);
2643out_pnet:
2644        smc_pnet_exit();
2645out_nl:
2646        smc_nl_exit();
2647out_pernet_subsys:
2648        unregister_pernet_subsys(&smc_net_ops);
2649
2650        return rc;
2651}
2652
2653static void __exit smc_exit(void)
2654{
2655        static_branch_disable(&tcp_have_smc);
2656        sock_unregister(PF_SMC);
2657        smc_core_exit();
2658        smc_ib_unregister_client();
2659        destroy_workqueue(smc_close_wq);
2660        destroy_workqueue(smc_hs_wq);
2661        proto_unregister(&smc_proto6);
2662        proto_unregister(&smc_proto);
2663        smc_pnet_exit();
2664        smc_nl_exit();
2665        unregister_pernet_subsys(&smc_net_stat_ops);
2666        unregister_pernet_subsys(&smc_net_ops);
2667        rcu_barrier();
2668}
2669
2670module_init(smc_init);
2671module_exit(smc_exit);
2672
2673MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2674MODULE_DESCRIPTION("smc socket address family");
2675MODULE_LICENSE("GPL");
2676MODULE_ALIAS_NETPROTO(PF_SMC);
2677