linux/drivers/scsi/cxgbi/libcxgbi.c
<<
>>
Prefs
   1/*
   2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
   3 *
   4 * Copyright (c) 2010-2015 Chelsio Communications, Inc.
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation.
   9 *
  10 * Written by: Karen Xie (kxie@chelsio.com)
  11 * Written by: Rakesh Ranjan (rranjan@chelsio.com)
  12 */
  13
  14#define pr_fmt(fmt)     KBUILD_MODNAME ":%s: " fmt, __func__
  15
  16#include <linux/skbuff.h>
  17#include <linux/crypto.h>
  18#include <linux/scatterlist.h>
  19#include <linux/pci.h>
  20#include <scsi/scsi.h>
  21#include <scsi/scsi_cmnd.h>
  22#include <scsi/scsi_host.h>
  23#include <linux/if_vlan.h>
  24#include <linux/inet.h>
  25#include <net/dst.h>
  26#include <net/route.h>
  27#include <net/ipv6.h>
  28#include <net/ip6_route.h>
  29#include <net/addrconf.h>
  30
  31#include <linux/inetdevice.h>   /* ip_dev_find */
  32#include <linux/module.h>
  33#include <net/tcp.h>
  34
  35static unsigned int dbg_level;
  36
  37#include "libcxgbi.h"
  38
  39#define DRV_MODULE_NAME         "libcxgbi"
  40#define DRV_MODULE_DESC         "Chelsio iSCSI driver library"
  41#define DRV_MODULE_VERSION      "0.9.1-ko"
  42#define DRV_MODULE_RELDATE      "Apr. 2015"
  43
  44static char version[] =
  45        DRV_MODULE_DESC " " DRV_MODULE_NAME
  46        " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  47
  48MODULE_AUTHOR("Chelsio Communications, Inc.");
  49MODULE_DESCRIPTION(DRV_MODULE_DESC);
  50MODULE_VERSION(DRV_MODULE_VERSION);
  51MODULE_LICENSE("GPL");
  52
  53module_param(dbg_level, uint, 0644);
  54MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)");
  55
  56
  57/*
  58 * cxgbi device management
  59 * maintains a list of the cxgbi devices
  60 */
  61static LIST_HEAD(cdev_list);
  62static DEFINE_MUTEX(cdev_mutex);
  63
  64static LIST_HEAD(cdev_rcu_list);
  65static DEFINE_SPINLOCK(cdev_rcu_lock);
  66
  67static inline void cxgbi_decode_sw_tag(u32 sw_tag, int *idx, int *age)
  68{
  69        if (age)
  70                *age = sw_tag & 0x7FFF;
  71        if (idx)
  72                *idx = (sw_tag >> 16) & 0x7FFF;
  73}
  74
  75int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
  76                                unsigned int max_conn)
  77{
  78        struct cxgbi_ports_map *pmap = &cdev->pmap;
  79
  80        pmap->port_csk = kvzalloc(array_size(max_conn,
  81                                             sizeof(struct cxgbi_sock *)),
  82                                  GFP_KERNEL | __GFP_NOWARN);
  83        if (!pmap->port_csk) {
  84                pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn);
  85                return -ENOMEM;
  86        }
  87
  88        pmap->max_connect = max_conn;
  89        pmap->sport_base = base;
  90        spin_lock_init(&pmap->lock);
  91        return 0;
  92}
  93EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create);
  94
  95void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev)
  96{
  97        struct cxgbi_ports_map *pmap = &cdev->pmap;
  98        struct cxgbi_sock *csk;
  99        int i;
 100
 101        for (i = 0; i < pmap->max_connect; i++) {
 102                if (pmap->port_csk[i]) {
 103                        csk = pmap->port_csk[i];
 104                        pmap->port_csk[i] = NULL;
 105                        log_debug(1 << CXGBI_DBG_SOCK,
 106                                "csk 0x%p, cdev 0x%p, offload down.\n",
 107                                csk, cdev);
 108                        spin_lock_bh(&csk->lock);
 109                        cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN);
 110                        cxgbi_sock_closed(csk);
 111                        spin_unlock_bh(&csk->lock);
 112                        cxgbi_sock_put(csk);
 113                }
 114        }
 115}
 116EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup);
 117
 118static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
 119{
 120        log_debug(1 << CXGBI_DBG_DEV,
 121                "cdev 0x%p, p# %u.\n", cdev, cdev->nports);
 122        cxgbi_hbas_remove(cdev);
 123        cxgbi_device_portmap_cleanup(cdev);
 124        if (cdev->cdev2ppm)
 125                cxgbi_ppm_release(cdev->cdev2ppm(cdev));
 126        if (cdev->pmap.max_connect)
 127                kvfree(cdev->pmap.port_csk);
 128        kfree(cdev);
 129}
 130
 131struct cxgbi_device *cxgbi_device_register(unsigned int extra,
 132                                           unsigned int nports)
 133{
 134        struct cxgbi_device *cdev;
 135
 136        cdev = kzalloc(sizeof(*cdev) + extra + nports *
 137                        (sizeof(struct cxgbi_hba *) +
 138                         sizeof(struct net_device *)),
 139                        GFP_KERNEL);
 140        if (!cdev) {
 141                pr_warn("nport %d, OOM.\n", nports);
 142                return NULL;
 143        }
 144        cdev->ports = (struct net_device **)(cdev + 1);
 145        cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports *
 146                                                sizeof(struct net_device *));
 147        if (extra)
 148                cdev->dd_data = ((char *)cdev->hbas) +
 149                                nports * sizeof(struct cxgbi_hba *);
 150        spin_lock_init(&cdev->pmap.lock);
 151
 152        mutex_lock(&cdev_mutex);
 153        list_add_tail(&cdev->list_head, &cdev_list);
 154        mutex_unlock(&cdev_mutex);
 155
 156        spin_lock(&cdev_rcu_lock);
 157        list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list);
 158        spin_unlock(&cdev_rcu_lock);
 159
 160        log_debug(1 << CXGBI_DBG_DEV,
 161                "cdev 0x%p, p# %u.\n", cdev, nports);
 162        return cdev;
 163}
 164EXPORT_SYMBOL_GPL(cxgbi_device_register);
 165
 166void cxgbi_device_unregister(struct cxgbi_device *cdev)
 167{
 168        log_debug(1 << CXGBI_DBG_DEV,
 169                "cdev 0x%p, p# %u,%s.\n",
 170                cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : "");
 171
 172        mutex_lock(&cdev_mutex);
 173        list_del(&cdev->list_head);
 174        mutex_unlock(&cdev_mutex);
 175
 176        spin_lock(&cdev_rcu_lock);
 177        list_del_rcu(&cdev->rcu_node);
 178        spin_unlock(&cdev_rcu_lock);
 179        synchronize_rcu();
 180
 181        cxgbi_device_destroy(cdev);
 182}
 183EXPORT_SYMBOL_GPL(cxgbi_device_unregister);
 184
 185void cxgbi_device_unregister_all(unsigned int flag)
 186{
 187        struct cxgbi_device *cdev, *tmp;
 188
 189        mutex_lock(&cdev_mutex);
 190        list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
 191                if ((cdev->flags & flag) == flag) {
 192                        mutex_unlock(&cdev_mutex);
 193                        cxgbi_device_unregister(cdev);
 194                        mutex_lock(&cdev_mutex);
 195                }
 196        }
 197        mutex_unlock(&cdev_mutex);
 198}
 199EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all);
 200
 201struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev)
 202{
 203        struct cxgbi_device *cdev, *tmp;
 204
 205        mutex_lock(&cdev_mutex);
 206        list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
 207                if (cdev->lldev == lldev) {
 208                        mutex_unlock(&cdev_mutex);
 209                        return cdev;
 210                }
 211        }
 212        mutex_unlock(&cdev_mutex);
 213
 214        log_debug(1 << CXGBI_DBG_DEV,
 215                "lldev 0x%p, NO match found.\n", lldev);
 216        return NULL;
 217}
 218EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev);
 219
 220struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev,
 221                                                 int *port)
 222{
 223        struct net_device *vdev = NULL;
 224        struct cxgbi_device *cdev, *tmp;
 225        int i;
 226
 227        if (is_vlan_dev(ndev)) {
 228                vdev = ndev;
 229                ndev = vlan_dev_real_dev(ndev);
 230                log_debug(1 << CXGBI_DBG_DEV,
 231                        "vlan dev %s -> %s.\n", vdev->name, ndev->name);
 232        }
 233
 234        mutex_lock(&cdev_mutex);
 235        list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
 236                for (i = 0; i < cdev->nports; i++) {
 237                        if (ndev == cdev->ports[i]) {
 238                                cdev->hbas[i]->vdev = vdev;
 239                                mutex_unlock(&cdev_mutex);
 240                                if (port)
 241                                        *port = i;
 242                                return cdev;
 243                        }
 244                }
 245        }
 246        mutex_unlock(&cdev_mutex);
 247        log_debug(1 << CXGBI_DBG_DEV,
 248                "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
 249        return NULL;
 250}
 251EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev);
 252
 253struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
 254                                                     int *port)
 255{
 256        struct net_device *vdev = NULL;
 257        struct cxgbi_device *cdev;
 258        int i;
 259
 260        if (is_vlan_dev(ndev)) {
 261                vdev = ndev;
 262                ndev = vlan_dev_real_dev(ndev);
 263                pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
 264        }
 265
 266        rcu_read_lock();
 267        list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) {
 268                for (i = 0; i < cdev->nports; i++) {
 269                        if (ndev == cdev->ports[i]) {
 270                                cdev->hbas[i]->vdev = vdev;
 271                                rcu_read_unlock();
 272                                if (port)
 273                                        *port = i;
 274                                return cdev;
 275                        }
 276                }
 277        }
 278        rcu_read_unlock();
 279
 280        log_debug(1 << CXGBI_DBG_DEV,
 281                  "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
 282        return NULL;
 283}
 284EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
 285
 286static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
 287                                                     int *port)
 288{
 289        struct net_device *vdev = NULL;
 290        struct cxgbi_device *cdev, *tmp;
 291        int i;
 292
 293        if (is_vlan_dev(ndev)) {
 294                vdev = ndev;
 295                ndev = vlan_dev_real_dev(ndev);
 296                pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
 297        }
 298
 299        mutex_lock(&cdev_mutex);
 300        list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
 301                for (i = 0; i < cdev->nports; i++) {
 302                        if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr,
 303                                    MAX_ADDR_LEN)) {
 304                                cdev->hbas[i]->vdev = vdev;
 305                                mutex_unlock(&cdev_mutex);
 306                                if (port)
 307                                        *port = i;
 308                                return cdev;
 309                        }
 310                }
 311        }
 312        mutex_unlock(&cdev_mutex);
 313        log_debug(1 << CXGBI_DBG_DEV,
 314                  "ndev 0x%p, %s, NO match mac found.\n",
 315                  ndev, ndev->name);
 316        return NULL;
 317}
 318
 319void cxgbi_hbas_remove(struct cxgbi_device *cdev)
 320{
 321        int i;
 322        struct cxgbi_hba *chba;
 323
 324        log_debug(1 << CXGBI_DBG_DEV,
 325                "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
 326
 327        for (i = 0; i < cdev->nports; i++) {
 328                chba = cdev->hbas[i];
 329                if (chba) {
 330                        cdev->hbas[i] = NULL;
 331                        iscsi_host_remove(chba->shost);
 332                        pci_dev_put(cdev->pdev);
 333                        iscsi_host_free(chba->shost);
 334                }
 335        }
 336}
 337EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
 338
 339int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
 340                unsigned int max_conns, struct scsi_host_template *sht,
 341                struct scsi_transport_template *stt)
 342{
 343        struct cxgbi_hba *chba;
 344        struct Scsi_Host *shost;
 345        int i, err;
 346
 347        log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
 348
 349        for (i = 0; i < cdev->nports; i++) {
 350                shost = iscsi_host_alloc(sht, sizeof(*chba), 1);
 351                if (!shost) {
 352                        pr_info("0x%p, p%d, %s, host alloc failed.\n",
 353                                cdev, i, cdev->ports[i]->name);
 354                        err = -ENOMEM;
 355                        goto err_out;
 356                }
 357
 358                shost->transportt = stt;
 359                shost->max_lun = max_lun;
 360                shost->max_id = max_conns - 1;
 361                shost->max_channel = 0;
 362                shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
 363
 364                chba = iscsi_host_priv(shost);
 365                chba->cdev = cdev;
 366                chba->ndev = cdev->ports[i];
 367                chba->shost = shost;
 368
 369                shost->can_queue = sht->can_queue - ISCSI_MGMT_CMDS_MAX;
 370
 371                log_debug(1 << CXGBI_DBG_DEV,
 372                        "cdev 0x%p, p#%d %s: chba 0x%p.\n",
 373                        cdev, i, cdev->ports[i]->name, chba);
 374
 375                pci_dev_get(cdev->pdev);
 376                err = iscsi_host_add(shost, &cdev->pdev->dev);
 377                if (err) {
 378                        pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
 379                                cdev, i, cdev->ports[i]->name);
 380                        pci_dev_put(cdev->pdev);
 381                        scsi_host_put(shost);
 382                        goto  err_out;
 383                }
 384
 385                cdev->hbas[i] = chba;
 386        }
 387
 388        return 0;
 389
 390err_out:
 391        cxgbi_hbas_remove(cdev);
 392        return err;
 393}
 394EXPORT_SYMBOL_GPL(cxgbi_hbas_add);
 395
 396/*
 397 * iSCSI offload
 398 *
 399 * - source port management
 400 *   To find a free source port in the port allocation map we use a very simple
 401 *   rotor scheme to look for the next free port.
 402 *
 403 *   If a source port has been specified make sure that it doesn't collide with
 404 *   our normal source port allocation map.  If it's outside the range of our
 405 *   allocation/deallocation scheme just let them use it.
 406 *
 407 *   If the source port is outside our allocation range, the caller is
 408 *   responsible for keeping track of their port usage.
 409 */
 410
 411static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev,
 412                                            unsigned char port_id)
 413{
 414        struct cxgbi_ports_map *pmap = &cdev->pmap;
 415        unsigned int i;
 416        unsigned int used;
 417
 418        if (!pmap->max_connect || !pmap->used)
 419                return NULL;
 420
 421        spin_lock_bh(&pmap->lock);
 422        used = pmap->used;
 423        for (i = 0; used && i < pmap->max_connect; i++) {
 424                struct cxgbi_sock *csk = pmap->port_csk[i];
 425
 426                if (csk) {
 427                        if (csk->port_id == port_id) {
 428                                spin_unlock_bh(&pmap->lock);
 429                                return csk;
 430                        }
 431                        used--;
 432                }
 433        }
 434        spin_unlock_bh(&pmap->lock);
 435
 436        return NULL;
 437}
 438
 439static int sock_get_port(struct cxgbi_sock *csk)
 440{
 441        struct cxgbi_device *cdev = csk->cdev;
 442        struct cxgbi_ports_map *pmap = &cdev->pmap;
 443        unsigned int start;
 444        int idx;
 445        __be16 *port;
 446
 447        if (!pmap->max_connect) {
 448                pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
 449                           cdev, csk->port_id, cdev->ports[csk->port_id]->name);
 450                return -EADDRNOTAVAIL;
 451        }
 452
 453        if (csk->csk_family == AF_INET)
 454                port = &csk->saddr.sin_port;
 455        else /* ipv6 */
 456                port = &csk->saddr6.sin6_port;
 457
 458        if (*port) {
 459                pr_err("source port NON-ZERO %u.\n",
 460                        ntohs(*port));
 461                return -EADDRINUSE;
 462        }
 463
 464        spin_lock_bh(&pmap->lock);
 465        if (pmap->used >= pmap->max_connect) {
 466                spin_unlock_bh(&pmap->lock);
 467                pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
 468                        cdev, csk->port_id, cdev->ports[csk->port_id]->name);
 469                return -EADDRNOTAVAIL;
 470        }
 471
 472        start = idx = pmap->next;
 473        do {
 474                if (++idx >= pmap->max_connect)
 475                        idx = 0;
 476                if (!pmap->port_csk[idx]) {
 477                        pmap->used++;
 478                        *port = htons(pmap->sport_base + idx);
 479                        pmap->next = idx;
 480                        pmap->port_csk[idx] = csk;
 481                        spin_unlock_bh(&pmap->lock);
 482                        cxgbi_sock_get(csk);
 483                        log_debug(1 << CXGBI_DBG_SOCK,
 484                                "cdev 0x%p, p#%u %s, p %u, %u.\n",
 485                                cdev, csk->port_id,
 486                                cdev->ports[csk->port_id]->name,
 487                                pmap->sport_base + idx, pmap->next);
 488                        return 0;
 489                }
 490        } while (idx != start);
 491        spin_unlock_bh(&pmap->lock);
 492
 493        /* should not happen */
 494        pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
 495                cdev, csk->port_id, cdev->ports[csk->port_id]->name,
 496                pmap->next);
 497        return -EADDRNOTAVAIL;
 498}
 499
 500static void sock_put_port(struct cxgbi_sock *csk)
 501{
 502        struct cxgbi_device *cdev = csk->cdev;
 503        struct cxgbi_ports_map *pmap = &cdev->pmap;
 504        __be16 *port;
 505
 506        if (csk->csk_family == AF_INET)
 507                port = &csk->saddr.sin_port;
 508        else /* ipv6 */
 509                port = &csk->saddr6.sin6_port;
 510
 511        if (*port) {
 512                int idx = ntohs(*port) - pmap->sport_base;
 513
 514                *port = 0;
 515                if (idx < 0 || idx >= pmap->max_connect) {
 516                        pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
 517                                cdev, csk->port_id,
 518                                cdev->ports[csk->port_id]->name,
 519                                ntohs(*port));
 520                        return;
 521                }
 522
 523                spin_lock_bh(&pmap->lock);
 524                pmap->port_csk[idx] = NULL;
 525                pmap->used--;
 526                spin_unlock_bh(&pmap->lock);
 527
 528                log_debug(1 << CXGBI_DBG_SOCK,
 529                        "cdev 0x%p, p#%u %s, release %u.\n",
 530                        cdev, csk->port_id, cdev->ports[csk->port_id]->name,
 531                        pmap->sport_base + idx);
 532
 533                cxgbi_sock_put(csk);
 534        }
 535}
 536
 537/*
 538 * iscsi tcp connection
 539 */
 540void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
 541{
 542        if (csk->cpl_close) {
 543                kfree_skb(csk->cpl_close);
 544                csk->cpl_close = NULL;
 545        }
 546        if (csk->cpl_abort_req) {
 547                kfree_skb(csk->cpl_abort_req);
 548                csk->cpl_abort_req = NULL;
 549        }
 550        if (csk->cpl_abort_rpl) {
 551                kfree_skb(csk->cpl_abort_rpl);
 552                csk->cpl_abort_rpl = NULL;
 553        }
 554}
 555EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
 556
 557static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
 558{
 559        struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
 560
 561        if (!csk) {
 562                pr_info("alloc csk %zu failed.\n", sizeof(*csk));
 563                return NULL;
 564        }
 565
 566        if (cdev->csk_alloc_cpls(csk) < 0) {
 567                pr_info("csk 0x%p, alloc cpls failed.\n", csk);
 568                kfree(csk);
 569                return NULL;
 570        }
 571
 572        spin_lock_init(&csk->lock);
 573        kref_init(&csk->refcnt);
 574        skb_queue_head_init(&csk->receive_queue);
 575        skb_queue_head_init(&csk->write_queue);
 576        timer_setup(&csk->retry_timer, NULL, 0);
 577        init_completion(&csk->cmpl);
 578        rwlock_init(&csk->callback_lock);
 579        csk->cdev = cdev;
 580        csk->flags = 0;
 581        cxgbi_sock_set_state(csk, CTP_CLOSED);
 582
 583        log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk);
 584
 585        return csk;
 586}
 587
 588static struct rtable *find_route_ipv4(struct flowi4 *fl4,
 589                                      __be32 saddr, __be32 daddr,
 590                                      __be16 sport, __be16 dport, u8 tos,
 591                                      int ifindex)
 592{
 593        struct rtable *rt;
 594
 595        rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
 596                                   dport, sport, IPPROTO_TCP, tos, ifindex);
 597        if (IS_ERR(rt))
 598                return NULL;
 599
 600        return rt;
 601}
 602
 603static struct cxgbi_sock *
 604cxgbi_check_route(struct sockaddr *dst_addr, int ifindex)
 605{
 606        struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
 607        struct dst_entry *dst;
 608        struct net_device *ndev;
 609        struct cxgbi_device *cdev;
 610        struct rtable *rt = NULL;
 611        struct neighbour *n;
 612        struct flowi4 fl4;
 613        struct cxgbi_sock *csk = NULL;
 614        unsigned int mtu = 0;
 615        int port = 0xFFFF;
 616        int err = 0;
 617
 618        rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0,
 619                             daddr->sin_port, 0, ifindex);
 620        if (!rt) {
 621                pr_info("no route to ipv4 0x%x, port %u.\n",
 622                        be32_to_cpu(daddr->sin_addr.s_addr),
 623                        be16_to_cpu(daddr->sin_port));
 624                err = -ENETUNREACH;
 625                goto err_out;
 626        }
 627        dst = &rt->dst;
 628        n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr);
 629        if (!n) {
 630                err = -ENODEV;
 631                goto rel_rt;
 632        }
 633        ndev = n->dev;
 634
 635        if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
 636                pr_info("multi-cast route %pI4, port %u, dev %s.\n",
 637                        &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
 638                        ndev->name);
 639                err = -ENETUNREACH;
 640                goto rel_neigh;
 641        }
 642
 643        if (ndev->flags & IFF_LOOPBACK) {
 644                ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr);
 645                if (!ndev) {
 646                        err = -ENETUNREACH;
 647                        goto rel_neigh;
 648                }
 649                mtu = ndev->mtu;
 650                pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
 651                        n->dev->name, ndev->name, mtu);
 652        }
 653
 654        if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
 655                pr_info("%s interface not up.\n", ndev->name);
 656                err = -ENETDOWN;
 657                goto rel_neigh;
 658        }
 659
 660        cdev = cxgbi_device_find_by_netdev(ndev, &port);
 661        if (!cdev)
 662                cdev = cxgbi_device_find_by_mac(ndev, &port);
 663        if (!cdev) {
 664                pr_info("dst %pI4, %s, NOT cxgbi device.\n",
 665                        &daddr->sin_addr.s_addr, ndev->name);
 666                err = -ENETUNREACH;
 667                goto rel_neigh;
 668        }
 669        log_debug(1 << CXGBI_DBG_SOCK,
 670                "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
 671                &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
 672                           port, ndev->name, cdev);
 673
 674        csk = cxgbi_sock_create(cdev);
 675        if (!csk) {
 676                err = -ENOMEM;
 677                goto rel_neigh;
 678        }
 679        csk->cdev = cdev;
 680        csk->port_id = port;
 681        csk->mtu = mtu;
 682        csk->dst = dst;
 683
 684        csk->csk_family = AF_INET;
 685        csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr;
 686        csk->daddr.sin_port = daddr->sin_port;
 687        csk->daddr.sin_family = daddr->sin_family;
 688        csk->saddr.sin_family = daddr->sin_family;
 689        csk->saddr.sin_addr.s_addr = fl4.saddr;
 690        neigh_release(n);
 691
 692        return csk;
 693
 694rel_neigh:
 695        neigh_release(n);
 696
 697rel_rt:
 698        ip_rt_put(rt);
 699err_out:
 700        return ERR_PTR(err);
 701}
 702
 703#if IS_ENABLED(CONFIG_IPV6)
 704static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
 705                                        const struct in6_addr *daddr,
 706                                        int ifindex)
 707{
 708        struct flowi6 fl;
 709
 710        memset(&fl, 0, sizeof(fl));
 711        fl.flowi6_oif = ifindex;
 712        if (saddr)
 713                memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
 714        if (daddr)
 715                memcpy(&fl.daddr, daddr, sizeof(struct in6_addr));
 716        return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
 717}
 718
 719static struct cxgbi_sock *
 720cxgbi_check_route6(struct sockaddr *dst_addr, int ifindex)
 721{
 722        struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
 723        struct dst_entry *dst;
 724        struct net_device *ndev;
 725        struct cxgbi_device *cdev;
 726        struct rt6_info *rt = NULL;
 727        struct neighbour *n;
 728        struct in6_addr pref_saddr;
 729        struct cxgbi_sock *csk = NULL;
 730        unsigned int mtu = 0;
 731        int port = 0xFFFF;
 732        int err = 0;
 733
 734        rt = find_route_ipv6(NULL, &daddr6->sin6_addr, ifindex);
 735
 736        if (!rt) {
 737                pr_info("no route to ipv6 %pI6 port %u\n",
 738                        daddr6->sin6_addr.s6_addr,
 739                        be16_to_cpu(daddr6->sin6_port));
 740                err = -ENETUNREACH;
 741                goto err_out;
 742        }
 743
 744        dst = &rt->dst;
 745
 746        n = dst_neigh_lookup(dst, &daddr6->sin6_addr);
 747
 748        if (!n) {
 749                pr_info("%pI6, port %u, dst no neighbour.\n",
 750                        daddr6->sin6_addr.s6_addr,
 751                        be16_to_cpu(daddr6->sin6_port));
 752                err = -ENETUNREACH;
 753                goto rel_rt;
 754        }
 755        ndev = n->dev;
 756
 757        if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
 758                pr_info("%s interface not up.\n", ndev->name);
 759                err = -ENETDOWN;
 760                goto rel_rt;
 761        }
 762
 763        if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) {
 764                pr_info("multi-cast route %pI6 port %u, dev %s.\n",
 765                        daddr6->sin6_addr.s6_addr,
 766                        ntohs(daddr6->sin6_port), ndev->name);
 767                err = -ENETUNREACH;
 768                goto rel_rt;
 769        }
 770
 771        cdev = cxgbi_device_find_by_netdev(ndev, &port);
 772        if (!cdev)
 773                cdev = cxgbi_device_find_by_mac(ndev, &port);
 774        if (!cdev) {
 775                pr_info("dst %pI6 %s, NOT cxgbi device.\n",
 776                        daddr6->sin6_addr.s6_addr, ndev->name);
 777                err = -ENETUNREACH;
 778                goto rel_rt;
 779        }
 780        log_debug(1 << CXGBI_DBG_SOCK,
 781                  "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
 782                  daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port,
 783                  ndev->name, cdev);
 784
 785        csk = cxgbi_sock_create(cdev);
 786        if (!csk) {
 787                err = -ENOMEM;
 788                goto rel_rt;
 789        }
 790        csk->cdev = cdev;
 791        csk->port_id = port;
 792        csk->mtu = mtu;
 793        csk->dst = dst;
 794
 795        rt6_get_prefsrc(rt, &pref_saddr);
 796        if (ipv6_addr_any(&pref_saddr)) {
 797                struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt);
 798
 799                err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL,
 800                                         &daddr6->sin6_addr, 0, &pref_saddr);
 801                if (err) {
 802                        pr_info("failed to get source address to reach %pI6\n",
 803                                &daddr6->sin6_addr);
 804                        goto rel_rt;
 805                }
 806        }
 807
 808        csk->csk_family = AF_INET6;
 809        csk->daddr6.sin6_addr = daddr6->sin6_addr;
 810        csk->daddr6.sin6_port = daddr6->sin6_port;
 811        csk->daddr6.sin6_family = daddr6->sin6_family;
 812        csk->saddr6.sin6_family = daddr6->sin6_family;
 813        csk->saddr6.sin6_addr = pref_saddr;
 814
 815        neigh_release(n);
 816        return csk;
 817
 818rel_rt:
 819        if (n)
 820                neigh_release(n);
 821
 822        ip6_rt_put(rt);
 823        if (csk)
 824                cxgbi_sock_closed(csk);
 825err_out:
 826        return ERR_PTR(err);
 827}
 828#endif /* IS_ENABLED(CONFIG_IPV6) */
 829
 830void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn,
 831                        unsigned int opt)
 832{
 833        csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn;
 834        dst_confirm(csk->dst);
 835        smp_mb();
 836        cxgbi_sock_set_state(csk, CTP_ESTABLISHED);
 837}
 838EXPORT_SYMBOL_GPL(cxgbi_sock_established);
 839
 840static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk)
 841{
 842        log_debug(1 << CXGBI_DBG_SOCK,
 843                "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
 844                csk, csk->state, csk->flags, csk->user_data);
 845
 846        if (csk->state != CTP_ESTABLISHED) {
 847                read_lock_bh(&csk->callback_lock);
 848                if (csk->user_data)
 849                        iscsi_conn_failure(csk->user_data,
 850                                        ISCSI_ERR_TCP_CONN_CLOSE);
 851                read_unlock_bh(&csk->callback_lock);
 852        }
 853}
 854
 855void cxgbi_sock_closed(struct cxgbi_sock *csk)
 856{
 857        log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
 858                csk, (csk)->state, (csk)->flags, (csk)->tid);
 859        cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
 860        if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED)
 861                return;
 862        if (csk->saddr.sin_port)
 863                sock_put_port(csk);
 864        if (csk->dst)
 865                dst_release(csk->dst);
 866        csk->cdev->csk_release_offload_resources(csk);
 867        cxgbi_sock_set_state(csk, CTP_CLOSED);
 868        cxgbi_inform_iscsi_conn_closing(csk);
 869        cxgbi_sock_put(csk);
 870}
 871EXPORT_SYMBOL_GPL(cxgbi_sock_closed);
 872
 873static void need_active_close(struct cxgbi_sock *csk)
 874{
 875        int data_lost;
 876        int close_req = 0;
 877
 878        log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
 879                csk, (csk)->state, (csk)->flags, (csk)->tid);
 880        spin_lock_bh(&csk->lock);
 881        if (csk->dst)
 882                dst_confirm(csk->dst);
 883        data_lost = skb_queue_len(&csk->receive_queue);
 884        __skb_queue_purge(&csk->receive_queue);
 885
 886        if (csk->state == CTP_ACTIVE_OPEN)
 887                cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
 888        else if (csk->state == CTP_ESTABLISHED) {
 889                close_req = 1;
 890                cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE);
 891        } else if (csk->state == CTP_PASSIVE_CLOSE) {
 892                close_req = 1;
 893                cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
 894        }
 895
 896        if (close_req) {
 897                if (!cxgbi_sock_flag(csk, CTPF_LOGOUT_RSP_RCVD) ||
 898                    data_lost)
 899                        csk->cdev->csk_send_abort_req(csk);
 900                else
 901                        csk->cdev->csk_send_close_req(csk);
 902        }
 903
 904        spin_unlock_bh(&csk->lock);
 905}
 906
 907void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno)
 908{
 909        pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
 910                        csk, csk->state, csk->flags,
 911                        &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port,
 912                        &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port,
 913                        errno);
 914
 915        cxgbi_sock_set_state(csk, CTP_CONNECTING);
 916        csk->err = errno;
 917        cxgbi_sock_closed(csk);
 918}
 919EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open);
 920
 921void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb)
 922{
 923        struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk;
 924        struct module *owner = csk->cdev->owner;
 925
 926        log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
 927                csk, (csk)->state, (csk)->flags, (csk)->tid);
 928        cxgbi_sock_get(csk);
 929        spin_lock_bh(&csk->lock);
 930        if (csk->state == CTP_ACTIVE_OPEN)
 931                cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH);
 932        spin_unlock_bh(&csk->lock);
 933        cxgbi_sock_put(csk);
 934        __kfree_skb(skb);
 935
 936        module_put(owner);
 937}
 938EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure);
 939
 940void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk)
 941{
 942        cxgbi_sock_get(csk);
 943        spin_lock_bh(&csk->lock);
 944
 945        cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD);
 946        if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
 947                cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING);
 948                if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD))
 949                        pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
 950                               csk, csk->state, csk->flags, csk->tid);
 951                cxgbi_sock_closed(csk);
 952        }
 953
 954        spin_unlock_bh(&csk->lock);
 955        cxgbi_sock_put(csk);
 956}
 957EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl);
 958
 959void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk)
 960{
 961        log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
 962                csk, (csk)->state, (csk)->flags, (csk)->tid);
 963        cxgbi_sock_get(csk);
 964        spin_lock_bh(&csk->lock);
 965
 966        if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
 967                goto done;
 968
 969        switch (csk->state) {
 970        case CTP_ESTABLISHED:
 971                cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE);
 972                break;
 973        case CTP_ACTIVE_CLOSE:
 974                cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
 975                break;
 976        case CTP_CLOSE_WAIT_1:
 977                cxgbi_sock_closed(csk);
 978                break;
 979        case CTP_ABORTING:
 980                break;
 981        default:
 982                pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
 983                        csk, csk->state, csk->flags, csk->tid);
 984        }
 985        cxgbi_inform_iscsi_conn_closing(csk);
 986done:
 987        spin_unlock_bh(&csk->lock);
 988        cxgbi_sock_put(csk);
 989}
 990EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close);
 991
 992void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt)
 993{
 994        log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
 995                csk, (csk)->state, (csk)->flags, (csk)->tid);
 996        cxgbi_sock_get(csk);
 997        spin_lock_bh(&csk->lock);
 998
 999        csk->snd_una = snd_nxt - 1;
1000        if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
1001                goto done;
1002
1003        switch (csk->state) {
1004        case CTP_ACTIVE_CLOSE:
1005                cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1);
1006                break;
1007        case CTP_CLOSE_WAIT_1:
1008        case CTP_CLOSE_WAIT_2:
1009                cxgbi_sock_closed(csk);
1010                break;
1011        case CTP_ABORTING:
1012                break;
1013        default:
1014                pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
1015                        csk, csk->state, csk->flags, csk->tid);
1016        }
1017done:
1018        spin_unlock_bh(&csk->lock);
1019        cxgbi_sock_put(csk);
1020}
1021EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl);
1022
1023void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits,
1024                           unsigned int snd_una, int seq_chk)
1025{
1026        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1027                        "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
1028                        csk, csk->state, csk->flags, csk->tid, credits,
1029                        csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk);
1030
1031        spin_lock_bh(&csk->lock);
1032
1033        csk->wr_cred += credits;
1034        if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred)
1035                csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
1036
1037        while (credits) {
1038                struct sk_buff *p = cxgbi_sock_peek_wr(csk);
1039
1040                if (unlikely(!p)) {
1041                        pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
1042                                csk, csk->state, csk->flags, csk->tid, credits,
1043                                csk->wr_cred, csk->wr_una_cred);
1044                        break;
1045                }
1046
1047                if (unlikely(credits < p->csum)) {
1048                        pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
1049                                csk, csk->state, csk->flags, csk->tid,
1050                                credits, csk->wr_cred, csk->wr_una_cred,
1051                                p->csum);
1052                        p->csum -= credits;
1053                        break;
1054                } else {
1055                        cxgbi_sock_dequeue_wr(csk);
1056                        credits -= p->csum;
1057                        kfree_skb(p);
1058                }
1059        }
1060
1061        cxgbi_sock_check_wr_invariants(csk);
1062
1063        if (seq_chk) {
1064                if (unlikely(before(snd_una, csk->snd_una))) {
1065                        pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
1066                                csk, csk->state, csk->flags, csk->tid, snd_una,
1067                                csk->snd_una);
1068                        goto done;
1069                }
1070
1071                if (csk->snd_una != snd_una) {
1072                        csk->snd_una = snd_una;
1073                        dst_confirm(csk->dst);
1074                }
1075        }
1076
1077        if (skb_queue_len(&csk->write_queue)) {
1078                if (csk->cdev->csk_push_tx_frames(csk, 0))
1079                        cxgbi_conn_tx_open(csk);
1080        } else
1081                cxgbi_conn_tx_open(csk);
1082done:
1083        spin_unlock_bh(&csk->lock);
1084}
1085EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack);
1086
1087static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk,
1088                                             unsigned short mtu)
1089{
1090        int i = 0;
1091
1092        while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu)
1093                ++i;
1094
1095        return i;
1096}
1097
1098unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu)
1099{
1100        unsigned int idx;
1101        struct dst_entry *dst = csk->dst;
1102
1103        csk->advmss = dst_metric_advmss(dst);
1104
1105        if (csk->advmss > pmtu - 40)
1106                csk->advmss = pmtu - 40;
1107        if (csk->advmss < csk->cdev->mtus[0] - 40)
1108                csk->advmss = csk->cdev->mtus[0] - 40;
1109        idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40);
1110
1111        return idx;
1112}
1113EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss);
1114
1115void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb)
1116{
1117        cxgbi_skcb_tcp_seq(skb) = csk->write_seq;
1118        __skb_queue_tail(&csk->write_queue, skb);
1119}
1120EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail);
1121
1122void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk)
1123{
1124        struct sk_buff *skb;
1125
1126        while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL)
1127                kfree_skb(skb);
1128}
1129EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue);
1130
1131void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk)
1132{
1133        int pending = cxgbi_sock_count_pending_wrs(csk);
1134
1135        if (unlikely(csk->wr_cred + pending != csk->wr_max_cred))
1136                pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1137                        csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred);
1138}
1139EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
1140
1141static inline void
1142scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl,
1143                unsigned int *sgcnt, unsigned int *dlen,
1144                unsigned int prot)
1145{
1146        struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : &sc->sdb;
1147
1148        *sgl = sdb->table.sgl;
1149        *sgcnt = sdb->table.nents;
1150        *dlen = sdb->length;
1151        /* Caution: for protection sdb, sdb->length is invalid */
1152}
1153
1154void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *ppod,
1155                            struct cxgbi_task_tag_info *ttinfo,
1156                            struct scatterlist **sg_pp, unsigned int *sg_off)
1157{
1158        struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
1159        unsigned int offset = sg_off ? *sg_off : 0;
1160        dma_addr_t addr = 0UL;
1161        unsigned int len = 0;
1162        int i;
1163
1164        memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
1165
1166        if (sg) {
1167                addr = sg_dma_address(sg);
1168                len = sg_dma_len(sg);
1169        }
1170
1171        for (i = 0; i < PPOD_PAGES_MAX; i++) {
1172                if (sg) {
1173                        ppod->addr[i] = cpu_to_be64(addr + offset);
1174                        offset += PAGE_SIZE;
1175                        if (offset == (len + sg->offset)) {
1176                                offset = 0;
1177                                sg = sg_next(sg);
1178                                if (sg) {
1179                                        addr = sg_dma_address(sg);
1180                                        len = sg_dma_len(sg);
1181                                }
1182                        }
1183                } else {
1184                        ppod->addr[i] = 0ULL;
1185                }
1186        }
1187
1188        /*
1189         * the fifth address needs to be repeated in the next ppod, so do
1190         * not move sg
1191         */
1192        if (sg_pp) {
1193                *sg_pp = sg;
1194                *sg_off = offset;
1195        }
1196
1197        if (offset == len) {
1198                offset = 0;
1199                sg = sg_next(sg);
1200                if (sg) {
1201                        addr = sg_dma_address(sg);
1202                        len = sg_dma_len(sg);
1203                }
1204        }
1205        ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
1206}
1207EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod);
1208
1209/*
1210 * APIs interacting with open-iscsi libraries
1211 */
1212
1213int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev,
1214                        struct cxgbi_tag_format *tformat,
1215                        unsigned int iscsi_size, unsigned int llimit,
1216                        unsigned int start, unsigned int rsvd_factor,
1217                        unsigned int edram_start, unsigned int edram_size)
1218{
1219        int err = cxgbi_ppm_init(ppm_pp, cdev->ports[0], cdev->pdev,
1220                                cdev->lldev, tformat, iscsi_size, llimit, start,
1221                                rsvd_factor, edram_start, edram_size);
1222
1223        if (err >= 0) {
1224                struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp);
1225
1226                if (ppm->ppmax < 1024 ||
1227                    ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX)
1228                        cdev->flags |= CXGBI_FLAG_DDP_OFF;
1229                err = 0;
1230        } else {
1231                cdev->flags |= CXGBI_FLAG_DDP_OFF;
1232        }
1233
1234        return err;
1235}
1236EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup);
1237
1238static int cxgbi_ddp_sgl_check(struct scatterlist *sgl, int nents)
1239{
1240        int i;
1241        int last_sgidx = nents - 1;
1242        struct scatterlist *sg = sgl;
1243
1244        for (i = 0; i < nents; i++, sg = sg_next(sg)) {
1245                unsigned int len = sg->length + sg->offset;
1246
1247                if ((sg->offset & 0x3) || (i && sg->offset) ||
1248                    ((i != last_sgidx) && len != PAGE_SIZE)) {
1249                        log_debug(1 << CXGBI_DBG_DDP,
1250                                  "sg %u/%u, %u,%u, not aligned.\n",
1251                                  i, nents, sg->offset, sg->length);
1252                        goto err_out;
1253                }
1254        }
1255        return 0;
1256err_out:
1257        return -EINVAL;
1258}
1259
1260static int cxgbi_ddp_reserve(struct cxgbi_conn *cconn,
1261                             struct cxgbi_task_data *tdata, u32 sw_tag,
1262                             unsigned int xferlen)
1263{
1264        struct cxgbi_sock *csk = cconn->cep->csk;
1265        struct cxgbi_device *cdev = csk->cdev;
1266        struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
1267        struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
1268        struct scatterlist *sgl = ttinfo->sgl;
1269        unsigned int sgcnt = ttinfo->nents;
1270        unsigned int sg_offset = sgl->offset;
1271        int err;
1272
1273        if (cdev->flags & CXGBI_FLAG_DDP_OFF) {
1274                log_debug(1 << CXGBI_DBG_DDP,
1275                          "cdev 0x%p DDP off.\n", cdev);
1276                return -EINVAL;
1277        }
1278
1279        if (!ppm || xferlen < DDP_THRESHOLD || !sgcnt ||
1280            ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) {
1281                log_debug(1 << CXGBI_DBG_DDP,
1282                          "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
1283                          ppm, ppm ? ppm->tformat.pgsz_idx_dflt : DDP_PGIDX_MAX,
1284                          xferlen, ttinfo->nents);
1285                return -EINVAL;
1286        }
1287
1288        /* make sure the buffer is suitable for ddp */
1289        if (cxgbi_ddp_sgl_check(sgl, sgcnt) < 0)
1290                return -EINVAL;
1291
1292        ttinfo->nr_pages = (xferlen + sgl->offset + (1 << PAGE_SHIFT) - 1) >>
1293                            PAGE_SHIFT;
1294
1295        /*
1296         * the ddp tag will be used for the itt in the outgoing pdu,
1297         * the itt genrated by libiscsi is saved in the ppm and can be
1298         * retrieved via the ddp tag
1299         */
1300        err = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
1301                                      &ttinfo->tag, (unsigned long)sw_tag);
1302        if (err < 0) {
1303                cconn->ddp_full++;
1304                return err;
1305        }
1306        ttinfo->npods = err;
1307
1308         /* setup dma from scsi command sgl */
1309        sgl->offset = 0;
1310        err = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
1311        sgl->offset = sg_offset;
1312        if (err == 0) {
1313                pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
1314                        __func__, sw_tag, xferlen, sgcnt);
1315                goto rel_ppods;
1316        }
1317        if (err != ttinfo->nr_pages) {
1318                log_debug(1 << CXGBI_DBG_DDP,
1319                          "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n",
1320                          __func__, sw_tag, xferlen, sgcnt, err);
1321        }
1322
1323        ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_MAPPED;
1324        ttinfo->cid = csk->port_id;
1325
1326        cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
1327                                xferlen, &ttinfo->hdr);
1328
1329        if (cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ) {
1330                /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */
1331                ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_VALID;
1332        } else {
1333                /* write ppod from control queue now */
1334                err = cdev->csk_ddp_set_map(ppm, csk, ttinfo);
1335                if (err < 0)
1336                        goto rel_ppods;
1337        }
1338
1339        return 0;
1340
1341rel_ppods:
1342        cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
1343
1344        if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_MAPPED) {
1345                ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_MAPPED;
1346                dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
1347        }
1348        return -EINVAL;
1349}
1350
1351static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
1352{
1353        struct scsi_cmnd *sc = task->sc;
1354        struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
1355        struct cxgbi_conn *cconn = tcp_conn->dd_data;
1356        struct cxgbi_device *cdev = cconn->chba->cdev;
1357        struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
1358        u32 tag = ntohl((__force u32)hdr_itt);
1359
1360        log_debug(1 << CXGBI_DBG_DDP,
1361                  "cdev 0x%p, task 0x%p, release tag 0x%x.\n",
1362                  cdev, task, tag);
1363        if (sc && sc->sc_data_direction == DMA_FROM_DEVICE &&
1364            cxgbi_ppm_is_ddp_tag(ppm, tag)) {
1365                struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
1366                struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
1367
1368                if (!(cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ))
1369                        cdev->csk_ddp_clear_map(cdev, ppm, ttinfo);
1370                cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
1371                dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
1372                             DMA_FROM_DEVICE);
1373        }
1374}
1375
1376static inline u32 cxgbi_build_sw_tag(u32 idx, u32 age)
1377{
1378        /* assume idx and age both are < 0x7FFF (32767) */
1379        return (idx << 16) | age;
1380}
1381
1382static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
1383{
1384        struct scsi_cmnd *sc = task->sc;
1385        struct iscsi_conn *conn = task->conn;
1386        struct iscsi_session *sess = conn->session;
1387        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1388        struct cxgbi_conn *cconn = tcp_conn->dd_data;
1389        struct cxgbi_device *cdev = cconn->chba->cdev;
1390        struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
1391        u32 sw_tag = cxgbi_build_sw_tag(task->itt, sess->age);
1392        u32 tag = 0;
1393        int err = -EINVAL;
1394
1395        if (sc && sc->sc_data_direction == DMA_FROM_DEVICE) {
1396                struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
1397                struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
1398
1399                scmd_get_params(sc, &ttinfo->sgl, &ttinfo->nents,
1400                                &tdata->dlen, 0);
1401                err = cxgbi_ddp_reserve(cconn, tdata, sw_tag, tdata->dlen);
1402                if (!err)
1403                        tag = ttinfo->tag;
1404                else
1405                         log_debug(1 << CXGBI_DBG_DDP,
1406                                   "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
1407                                   cconn->cep->csk, task, tdata->dlen,
1408                                   ttinfo->nents);
1409        }
1410
1411        if (err < 0) {
1412                err = cxgbi_ppm_make_non_ddp_tag(ppm, sw_tag, &tag);
1413                if (err < 0)
1414                        return err;
1415        }
1416        /*  the itt need to sent in big-endian order */
1417        *hdr_itt = (__force itt_t)htonl(tag);
1418
1419        log_debug(1 << CXGBI_DBG_DDP,
1420                  "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
1421                  cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
1422        return 0;
1423}
1424
1425void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
1426{
1427        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1428        struct cxgbi_conn *cconn = tcp_conn->dd_data;
1429        struct cxgbi_device *cdev = cconn->chba->cdev;
1430        struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
1431        u32 tag = ntohl((__force u32)itt);
1432        u32 sw_bits;
1433
1434        if (ppm) {
1435                if (cxgbi_ppm_is_ddp_tag(ppm, tag))
1436                        sw_bits = cxgbi_ppm_get_tag_caller_data(ppm, tag);
1437                else
1438                        sw_bits = cxgbi_ppm_decode_non_ddp_tag(ppm, tag);
1439        } else {
1440                sw_bits = tag;
1441        }
1442
1443        cxgbi_decode_sw_tag(sw_bits, idx, age);
1444        log_debug(1 << CXGBI_DBG_DDP,
1445                  "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
1446                  cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
1447                  age ? *age : 0xFF);
1448}
1449EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
1450
1451void cxgbi_conn_tx_open(struct cxgbi_sock *csk)
1452{
1453        struct iscsi_conn *conn = csk->user_data;
1454
1455        if (conn) {
1456                log_debug(1 << CXGBI_DBG_SOCK,
1457                        "csk 0x%p, cid %d.\n", csk, conn->id);
1458                iscsi_conn_queue_work(conn);
1459        }
1460}
1461EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
1462
1463/*
1464 * pdu receive, interact with libiscsi_tcp
1465 */
1466static inline int read_pdu_skb(struct iscsi_conn *conn,
1467                               struct sk_buff *skb,
1468                               unsigned int offset,
1469                               int offloaded)
1470{
1471        int status = 0;
1472        int bytes_read;
1473
1474        bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
1475        switch (status) {
1476        case ISCSI_TCP_CONN_ERR:
1477                pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
1478                          skb, offset, offloaded);
1479                return -EIO;
1480        case ISCSI_TCP_SUSPENDED:
1481                log_debug(1 << CXGBI_DBG_PDU_RX,
1482                        "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
1483                        skb, offset, offloaded, bytes_read);
1484                /* no transfer - just have caller flush queue */
1485                return bytes_read;
1486        case ISCSI_TCP_SKB_DONE:
1487                pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
1488                        skb, offset, offloaded);
1489                /*
1490                 * pdus should always fit in the skb and we should get
1491                 * segment done notifcation.
1492                 */
1493                iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
1494                return -EFAULT;
1495        case ISCSI_TCP_SEGMENT_DONE:
1496                log_debug(1 << CXGBI_DBG_PDU_RX,
1497                        "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
1498                        skb, offset, offloaded, bytes_read);
1499                return bytes_read;
1500        default:
1501                pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
1502                        skb, offset, offloaded, status);
1503                return -EINVAL;
1504        }
1505}
1506
1507static int
1508skb_read_pdu_bhs(struct cxgbi_sock *csk, struct iscsi_conn *conn,
1509                 struct sk_buff *skb)
1510{
1511        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1512        int err;
1513
1514        log_debug(1 << CXGBI_DBG_PDU_RX,
1515                "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1516                conn, skb, skb->len, cxgbi_skcb_flags(skb));
1517
1518        if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
1519                pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb);
1520                iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
1521                return -EIO;
1522        }
1523
1524        if (conn->hdrdgst_en &&
1525            cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
1526                pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
1527                iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
1528                return -EIO;
1529        }
1530
1531        if (cxgbi_skcb_test_flag(skb, SKCBF_RX_ISCSI_COMPL) &&
1532            cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA_DDPD)) {
1533                /* If completion flag is set and data is directly
1534                 * placed in to the host memory then update
1535                 * task->exp_datasn to the datasn in completion
1536                 * iSCSI hdr as T6 adapter generates completion only
1537                 * for the last pdu of a sequence.
1538                 */
1539                itt_t itt = ((struct iscsi_data *)skb->data)->itt;
1540                struct iscsi_task *task = iscsi_itt_to_ctask(conn, itt);
1541                u32 data_sn = be32_to_cpu(((struct iscsi_data *)
1542                                                        skb->data)->datasn);
1543                if (task && task->sc) {
1544                        struct iscsi_tcp_task *tcp_task = task->dd_data;
1545
1546                        tcp_task->exp_datasn = data_sn;
1547                }
1548        }
1549
1550        err = read_pdu_skb(conn, skb, 0, 0);
1551        if (likely(err >= 0)) {
1552                struct iscsi_hdr *hdr = (struct iscsi_hdr *)skb->data;
1553                u8 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1554
1555                if (unlikely(opcode == ISCSI_OP_LOGOUT_RSP))
1556                        cxgbi_sock_set_flag(csk, CTPF_LOGOUT_RSP_RCVD);
1557        }
1558
1559        return err;
1560}
1561
1562static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
1563                             struct sk_buff *skb, unsigned int offset)
1564{
1565        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1566        bool offloaded = 0;
1567        int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
1568
1569        log_debug(1 << CXGBI_DBG_PDU_RX,
1570                "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1571                conn, skb, skb->len, cxgbi_skcb_flags(skb));
1572
1573        if (conn->datadgst_en &&
1574            cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
1575                pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
1576                        conn, lskb, cxgbi_skcb_flags(lskb));
1577                iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1578                return -EIO;
1579        }
1580
1581        if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
1582                return 0;
1583
1584        /* coalesced, add header digest length */
1585        if (lskb == skb && conn->hdrdgst_en)
1586                offset += ISCSI_DIGEST_SIZE;
1587
1588        if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
1589                offloaded = 1;
1590
1591        if (opcode == ISCSI_OP_SCSI_DATA_IN)
1592                log_debug(1 << CXGBI_DBG_PDU_RX,
1593                        "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
1594                        skb, opcode, ntohl(tcp_conn->in.hdr->itt),
1595                        tcp_conn->in.datalen, offloaded ? "is" : "not");
1596
1597        return read_pdu_skb(conn, skb, offset, offloaded);
1598}
1599
1600static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
1601{
1602        struct cxgbi_device *cdev = csk->cdev;
1603        int must_send;
1604        u32 credits;
1605
1606        log_debug(1 << CXGBI_DBG_PDU_RX,
1607                "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
1608                csk, csk->state, csk->flags, csk->tid, csk->copied_seq,
1609                csk->rcv_wup, cdev->rx_credit_thres,
1610                csk->rcv_win);
1611
1612        if (!cdev->rx_credit_thres)
1613                return;
1614
1615        if (csk->state != CTP_ESTABLISHED)
1616                return;
1617
1618        credits = csk->copied_seq - csk->rcv_wup;
1619        if (unlikely(!credits))
1620                return;
1621        must_send = credits + 16384 >= csk->rcv_win;
1622        if (must_send || credits >= cdev->rx_credit_thres)
1623                csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
1624}
1625
1626void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
1627{
1628        struct cxgbi_device *cdev = csk->cdev;
1629        struct iscsi_conn *conn = csk->user_data;
1630        struct sk_buff *skb;
1631        unsigned int read = 0;
1632        int err = 0;
1633
1634        log_debug(1 << CXGBI_DBG_PDU_RX,
1635                "csk 0x%p, conn 0x%p.\n", csk, conn);
1636
1637        if (unlikely(!conn || conn->suspend_rx)) {
1638                log_debug(1 << CXGBI_DBG_PDU_RX,
1639                        "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
1640                        csk, conn, conn ? conn->id : 0xFF,
1641                        conn ? conn->suspend_rx : 0xFF);
1642                return;
1643        }
1644
1645        while (!err) {
1646                skb = skb_peek(&csk->receive_queue);
1647                if (!skb ||
1648                    !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
1649                        if (skb)
1650                                log_debug(1 << CXGBI_DBG_PDU_RX,
1651                                        "skb 0x%p, NOT ready 0x%lx.\n",
1652                                        skb, cxgbi_skcb_flags(skb));
1653                        break;
1654                }
1655                __skb_unlink(skb, &csk->receive_queue);
1656
1657                read += cxgbi_skcb_rx_pdulen(skb);
1658                log_debug(1 << CXGBI_DBG_PDU_RX,
1659                        "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
1660                        csk, skb, skb->len, cxgbi_skcb_flags(skb),
1661                        cxgbi_skcb_rx_pdulen(skb));
1662
1663                if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
1664                        err = skb_read_pdu_bhs(csk, conn, skb);
1665                        if (err < 0) {
1666                                pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
1667                                        "f 0x%lx, plen %u.\n",
1668                                        csk, skb, skb->len,
1669                                        cxgbi_skcb_flags(skb),
1670                                        cxgbi_skcb_rx_pdulen(skb));
1671                                goto skb_done;
1672                        }
1673                        err = skb_read_pdu_data(conn, skb, skb,
1674                                                err + cdev->skb_rx_extra);
1675                        if (err < 0)
1676                                pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
1677                                        "f 0x%lx, plen %u.\n",
1678                                        csk, skb, skb->len,
1679                                        cxgbi_skcb_flags(skb),
1680                                        cxgbi_skcb_rx_pdulen(skb));
1681                } else {
1682                        err = skb_read_pdu_bhs(csk, conn, skb);
1683                        if (err < 0) {
1684                                pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
1685                                        "f 0x%lx, plen %u.\n",
1686                                        csk, skb, skb->len,
1687                                        cxgbi_skcb_flags(skb),
1688                                        cxgbi_skcb_rx_pdulen(skb));
1689                                goto skb_done;
1690                        }
1691
1692                        if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
1693                                struct sk_buff *dskb;
1694
1695                                dskb = skb_peek(&csk->receive_queue);
1696                                if (!dskb) {
1697                                        pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
1698                                                " plen %u, NO data.\n",
1699                                                csk, skb, skb->len,
1700                                                cxgbi_skcb_flags(skb),
1701                                                cxgbi_skcb_rx_pdulen(skb));
1702                                        err = -EIO;
1703                                        goto skb_done;
1704                                }
1705                                __skb_unlink(dskb, &csk->receive_queue);
1706
1707                                err = skb_read_pdu_data(conn, skb, dskb, 0);
1708                                if (err < 0)
1709                                        pr_err("data, csk 0x%p, skb 0x%p,%u, "
1710                                                "f 0x%lx, plen %u, dskb 0x%p,"
1711                                                "%u.\n",
1712                                                csk, skb, skb->len,
1713                                                cxgbi_skcb_flags(skb),
1714                                                cxgbi_skcb_rx_pdulen(skb),
1715                                                dskb, dskb->len);
1716                                __kfree_skb(dskb);
1717                        } else
1718                                err = skb_read_pdu_data(conn, skb, skb, 0);
1719                }
1720skb_done:
1721                __kfree_skb(skb);
1722
1723                if (err < 0)
1724                        break;
1725        }
1726
1727        log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read);
1728        if (read) {
1729                csk->copied_seq += read;
1730                csk_return_rx_credits(csk, read);
1731                conn->rxdata_octets += read;
1732        }
1733
1734        if (err < 0) {
1735                pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
1736                        csk, conn, err, read);
1737                iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1738        }
1739}
1740EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready);
1741
1742static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
1743                                unsigned int offset, unsigned int *off,
1744                                struct scatterlist **sgp)
1745{
1746        int i;
1747        struct scatterlist *sg;
1748
1749        for_each_sg(sgl, sg, sgcnt, i) {
1750                if (offset < sg->length) {
1751                        *off = offset;
1752                        *sgp = sg;
1753                        return 0;
1754                }
1755                offset -= sg->length;
1756        }
1757        return -EFAULT;
1758}
1759
1760static int
1761sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
1762                  unsigned int dlen, struct page_frag *frags,
1763                  int frag_max, u32 *dlimit)
1764{
1765        unsigned int datalen = dlen;
1766        unsigned int sglen = sg->length - sgoffset;
1767        struct page *page = sg_page(sg);
1768        int i;
1769
1770        i = 0;
1771        do {
1772                unsigned int copy;
1773
1774                if (!sglen) {
1775                        sg = sg_next(sg);
1776                        if (!sg) {
1777                                pr_warn("sg %d NULL, len %u/%u.\n",
1778                                        i, datalen, dlen);
1779                                return -EINVAL;
1780                        }
1781                        sgoffset = 0;
1782                        sglen = sg->length;
1783                        page = sg_page(sg);
1784
1785                }
1786                copy = min(datalen, sglen);
1787                if (i && page == frags[i - 1].page &&
1788                    sgoffset + sg->offset ==
1789                        frags[i - 1].offset + frags[i - 1].size) {
1790                        frags[i - 1].size += copy;
1791                } else {
1792                        if (i >= frag_max) {
1793                                pr_warn("too many pages %u, dlen %u.\n",
1794                                        frag_max, dlen);
1795                                *dlimit = dlen - datalen;
1796                                return -EINVAL;
1797                        }
1798
1799                        frags[i].page = page;
1800                        frags[i].offset = sg->offset + sgoffset;
1801                        frags[i].size = copy;
1802                        i++;
1803                }
1804                datalen -= copy;
1805                sgoffset += copy;
1806                sglen -= copy;
1807        } while (datalen);
1808
1809        return i;
1810}
1811
1812static void cxgbi_task_data_sgl_check(struct iscsi_task *task)
1813{
1814        struct scsi_cmnd *sc = task->sc;
1815        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
1816        struct scatterlist *sg, *sgl = NULL;
1817        u32 sgcnt = 0;
1818        int i;
1819
1820        tdata->flags = CXGBI_TASK_SGL_CHECKED;
1821        if (!sc)
1822                return;
1823
1824        scmd_get_params(sc, &sgl, &sgcnt, &tdata->dlen, 0);
1825        if (!sgl || !sgcnt) {
1826                tdata->flags |= CXGBI_TASK_SGL_COPY;
1827                return;
1828        }
1829
1830        for_each_sg(sgl, sg, sgcnt, i) {
1831                if (page_count(sg_page(sg)) < 1) {
1832                        tdata->flags |= CXGBI_TASK_SGL_COPY;
1833                        return;
1834                }
1835        }
1836}
1837
1838static int
1839cxgbi_task_data_sgl_read(struct iscsi_task *task, u32 offset, u32 count,
1840                         u32 *dlimit)
1841{
1842        struct scsi_cmnd *sc = task->sc;
1843        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
1844        struct scatterlist *sgl = NULL;
1845        struct scatterlist *sg;
1846        u32 dlen = 0;
1847        u32 sgcnt;
1848        int err;
1849
1850        if (!sc)
1851                return 0;
1852
1853        scmd_get_params(sc, &sgl, &sgcnt, &dlen, 0);
1854        if (!sgl || !sgcnt)
1855                return 0;
1856
1857        err = sgl_seek_offset(sgl, sgcnt, offset, &tdata->sgoffset, &sg);
1858        if (err < 0) {
1859                pr_warn("tpdu max, sgl %u, bad offset %u/%u.\n",
1860                        sgcnt, offset, tdata->dlen);
1861                return err;
1862        }
1863        err = sgl_read_to_frags(sg, tdata->sgoffset, count,
1864                                tdata->frags, MAX_SKB_FRAGS, dlimit);
1865        if (err < 0) {
1866                log_debug(1 << CXGBI_DBG_ISCSI,
1867                          "sgl max limit, sgl %u, offset %u, %u/%u, dlimit %u.\n",
1868                          sgcnt, offset, count, tdata->dlen, *dlimit);
1869                return err;
1870        }
1871        tdata->offset = offset;
1872        tdata->count = count;
1873        tdata->nr_frags = err;
1874        tdata->total_count = count;
1875        tdata->total_offset = offset;
1876
1877        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
1878                  "%s: offset %u, count %u,\n"
1879                  "err %u, total_count %u, total_offset %u\n",
1880                  __func__, offset, count, err,  tdata->total_count, tdata->total_offset);
1881
1882        return 0;
1883}
1884
1885int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 op)
1886{
1887        struct iscsi_conn *conn = task->conn;
1888        struct iscsi_session *session = task->conn->session;
1889        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1890        struct cxgbi_conn *cconn = tcp_conn->dd_data;
1891        struct cxgbi_device *cdev = cconn->chba->cdev;
1892        struct cxgbi_sock *csk = cconn->cep ? cconn->cep->csk : NULL;
1893        struct iscsi_tcp_task *tcp_task = task->dd_data;
1894        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
1895        struct scsi_cmnd *sc = task->sc;
1896        u32 headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
1897        u32 max_txdata_len = conn->max_xmit_dlength;
1898        u32 iso_tx_rsvd = 0, local_iso_info = 0;
1899        u32 last_tdata_offset, last_tdata_count;
1900        int err = 0;
1901
1902        if (!tcp_task) {
1903                pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p.\n",
1904                       task, tcp_task, tdata);
1905                return -ENOMEM;
1906        }
1907        if (!csk) {
1908                pr_err("task 0x%p, csk gone.\n", task);
1909                return -EPIPE;
1910        }
1911
1912        op &= ISCSI_OPCODE_MASK;
1913
1914        tcp_task->dd_data = tdata;
1915        task->hdr = NULL;
1916
1917        last_tdata_count = tdata->count;
1918        last_tdata_offset = tdata->offset;
1919
1920        if ((op == ISCSI_OP_SCSI_DATA_OUT) ||
1921            ((op == ISCSI_OP_SCSI_CMD) &&
1922             (sc->sc_data_direction == DMA_TO_DEVICE))) {
1923                u32 remaining_data_tosend, dlimit = 0;
1924                u32 max_pdu_size, max_num_pdu, num_pdu;
1925                u32 count;
1926
1927                /* Preserve conn->max_xmit_dlength because it can get updated to
1928                 * ISO data size.
1929                 */
1930                if (task->state == ISCSI_TASK_PENDING)
1931                        tdata->max_xmit_dlength = conn->max_xmit_dlength;
1932
1933                if (!tdata->offset)
1934                        cxgbi_task_data_sgl_check(task);
1935
1936                remaining_data_tosend =
1937                        tdata->dlen - tdata->offset - tdata->count;
1938
1939recalculate_sgl:
1940                max_txdata_len = tdata->max_xmit_dlength;
1941                log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
1942                          "tdata->dlen %u, remaining to send %u "
1943                          "conn->max_xmit_dlength %u, "
1944                          "tdata->max_xmit_dlength %u\n",
1945                          tdata->dlen, remaining_data_tosend,
1946                          conn->max_xmit_dlength, tdata->max_xmit_dlength);
1947
1948                if (cdev->skb_iso_txhdr && !csk->disable_iso &&
1949                    (remaining_data_tosend > tdata->max_xmit_dlength) &&
1950                    !(remaining_data_tosend % 4)) {
1951                        u32 max_iso_data;
1952
1953                        if ((op == ISCSI_OP_SCSI_CMD) &&
1954                            session->initial_r2t_en)
1955                                goto no_iso;
1956
1957                        max_pdu_size = tdata->max_xmit_dlength +
1958                                       ISCSI_PDU_NONPAYLOAD_LEN;
1959                        max_iso_data = rounddown(CXGBI_MAX_ISO_DATA_IN_SKB,
1960                                                 csk->advmss);
1961                        max_num_pdu = max_iso_data / max_pdu_size;
1962
1963                        num_pdu = (remaining_data_tosend +
1964                                   tdata->max_xmit_dlength - 1) /
1965                                  tdata->max_xmit_dlength;
1966
1967                        if (num_pdu > max_num_pdu)
1968                                num_pdu = max_num_pdu;
1969
1970                        conn->max_xmit_dlength = tdata->max_xmit_dlength * num_pdu;
1971                        max_txdata_len = conn->max_xmit_dlength;
1972                        iso_tx_rsvd = cdev->skb_iso_txhdr;
1973                        local_iso_info = sizeof(struct cxgbi_iso_info);
1974
1975                        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
1976                                  "max_pdu_size %u, max_num_pdu %u, "
1977                                  "max_txdata %u, num_pdu %u\n",
1978                                  max_pdu_size, max_num_pdu,
1979                                  max_txdata_len, num_pdu);
1980                }
1981no_iso:
1982                count  = min_t(u32, max_txdata_len, remaining_data_tosend);
1983                err = cxgbi_task_data_sgl_read(task,
1984                                               tdata->offset + tdata->count,
1985                                               count, &dlimit);
1986                if (unlikely(err < 0)) {
1987                        log_debug(1 << CXGBI_DBG_ISCSI,
1988                                  "task 0x%p, tcp_task 0x%p, tdata 0x%p, "
1989                                  "sgl err %d, count %u, dlimit %u\n",
1990                                  task, tcp_task, tdata, err, count, dlimit);
1991                        if (dlimit) {
1992                                remaining_data_tosend =
1993                                        rounddown(dlimit,
1994                                                  tdata->max_xmit_dlength);
1995                                if (!remaining_data_tosend)
1996                                        remaining_data_tosend = dlimit;
1997
1998                                dlimit = 0;
1999
2000                                conn->max_xmit_dlength = remaining_data_tosend;
2001                                goto recalculate_sgl;
2002                        }
2003
2004                        pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p, "
2005                                "sgl err %d\n",
2006                                task, tcp_task, tdata, err);
2007                        goto ret_err;
2008                }
2009
2010                if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
2011                    (tdata->nr_frags > MAX_SKB_FRAGS))
2012                        headroom += conn->max_xmit_dlength;
2013        }
2014
2015        tdata->skb = alloc_skb(local_iso_info + cdev->skb_tx_rsvd +
2016                               iso_tx_rsvd + headroom, GFP_ATOMIC);
2017        if (!tdata->skb) {
2018                tdata->count = last_tdata_count;
2019                tdata->offset = last_tdata_offset;
2020                err = -ENOMEM;
2021                goto ret_err;
2022        }
2023
2024        skb_reserve(tdata->skb, local_iso_info + cdev->skb_tx_rsvd +
2025                    iso_tx_rsvd);
2026
2027        if (task->sc) {
2028                task->hdr = (struct iscsi_hdr *)tdata->skb->data;
2029        } else {
2030                task->hdr = kzalloc(SKB_TX_ISCSI_PDU_HEADER_MAX, GFP_ATOMIC);
2031                if (!task->hdr) {
2032                        __kfree_skb(tdata->skb);
2033                        tdata->skb = NULL;
2034                        return -ENOMEM;
2035                }
2036        }
2037
2038        task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX;
2039
2040        if (iso_tx_rsvd)
2041                cxgbi_skcb_set_flag(tdata->skb, SKCBF_TX_ISO);
2042
2043        /* data_out uses scsi_cmd's itt */
2044        if (op != ISCSI_OP_SCSI_DATA_OUT)
2045                task_reserve_itt(task, &task->hdr->itt);
2046
2047        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2048                  "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
2049                  task, op, tdata->skb, cdev->skb_tx_rsvd, headroom,
2050                  conn->max_xmit_dlength, be32_to_cpu(task->hdr->itt));
2051
2052        return 0;
2053
2054ret_err:
2055        conn->max_xmit_dlength = tdata->max_xmit_dlength;
2056        return err;
2057}
2058EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
2059
2060static int
2061cxgbi_prep_iso_info(struct iscsi_task *task, struct sk_buff *skb,
2062                    u32 count)
2063{
2064        struct cxgbi_iso_info *iso_info = (struct cxgbi_iso_info *)skb->head;
2065        struct iscsi_r2t_info *r2t;
2066        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2067        struct iscsi_conn *conn = task->conn;
2068        struct iscsi_session *session = conn->session;
2069        struct iscsi_tcp_task *tcp_task = task->dd_data;
2070        u32 burst_size = 0, r2t_dlength = 0, dlength;
2071        u32 max_pdu_len = tdata->max_xmit_dlength;
2072        u32 segment_offset = 0;
2073        u32 num_pdu;
2074
2075        if (unlikely(!cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
2076                return 0;
2077
2078        memset(iso_info, 0, sizeof(struct cxgbi_iso_info));
2079
2080        if (task->hdr->opcode == ISCSI_OP_SCSI_CMD && session->imm_data_en) {
2081                iso_info->flags |= CXGBI_ISO_INFO_IMM_ENABLE;
2082                burst_size = count;
2083        }
2084
2085        dlength = ntoh24(task->hdr->dlength);
2086        dlength = min(dlength, max_pdu_len);
2087        hton24(task->hdr->dlength, dlength);
2088
2089        num_pdu = (count + max_pdu_len - 1) / max_pdu_len;
2090
2091        if (iscsi_task_has_unsol_data(task))
2092                r2t = &task->unsol_r2t;
2093        else
2094                r2t = tcp_task->r2t;
2095
2096        if (r2t) {
2097                log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2098                          "count %u, tdata->count %u, num_pdu %u,"
2099                          "task->hdr_len %u, r2t->data_length %u, r2t->sent %u\n",
2100                          count, tdata->count, num_pdu, task->hdr_len,
2101                          r2t->data_length, r2t->sent);
2102
2103                r2t_dlength = r2t->data_length - r2t->sent;
2104                segment_offset = r2t->sent;
2105                r2t->datasn += num_pdu - 1;
2106        }
2107
2108        if (!r2t || !r2t->sent)
2109                iso_info->flags |= CXGBI_ISO_INFO_FSLICE;
2110
2111        if (task->hdr->flags & ISCSI_FLAG_CMD_FINAL)
2112                iso_info->flags |= CXGBI_ISO_INFO_LSLICE;
2113
2114        task->hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
2115
2116        iso_info->op = task->hdr->opcode;
2117        iso_info->ahs = task->hdr->hlength;
2118        iso_info->num_pdu = num_pdu;
2119        iso_info->mpdu = max_pdu_len;
2120        iso_info->burst_size = (burst_size + r2t_dlength) >> 2;
2121        iso_info->len = count + task->hdr_len;
2122        iso_info->segment_offset = segment_offset;
2123
2124        cxgbi_skcb_tx_iscsi_hdrlen(skb) = task->hdr_len;
2125        return 0;
2126}
2127
2128static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
2129{
2130        if (hcrc || dcrc) {
2131                u8 submode = 0;
2132
2133                if (hcrc)
2134                        submode |= 1;
2135                if (dcrc)
2136                        submode |= 2;
2137                cxgbi_skcb_tx_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
2138        } else
2139                cxgbi_skcb_tx_ulp_mode(skb) = 0;
2140}
2141
2142static struct page *rsvd_page;
2143
2144int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
2145                              unsigned int count)
2146{
2147        struct iscsi_conn *conn = task->conn;
2148        struct iscsi_tcp_task *tcp_task = task->dd_data;
2149        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2150        struct sk_buff *skb;
2151        struct scsi_cmnd *sc = task->sc;
2152        u32 expected_count, expected_offset;
2153        u32 datalen = count, dlimit = 0;
2154        u32 i, padlen = iscsi_padding(count);
2155        struct page *pg;
2156        int err;
2157
2158        if (!tcp_task || (tcp_task->dd_data != tdata)) {
2159                pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
2160                       task, task->sc, tcp_task,
2161                       tcp_task ? tcp_task->dd_data : NULL, tdata);
2162                return -EINVAL;
2163        }
2164        skb = tdata->skb;
2165
2166        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2167                  "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
2168                  task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
2169                  be32_to_cpu(task->cmdsn), be32_to_cpu(task->hdr->itt), offset, count);
2170
2171        skb_put(skb, task->hdr_len);
2172        tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
2173        if (!count) {
2174                tdata->count = count;
2175                tdata->offset = offset;
2176                tdata->nr_frags = 0;
2177                tdata->total_offset = 0;
2178                tdata->total_count = 0;
2179                if (tdata->max_xmit_dlength)
2180                        conn->max_xmit_dlength = tdata->max_xmit_dlength;
2181                cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
2182                return 0;
2183        }
2184
2185        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2186                  "data->total_count %u, tdata->total_offset %u\n",
2187                  tdata->total_count, tdata->total_offset);
2188
2189        expected_count = tdata->total_count;
2190        expected_offset = tdata->total_offset;
2191
2192        if ((count != expected_count) ||
2193            (offset != expected_offset)) {
2194                err = cxgbi_task_data_sgl_read(task, offset, count, &dlimit);
2195                if (err < 0) {
2196                        pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p "
2197                               "dlimit %u, sgl err %d.\n", task, task->sc,
2198                               tcp_task, tcp_task ? tcp_task->dd_data : NULL,
2199                               tdata, dlimit, err);
2200                        return err;
2201                }
2202        }
2203
2204        /* Restore original value of conn->max_xmit_dlength because
2205         * it can get updated to ISO data size.
2206         */
2207        conn->max_xmit_dlength = tdata->max_xmit_dlength;
2208
2209        if (sc) {
2210                struct page_frag *frag = tdata->frags;
2211
2212                if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
2213                    (tdata->nr_frags > MAX_SKB_FRAGS) ||
2214                    (padlen && (tdata->nr_frags ==
2215                                        MAX_SKB_FRAGS))) {
2216                        char *dst = skb->data + task->hdr_len;
2217
2218                        /* data fits in the skb's headroom */
2219                        for (i = 0; i < tdata->nr_frags; i++, frag++) {
2220                                char *src = kmap_atomic(frag->page);
2221
2222                                memcpy(dst, src + frag->offset, frag->size);
2223                                dst += frag->size;
2224                                kunmap_atomic(src);
2225                        }
2226
2227                        if (padlen) {
2228                                memset(dst, 0, padlen);
2229                                padlen = 0;
2230                        }
2231                        skb_put(skb, count + padlen);
2232                } else {
2233                        for (i = 0; i < tdata->nr_frags; i++, frag++) {
2234                                get_page(frag->page);
2235                                skb_fill_page_desc(skb, i, frag->page,
2236                                                   frag->offset, frag->size);
2237                        }
2238
2239                        skb->len += count;
2240                        skb->data_len += count;
2241                        skb->truesize += count;
2242                }
2243        } else {
2244                pg = virt_to_head_page(task->data);
2245                get_page(pg);
2246                skb_fill_page_desc(skb, 0, pg,
2247                                   task->data - (char *)page_address(pg),
2248                                   count);
2249                skb->len += count;
2250                skb->data_len += count;
2251                skb->truesize += count;
2252        }
2253
2254        if (padlen) {
2255                get_page(rsvd_page);
2256                skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
2257                                   rsvd_page, 0, padlen);
2258
2259                skb->data_len += padlen;
2260                skb->truesize += padlen;
2261                skb->len += padlen;
2262        }
2263
2264        if (likely(count > tdata->max_xmit_dlength))
2265                cxgbi_prep_iso_info(task, skb, count);
2266        else
2267                cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
2268
2269        return 0;
2270}
2271EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
2272
2273static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb)
2274{
2275        struct cxgbi_device *cdev = csk->cdev;
2276        struct cxgbi_iso_info *iso_cpl;
2277        u32 frags = skb_shinfo(skb)->nr_frags;
2278        u32 extra_len, num_pdu, hdr_len;
2279        u32 iso_tx_rsvd = 0;
2280
2281        if (csk->state != CTP_ESTABLISHED) {
2282                log_debug(1 << CXGBI_DBG_PDU_TX,
2283                          "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
2284                          csk, csk->state, csk->flags, csk->tid);
2285                return -EPIPE;
2286        }
2287
2288        if (csk->err) {
2289                log_debug(1 << CXGBI_DBG_PDU_TX,
2290                          "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
2291                          csk, csk->state, csk->flags, csk->tid, csk->err);
2292                return -EPIPE;
2293        }
2294
2295        if ((cdev->flags & CXGBI_FLAG_DEV_T3) &&
2296            before((csk->snd_win + csk->snd_una), csk->write_seq)) {
2297                log_debug(1 << CXGBI_DBG_PDU_TX,
2298                          "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
2299                          csk, csk->state, csk->flags, csk->tid, csk->write_seq,
2300                          csk->snd_una, csk->snd_win);
2301                return -ENOBUFS;
2302        }
2303
2304        if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
2305                iso_tx_rsvd = cdev->skb_iso_txhdr;
2306
2307        if (unlikely(skb_headroom(skb) < (cdev->skb_tx_rsvd + iso_tx_rsvd))) {
2308                pr_err("csk 0x%p, skb head %u < %u.\n",
2309                       csk, skb_headroom(skb), cdev->skb_tx_rsvd);
2310                return -EINVAL;
2311        }
2312
2313        if (skb->len != skb->data_len)
2314                frags++;
2315
2316        if (frags >= SKB_WR_LIST_SIZE) {
2317                pr_err("csk 0x%p, frags %u, %u,%u >%lu.\n",
2318                       csk, skb_shinfo(skb)->nr_frags, skb->len,
2319                       skb->data_len, SKB_WR_LIST_SIZE);
2320                return -EINVAL;
2321        }
2322
2323        cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
2324        skb_reset_transport_header(skb);
2325        cxgbi_sock_skb_entail(csk, skb);
2326
2327        extra_len = cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
2328
2329        if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) {
2330                iso_cpl = (struct cxgbi_iso_info *)skb->head;
2331                num_pdu = iso_cpl->num_pdu;
2332                hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
2333                extra_len = (cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)) *
2334                             num_pdu) + (hdr_len * (num_pdu - 1));
2335        }
2336
2337        csk->write_seq += (skb->len + extra_len);
2338
2339        return 0;
2340}
2341
2342static int cxgbi_sock_send_skb(struct cxgbi_sock *csk, struct sk_buff *skb)
2343{
2344        struct cxgbi_device *cdev = csk->cdev;
2345        int len = skb->len;
2346        int err;
2347
2348        spin_lock_bh(&csk->lock);
2349        err = cxgbi_sock_tx_queue_up(csk, skb);
2350        if (err < 0) {
2351                spin_unlock_bh(&csk->lock);
2352                return err;
2353        }
2354
2355        if (likely(skb_queue_len(&csk->write_queue)))
2356                cdev->csk_push_tx_frames(csk, 0);
2357        spin_unlock_bh(&csk->lock);
2358        return len;
2359}
2360
2361int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
2362{
2363        struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
2364        struct cxgbi_conn *cconn = tcp_conn->dd_data;
2365        struct iscsi_tcp_task *tcp_task = task->dd_data;
2366        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2367        struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
2368        struct sk_buff *skb;
2369        struct cxgbi_sock *csk = NULL;
2370        u32 pdulen = 0;
2371        u32 datalen;
2372        int err;
2373
2374        if (!tcp_task || (tcp_task->dd_data != tdata)) {
2375                pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
2376                       task, task->sc, tcp_task,
2377                       tcp_task ? tcp_task->dd_data : NULL, tdata);
2378                return -EINVAL;
2379        }
2380
2381        skb = tdata->skb;
2382        if (!skb) {
2383                log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2384                          "task 0x%p, skb NULL.\n", task);
2385                return 0;
2386        }
2387
2388        if (cconn && cconn->cep)
2389                csk = cconn->cep->csk;
2390
2391        if (!csk) {
2392                log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2393                          "task 0x%p, csk gone.\n", task);
2394                return -EPIPE;
2395        }
2396
2397        tdata->skb = NULL;
2398        datalen = skb->data_len;
2399
2400        /* write ppod first if using ofldq to write ppod */
2401        if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_VALID) {
2402                struct cxgbi_ppm *ppm = csk->cdev->cdev2ppm(csk->cdev);
2403
2404                ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_VALID;
2405                if (csk->cdev->csk_ddp_set_map(ppm, csk, ttinfo) < 0)
2406                        pr_err("task 0x%p, ppod writing using ofldq failed.\n",
2407                               task);
2408                        /* continue. Let fl get the data */
2409        }
2410
2411        if (!task->sc)
2412                memcpy(skb->data, task->hdr, SKB_TX_ISCSI_PDU_HEADER_MAX);
2413
2414        err = cxgbi_sock_send_skb(csk, skb);
2415        if (err > 0) {
2416                pdulen += err;
2417
2418                log_debug(1 << CXGBI_DBG_PDU_TX, "task 0x%p,0x%p, rv %d.\n",
2419                          task, task->sc, err);
2420
2421                if (task->conn->hdrdgst_en)
2422                        pdulen += ISCSI_DIGEST_SIZE;
2423
2424                if (datalen && task->conn->datadgst_en)
2425                        pdulen += ISCSI_DIGEST_SIZE;
2426
2427                task->conn->txdata_octets += pdulen;
2428
2429                if (unlikely(cxgbi_is_iso_config(csk) && cxgbi_is_iso_disabled(csk))) {
2430                        if (time_after(jiffies, csk->prev_iso_ts + HZ)) {
2431                                csk->disable_iso = false;
2432                                csk->prev_iso_ts = 0;
2433                                log_debug(1 << CXGBI_DBG_PDU_TX,
2434                                          "enable iso: csk 0x%p\n", csk);
2435                        }
2436                }
2437
2438                return 0;
2439        }
2440
2441        if (err == -EAGAIN || err == -ENOBUFS) {
2442                log_debug(1 << CXGBI_DBG_PDU_TX,
2443                          "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
2444                          task, skb, skb->len, skb->data_len, err);
2445                /* reset skb to send when we are called again */
2446                tdata->skb = skb;
2447
2448                if (cxgbi_is_iso_config(csk) && !cxgbi_is_iso_disabled(csk) &&
2449                    (csk->no_tx_credits++ >= 2)) {
2450                        csk->disable_iso = true;
2451                        csk->prev_iso_ts = jiffies;
2452                        log_debug(1 << CXGBI_DBG_PDU_TX,
2453                                  "disable iso:csk 0x%p, ts:%lu\n",
2454                                  csk, csk->prev_iso_ts);
2455                }
2456
2457                return err;
2458        }
2459
2460        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2461                  "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
2462                  task->itt, skb, skb->len, skb->data_len, err);
2463        __kfree_skb(skb);
2464        iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
2465        iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
2466        return err;
2467}
2468EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu);
2469
2470void cxgbi_cleanup_task(struct iscsi_task *task)
2471{
2472        struct iscsi_tcp_task *tcp_task = task->dd_data;
2473        struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2474
2475        if (!tcp_task || (tcp_task->dd_data != tdata)) {
2476                pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
2477                        task, task->sc, tcp_task,
2478                        tcp_task ? tcp_task->dd_data : NULL, tdata);
2479                return;
2480        }
2481
2482        log_debug(1 << CXGBI_DBG_ISCSI,
2483                "task 0x%p, skb 0x%p, itt 0x%x.\n",
2484                task, tdata->skb, task->hdr_itt);
2485
2486        tcp_task->dd_data = NULL;
2487
2488        if (!task->sc)
2489                kfree(task->hdr);
2490        task->hdr = NULL;
2491
2492        /*  never reached the xmit task callout */
2493        if (tdata->skb) {
2494                __kfree_skb(tdata->skb);
2495                tdata->skb = NULL;
2496        }
2497
2498        task_release_itt(task, task->hdr_itt);
2499        memset(tdata, 0, sizeof(*tdata));
2500
2501        iscsi_tcp_cleanup_task(task);
2502}
2503EXPORT_SYMBOL_GPL(cxgbi_cleanup_task);
2504
2505void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn,
2506                                struct iscsi_stats *stats)
2507{
2508        struct iscsi_conn *conn = cls_conn->dd_data;
2509
2510        stats->txdata_octets = conn->txdata_octets;
2511        stats->rxdata_octets = conn->rxdata_octets;
2512        stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
2513        stats->dataout_pdus = conn->dataout_pdus_cnt;
2514        stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
2515        stats->datain_pdus = conn->datain_pdus_cnt;
2516        stats->r2t_pdus = conn->r2t_pdus_cnt;
2517        stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
2518        stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
2519        stats->digest_err = 0;
2520        stats->timeout_err = 0;
2521        stats->custom_length = 1;
2522        strcpy(stats->custom[0].desc, "eh_abort_cnt");
2523        stats->custom[0].value = conn->eh_abort_cnt;
2524}
2525EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats);
2526
2527static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn)
2528{
2529        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2530        struct cxgbi_conn *cconn = tcp_conn->dd_data;
2531        struct cxgbi_device *cdev = cconn->chba->cdev;
2532        unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd);
2533        unsigned int max_def = 512 * MAX_SKB_FRAGS;
2534        unsigned int max = max(max_def, headroom);
2535
2536        max = min(cconn->chba->cdev->tx_max_size, max);
2537        if (conn->max_xmit_dlength)
2538                conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
2539        else
2540                conn->max_xmit_dlength = max;
2541        cxgbi_align_pdu_size(conn->max_xmit_dlength);
2542
2543        return 0;
2544}
2545
2546static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn)
2547{
2548        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2549        struct cxgbi_conn *cconn = tcp_conn->dd_data;
2550        unsigned int max = cconn->chba->cdev->rx_max_size;
2551
2552        cxgbi_align_pdu_size(max);
2553
2554        if (conn->max_recv_dlength) {
2555                if (conn->max_recv_dlength > max) {
2556                        pr_err("MaxRecvDataSegmentLength %u > %u.\n",
2557                                conn->max_recv_dlength, max);
2558                        return -EINVAL;
2559                }
2560                conn->max_recv_dlength = min(conn->max_recv_dlength, max);
2561                cxgbi_align_pdu_size(conn->max_recv_dlength);
2562        } else
2563                conn->max_recv_dlength = max;
2564
2565        return 0;
2566}
2567
2568int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn,
2569                        enum iscsi_param param, char *buf, int buflen)
2570{
2571        struct iscsi_conn *conn = cls_conn->dd_data;
2572        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2573        struct cxgbi_conn *cconn = tcp_conn->dd_data;
2574        struct cxgbi_sock *csk = cconn->cep->csk;
2575        int err;
2576
2577        log_debug(1 << CXGBI_DBG_ISCSI,
2578                "cls_conn 0x%p, param %d, buf(%d) %s.\n",
2579                cls_conn, param, buflen, buf);
2580
2581        switch (param) {
2582        case ISCSI_PARAM_HDRDGST_EN:
2583                err = iscsi_set_param(cls_conn, param, buf, buflen);
2584                if (!err && conn->hdrdgst_en)
2585                        err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2586                                                        conn->hdrdgst_en,
2587                                                        conn->datadgst_en);
2588                break;
2589        case ISCSI_PARAM_DATADGST_EN:
2590                err = iscsi_set_param(cls_conn, param, buf, buflen);
2591                if (!err && conn->datadgst_en)
2592                        err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2593                                                        conn->hdrdgst_en,
2594                                                        conn->datadgst_en);
2595                break;
2596        case ISCSI_PARAM_MAX_R2T:
2597                return iscsi_tcp_set_max_r2t(conn, buf);
2598        case ISCSI_PARAM_MAX_RECV_DLENGTH:
2599                err = iscsi_set_param(cls_conn, param, buf, buflen);
2600                if (!err)
2601                        err = cxgbi_conn_max_recv_dlength(conn);
2602                break;
2603        case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2604                err = iscsi_set_param(cls_conn, param, buf, buflen);
2605                if (!err)
2606                        err = cxgbi_conn_max_xmit_dlength(conn);
2607                break;
2608        default:
2609                return iscsi_set_param(cls_conn, param, buf, buflen);
2610        }
2611        return err;
2612}
2613EXPORT_SYMBOL_GPL(cxgbi_set_conn_param);
2614
2615int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
2616                       char *buf)
2617{
2618        struct cxgbi_endpoint *cep = ep->dd_data;
2619        struct cxgbi_sock *csk;
2620
2621        log_debug(1 << CXGBI_DBG_ISCSI,
2622                "cls_conn 0x%p, param %d.\n", ep, param);
2623
2624        switch (param) {
2625        case ISCSI_PARAM_CONN_PORT:
2626        case ISCSI_PARAM_CONN_ADDRESS:
2627                if (!cep)
2628                        return -ENOTCONN;
2629
2630                csk = cep->csk;
2631                if (!csk)
2632                        return -ENOTCONN;
2633
2634                return iscsi_conn_get_addr_param((struct sockaddr_storage *)
2635                                                 &csk->daddr, param, buf);
2636        default:
2637                break;
2638        }
2639        return -ENOSYS;
2640}
2641EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
2642
2643struct iscsi_cls_conn *
2644cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid)
2645{
2646        struct iscsi_cls_conn *cls_conn;
2647        struct iscsi_conn *conn;
2648        struct iscsi_tcp_conn *tcp_conn;
2649        struct cxgbi_conn *cconn;
2650
2651        cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
2652        if (!cls_conn)
2653                return NULL;
2654
2655        conn = cls_conn->dd_data;
2656        tcp_conn = conn->dd_data;
2657        cconn = tcp_conn->dd_data;
2658        cconn->iconn = conn;
2659
2660        log_debug(1 << CXGBI_DBG_ISCSI,
2661                "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
2662                cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn);
2663
2664        return cls_conn;
2665}
2666EXPORT_SYMBOL_GPL(cxgbi_create_conn);
2667
2668int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
2669                                struct iscsi_cls_conn *cls_conn,
2670                                u64 transport_eph, int is_leading)
2671{
2672        struct iscsi_conn *conn = cls_conn->dd_data;
2673        struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2674        struct cxgbi_conn *cconn = tcp_conn->dd_data;
2675        struct cxgbi_ppm *ppm;
2676        struct iscsi_endpoint *ep;
2677        struct cxgbi_endpoint *cep;
2678        struct cxgbi_sock *csk;
2679        int err;
2680
2681        ep = iscsi_lookup_endpoint(transport_eph);
2682        if (!ep)
2683                return -EINVAL;
2684
2685        /*  setup ddp pagesize */
2686        cep = ep->dd_data;
2687        csk = cep->csk;
2688
2689        ppm = csk->cdev->cdev2ppm(csk->cdev);
2690        err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid,
2691                                             ppm->tformat.pgsz_idx_dflt);
2692        if (err < 0)
2693                goto put_ep;
2694
2695        err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
2696        if (err) {
2697                err = -EINVAL;
2698                goto put_ep;
2699        }
2700
2701        /*  calculate the tag idx bits needed for this conn based on cmds_max */
2702        cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
2703
2704        write_lock_bh(&csk->callback_lock);
2705        csk->user_data = conn;
2706        cconn->chba = cep->chba;
2707        cconn->cep = cep;
2708        cep->cconn = cconn;
2709        write_unlock_bh(&csk->callback_lock);
2710
2711        cxgbi_conn_max_xmit_dlength(conn);
2712        cxgbi_conn_max_recv_dlength(conn);
2713
2714        log_debug(1 << CXGBI_DBG_ISCSI,
2715                "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
2716                cls_session, cls_conn, ep, cconn, csk);
2717        /*  init recv engine */
2718        iscsi_tcp_hdr_recv_prep(tcp_conn);
2719
2720put_ep:
2721        iscsi_put_endpoint(ep);
2722        return err;
2723}
2724EXPORT_SYMBOL_GPL(cxgbi_bind_conn);
2725
2726struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep,
2727                                                u16 cmds_max, u16 qdepth,
2728                                                u32 initial_cmdsn)
2729{
2730        struct cxgbi_endpoint *cep;
2731        struct cxgbi_hba *chba;
2732        struct Scsi_Host *shost;
2733        struct iscsi_cls_session *cls_session;
2734        struct iscsi_session *session;
2735
2736        if (!ep) {
2737                pr_err("missing endpoint.\n");
2738                return NULL;
2739        }
2740
2741        cep = ep->dd_data;
2742        chba = cep->chba;
2743        shost = chba->shost;
2744
2745        BUG_ON(chba != iscsi_host_priv(shost));
2746
2747        cls_session = iscsi_session_setup(chba->cdev->itp, shost,
2748                                        cmds_max, 0,
2749                                        sizeof(struct iscsi_tcp_task) +
2750                                        sizeof(struct cxgbi_task_data),
2751                                        initial_cmdsn, ISCSI_MAX_TARGET);
2752        if (!cls_session)
2753                return NULL;
2754
2755        session = cls_session->dd_data;
2756        if (iscsi_tcp_r2tpool_alloc(session))
2757                goto remove_session;
2758
2759        log_debug(1 << CXGBI_DBG_ISCSI,
2760                "ep 0x%p, cls sess 0x%p.\n", ep, cls_session);
2761        return cls_session;
2762
2763remove_session:
2764        iscsi_session_teardown(cls_session);
2765        return NULL;
2766}
2767EXPORT_SYMBOL_GPL(cxgbi_create_session);
2768
2769void cxgbi_destroy_session(struct iscsi_cls_session *cls_session)
2770{
2771        log_debug(1 << CXGBI_DBG_ISCSI,
2772                "cls sess 0x%p.\n", cls_session);
2773
2774        iscsi_tcp_r2tpool_free(cls_session->dd_data);
2775        iscsi_session_teardown(cls_session);
2776}
2777EXPORT_SYMBOL_GPL(cxgbi_destroy_session);
2778
2779int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2780                        char *buf, int buflen)
2781{
2782        struct cxgbi_hba *chba = iscsi_host_priv(shost);
2783
2784        if (!chba->ndev) {
2785                shost_printk(KERN_ERR, shost, "Could not get host param. "
2786                                "netdev for host not set.\n");
2787                return -ENODEV;
2788        }
2789
2790        log_debug(1 << CXGBI_DBG_ISCSI,
2791                "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
2792                shost, chba, chba->ndev->name, param, buflen, buf);
2793
2794        switch (param) {
2795        case ISCSI_HOST_PARAM_IPADDRESS:
2796        {
2797                __be32 addr = in_aton(buf);
2798                log_debug(1 << CXGBI_DBG_ISCSI,
2799                        "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr);
2800                cxgbi_set_iscsi_ipv4(chba, addr);
2801                return 0;
2802        }
2803        case ISCSI_HOST_PARAM_HWADDRESS:
2804        case ISCSI_HOST_PARAM_NETDEV_NAME:
2805                return 0;
2806        default:
2807                return iscsi_host_set_param(shost, param, buf, buflen);
2808        }
2809}
2810EXPORT_SYMBOL_GPL(cxgbi_set_host_param);
2811
2812int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2813                        char *buf)
2814{
2815        struct cxgbi_hba *chba = iscsi_host_priv(shost);
2816        int len = 0;
2817
2818        if (!chba->ndev) {
2819                shost_printk(KERN_ERR, shost, "Could not get host param. "
2820                                "netdev for host not set.\n");
2821                return -ENODEV;
2822        }
2823
2824        log_debug(1 << CXGBI_DBG_ISCSI,
2825                "shost 0x%p, hba 0x%p,%s, param %d.\n",
2826                shost, chba, chba->ndev->name, param);
2827
2828        switch (param) {
2829        case ISCSI_HOST_PARAM_HWADDRESS:
2830                len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6);
2831                break;
2832        case ISCSI_HOST_PARAM_NETDEV_NAME:
2833                len = sprintf(buf, "%s\n", chba->ndev->name);
2834                break;
2835        case ISCSI_HOST_PARAM_IPADDRESS:
2836        {
2837                struct cxgbi_sock *csk = find_sock_on_port(chba->cdev,
2838                                                           chba->port_id);
2839                if (csk) {
2840                        len = sprintf(buf, "%pIS",
2841                                      (struct sockaddr *)&csk->saddr);
2842                }
2843                log_debug(1 << CXGBI_DBG_ISCSI,
2844                          "hba %s, addr %s.\n", chba->ndev->name, buf);
2845                break;
2846        }
2847        default:
2848                return iscsi_host_get_param(shost, param, buf);
2849        }
2850
2851        return len;
2852}
2853EXPORT_SYMBOL_GPL(cxgbi_get_host_param);
2854
2855struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
2856                                        struct sockaddr *dst_addr,
2857                                        int non_blocking)
2858{
2859        struct iscsi_endpoint *ep;
2860        struct cxgbi_endpoint *cep;
2861        struct cxgbi_hba *hba = NULL;
2862        struct cxgbi_sock *csk;
2863        int ifindex = 0;
2864        int err = -EINVAL;
2865
2866        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2867                "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
2868                shost, non_blocking, dst_addr);
2869
2870        if (shost) {
2871                hba = iscsi_host_priv(shost);
2872                if (!hba) {
2873                        pr_info("shost 0x%p, priv NULL.\n", shost);
2874                        goto err_out;
2875                }
2876        }
2877
2878check_route:
2879        if (dst_addr->sa_family == AF_INET) {
2880                csk = cxgbi_check_route(dst_addr, ifindex);
2881#if IS_ENABLED(CONFIG_IPV6)
2882        } else if (dst_addr->sa_family == AF_INET6) {
2883                csk = cxgbi_check_route6(dst_addr, ifindex);
2884#endif
2885        } else {
2886                pr_info("address family 0x%x NOT supported.\n",
2887                        dst_addr->sa_family);
2888                err = -EAFNOSUPPORT;
2889                return (struct iscsi_endpoint *)ERR_PTR(err);
2890        }
2891
2892        if (IS_ERR(csk))
2893                return (struct iscsi_endpoint *)csk;
2894        cxgbi_sock_get(csk);
2895
2896        if (!hba)
2897                hba = csk->cdev->hbas[csk->port_id];
2898        else if (hba != csk->cdev->hbas[csk->port_id]) {
2899                if (ifindex != hba->ndev->ifindex) {
2900                        cxgbi_sock_put(csk);
2901                        cxgbi_sock_closed(csk);
2902                        ifindex = hba->ndev->ifindex;
2903                        goto check_route;
2904                }
2905
2906                pr_info("Could not connect through requested host %u"
2907                        "hba 0x%p != 0x%p (%u).\n",
2908                        shost->host_no, hba,
2909                        csk->cdev->hbas[csk->port_id], csk->port_id);
2910                err = -ENOSPC;
2911                goto release_conn;
2912        }
2913
2914        err = sock_get_port(csk);
2915        if (err)
2916                goto release_conn;
2917
2918        cxgbi_sock_set_state(csk, CTP_CONNECTING);
2919        err = csk->cdev->csk_init_act_open(csk);
2920        if (err)
2921                goto release_conn;
2922
2923        if (cxgbi_sock_is_closing(csk)) {
2924                err = -ENOSPC;
2925                pr_info("csk 0x%p is closing.\n", csk);
2926                goto release_conn;
2927        }
2928
2929        ep = iscsi_create_endpoint(sizeof(*cep));
2930        if (!ep) {
2931                err = -ENOMEM;
2932                pr_info("iscsi alloc ep, OOM.\n");
2933                goto release_conn;
2934        }
2935
2936        cep = ep->dd_data;
2937        cep->csk = csk;
2938        cep->chba = hba;
2939
2940        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2941                "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
2942                ep, cep, csk, hba, hba->ndev->name);
2943        return ep;
2944
2945release_conn:
2946        cxgbi_sock_put(csk);
2947        cxgbi_sock_closed(csk);
2948err_out:
2949        return ERR_PTR(err);
2950}
2951EXPORT_SYMBOL_GPL(cxgbi_ep_connect);
2952
2953int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
2954{
2955        struct cxgbi_endpoint *cep = ep->dd_data;
2956        struct cxgbi_sock *csk = cep->csk;
2957
2958        if (!cxgbi_sock_is_established(csk))
2959                return 0;
2960        return 1;
2961}
2962EXPORT_SYMBOL_GPL(cxgbi_ep_poll);
2963
2964void cxgbi_ep_disconnect(struct iscsi_endpoint *ep)
2965{
2966        struct cxgbi_endpoint *cep = ep->dd_data;
2967        struct cxgbi_conn *cconn = cep->cconn;
2968        struct cxgbi_sock *csk = cep->csk;
2969
2970        log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2971                "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
2972                ep, cep, cconn, csk, csk->state, csk->flags);
2973
2974        if (cconn && cconn->iconn) {
2975                write_lock_bh(&csk->callback_lock);
2976                cep->csk->user_data = NULL;
2977                cconn->cep = NULL;
2978                write_unlock_bh(&csk->callback_lock);
2979        }
2980        iscsi_destroy_endpoint(ep);
2981
2982        if (likely(csk->state >= CTP_ESTABLISHED))
2983                need_active_close(csk);
2984        else
2985                cxgbi_sock_closed(csk);
2986
2987        cxgbi_sock_put(csk);
2988}
2989EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect);
2990
2991int cxgbi_iscsi_init(struct iscsi_transport *itp,
2992                        struct scsi_transport_template **stt)
2993{
2994        *stt = iscsi_register_transport(itp);
2995        if (*stt == NULL) {
2996                pr_err("unable to register %s transport 0x%p.\n",
2997                        itp->name, itp);
2998                return -ENODEV;
2999        }
3000        log_debug(1 << CXGBI_DBG_ISCSI,
3001                "%s, registered iscsi transport 0x%p.\n",
3002                itp->name, stt);
3003        return 0;
3004}
3005EXPORT_SYMBOL_GPL(cxgbi_iscsi_init);
3006
3007void cxgbi_iscsi_cleanup(struct iscsi_transport *itp,
3008                        struct scsi_transport_template **stt)
3009{
3010        if (*stt) {
3011                log_debug(1 << CXGBI_DBG_ISCSI,
3012                        "de-register transport 0x%p, %s, stt 0x%p.\n",
3013                        itp, itp->name, *stt);
3014                *stt = NULL;
3015                iscsi_unregister_transport(itp);
3016        }
3017}
3018EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup);
3019
3020umode_t cxgbi_attr_is_visible(int param_type, int param)
3021{
3022        switch (param_type) {
3023        case ISCSI_HOST_PARAM:
3024                switch (param) {
3025                case ISCSI_HOST_PARAM_NETDEV_NAME:
3026                case ISCSI_HOST_PARAM_HWADDRESS:
3027                case ISCSI_HOST_PARAM_IPADDRESS:
3028                case ISCSI_HOST_PARAM_INITIATOR_NAME:
3029                        return S_IRUGO;
3030                default:
3031                        return 0;
3032                }
3033        case ISCSI_PARAM:
3034                switch (param) {
3035                case ISCSI_PARAM_MAX_RECV_DLENGTH:
3036                case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3037                case ISCSI_PARAM_HDRDGST_EN:
3038                case ISCSI_PARAM_DATADGST_EN:
3039                case ISCSI_PARAM_CONN_ADDRESS:
3040                case ISCSI_PARAM_CONN_PORT:
3041                case ISCSI_PARAM_EXP_STATSN:
3042                case ISCSI_PARAM_PERSISTENT_ADDRESS:
3043                case ISCSI_PARAM_PERSISTENT_PORT:
3044                case ISCSI_PARAM_PING_TMO:
3045                case ISCSI_PARAM_RECV_TMO:
3046                case ISCSI_PARAM_INITIAL_R2T_EN:
3047                case ISCSI_PARAM_MAX_R2T:
3048                case ISCSI_PARAM_IMM_DATA_EN:
3049                case ISCSI_PARAM_FIRST_BURST:
3050                case ISCSI_PARAM_MAX_BURST:
3051                case ISCSI_PARAM_PDU_INORDER_EN:
3052                case ISCSI_PARAM_DATASEQ_INORDER_EN:
3053                case ISCSI_PARAM_ERL:
3054                case ISCSI_PARAM_TARGET_NAME:
3055                case ISCSI_PARAM_TPGT:
3056                case ISCSI_PARAM_USERNAME:
3057                case ISCSI_PARAM_PASSWORD:
3058                case ISCSI_PARAM_USERNAME_IN:
3059                case ISCSI_PARAM_PASSWORD_IN:
3060                case ISCSI_PARAM_FAST_ABORT:
3061                case ISCSI_PARAM_ABORT_TMO:
3062                case ISCSI_PARAM_LU_RESET_TMO:
3063                case ISCSI_PARAM_TGT_RESET_TMO:
3064                case ISCSI_PARAM_IFACE_NAME:
3065                case ISCSI_PARAM_INITIATOR_NAME:
3066                        return S_IRUGO;
3067                default:
3068                        return 0;
3069                }
3070        }
3071
3072        return 0;
3073}
3074EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
3075
3076static int __init libcxgbi_init_module(void)
3077{
3078        pr_info("%s", version);
3079
3080        BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
3081                     sizeof(struct cxgbi_skb_cb));
3082        rsvd_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
3083        if (!rsvd_page)
3084                return -ENOMEM;
3085
3086        return 0;
3087}
3088
3089static void __exit libcxgbi_exit_module(void)
3090{
3091        cxgbi_device_unregister_all(0xFF);
3092        put_page(rsvd_page);
3093        return;
3094}
3095
3096module_init(libcxgbi_init_module);
3097module_exit(libcxgbi_exit_module);
3098