linux/drivers/scsi/aic7xxx/aic7xxx_core.c
<<
>>
Prefs
   1/*
   2 * Core routines and tables shareable across OS platforms.
   3 *
   4 * Copyright (c) 1994-2002 Justin T. Gibbs.
   5 * Copyright (c) 2000-2002 Adaptec Inc.
   6 * All rights reserved.
   7 *
   8 * Redistribution and use in source and binary forms, with or without
   9 * modification, are permitted provided that the following conditions
  10 * are met:
  11 * 1. Redistributions of source code must retain the above copyright
  12 *    notice, this list of conditions, and the following disclaimer,
  13 *    without modification.
  14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15 *    substantially similar to the "NO WARRANTY" disclaimer below
  16 *    ("Disclaimer") and any redistribution must be conditioned upon
  17 *    including a substantially similar Disclaimer requirement for further
  18 *    binary redistribution.
  19 * 3. Neither the names of the above-listed copyright holders nor the names
  20 *    of any contributors may be used to endorse or promote products derived
  21 *    from this software without specific prior written permission.
  22 *
  23 * Alternatively, this software may be distributed under the terms of the
  24 * GNU General Public License ("GPL") version 2 as published by the Free
  25 * Software Foundation.
  26 *
  27 * NO WARRANTY
  28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 * POSSIBILITY OF SUCH DAMAGES.
  39 *
  40 * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
  41 */
  42
  43#include "aic7xxx_osm.h"
  44#include "aic7xxx_inline.h"
  45#include "aicasm/aicasm_insformat.h"
  46
  47/***************************** Lookup Tables **********************************/
  48static const char *const ahc_chip_names[] = {
  49        "NONE",
  50        "aic7770",
  51        "aic7850",
  52        "aic7855",
  53        "aic7859",
  54        "aic7860",
  55        "aic7870",
  56        "aic7880",
  57        "aic7895",
  58        "aic7895C",
  59        "aic7890/91",
  60        "aic7896/97",
  61        "aic7892",
  62        "aic7899"
  63};
  64
  65/*
  66 * Hardware error codes.
  67 */
  68struct ahc_hard_error_entry {
  69        uint8_t errno;
  70        const char *errmesg;
  71};
  72
  73static const struct ahc_hard_error_entry ahc_hard_errors[] = {
  74        { ILLHADDR,     "Illegal Host Access" },
  75        { ILLSADDR,     "Illegal Sequencer Address referenced" },
  76        { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  77        { SQPARERR,     "Sequencer Parity Error" },
  78        { DPARERR,      "Data-path Parity Error" },
  79        { MPARERR,      "Scratch or SCB Memory Parity Error" },
  80        { PCIERRSTAT,   "PCI Error detected" },
  81        { CIOPARERR,    "CIOBUS Parity Error" },
  82};
  83static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
  84
  85static const struct ahc_phase_table_entry ahc_phase_table[] =
  86{
  87        { P_DATAOUT,    NOP,                    "in Data-out phase"     },
  88        { P_DATAIN,     INITIATOR_ERROR,        "in Data-in phase"      },
  89        { P_DATAOUT_DT, NOP,                    "in DT Data-out phase"  },
  90        { P_DATAIN_DT,  INITIATOR_ERROR,        "in DT Data-in phase"   },
  91        { P_COMMAND,    NOP,                    "in Command phase"      },
  92        { P_MESGOUT,    NOP,                    "in Message-out phase"  },
  93        { P_STATUS,     INITIATOR_ERROR,        "in Status phase"       },
  94        { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  95        { P_BUSFREE,    NOP,                    "while idle"            },
  96        { 0,            NOP,                    "in unknown phase"      }
  97};
  98
  99/*
 100 * In most cases we only wish to itterate over real phases, so
 101 * exclude the last element from the count.
 102 */
 103static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
 104
 105/*
 106 * Valid SCSIRATE values.  (p. 3-17)
 107 * Provides a mapping of tranfer periods in ns to the proper value to
 108 * stick in the scsixfer reg.
 109 */
 110static const struct ahc_syncrate ahc_syncrates[] =
 111{
 112      /* ultra2    fast/ultra  period     rate */
 113        { 0x42,      0x000,      9,      "80.0" },
 114        { 0x03,      0x000,     10,      "40.0" },
 115        { 0x04,      0x000,     11,      "33.0" },
 116        { 0x05,      0x100,     12,      "20.0" },
 117        { 0x06,      0x110,     15,      "16.0" },
 118        { 0x07,      0x120,     18,      "13.4" },
 119        { 0x08,      0x000,     25,      "10.0" },
 120        { 0x19,      0x010,     31,      "8.0"  },
 121        { 0x1a,      0x020,     37,      "6.67" },
 122        { 0x1b,      0x030,     43,      "5.7"  },
 123        { 0x1c,      0x040,     50,      "5.0"  },
 124        { 0x00,      0x050,     56,      "4.4"  },
 125        { 0x00,      0x060,     62,      "4.0"  },
 126        { 0x00,      0x070,     68,      "3.6"  },
 127        { 0x00,      0x000,      0,      NULL   }
 128};
 129
 130/* Our Sequencer Program */
 131#include "aic7xxx_seq.h"
 132
 133/**************************** Function Declarations ***************************/
 134static void             ahc_force_renegotiation(struct ahc_softc *ahc,
 135                                                struct ahc_devinfo *devinfo);
 136static struct ahc_tmode_tstate*
 137                        ahc_alloc_tstate(struct ahc_softc *ahc,
 138                                         u_int scsi_id, char channel);
 139#ifdef AHC_TARGET_MODE
 140static void             ahc_free_tstate(struct ahc_softc *ahc,
 141                                        u_int scsi_id, char channel, int force);
 142#endif
 143static const struct ahc_syncrate*
 144                        ahc_devlimited_syncrate(struct ahc_softc *ahc,
 145                                                struct ahc_initiator_tinfo *,
 146                                                u_int *period,
 147                                                u_int *ppr_options,
 148                                                role_t role);
 149static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
 150static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
 151                                          struct ahc_devinfo *devinfo);
 152static void             ahc_scb_devinfo(struct ahc_softc *ahc,
 153                                        struct ahc_devinfo *devinfo,
 154                                        struct scb *scb);
 155static void             ahc_assert_atn(struct ahc_softc *ahc);
 156static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 157                                                   struct ahc_devinfo *devinfo,
 158                                                   struct scb *scb);
 159static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
 160                                               struct ahc_devinfo *devinfo);
 161static void             ahc_construct_sdtr(struct ahc_softc *ahc,
 162                                           struct ahc_devinfo *devinfo,
 163                                           u_int period, u_int offset);
 164static void             ahc_construct_wdtr(struct ahc_softc *ahc,
 165                                           struct ahc_devinfo *devinfo,
 166                                           u_int bus_width);
 167static void             ahc_construct_ppr(struct ahc_softc *ahc,
 168                                          struct ahc_devinfo *devinfo,
 169                                          u_int period, u_int offset,
 170                                          u_int bus_width, u_int ppr_options);
 171static void             ahc_clear_msg_state(struct ahc_softc *ahc);
 172static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
 173static void             ahc_handle_message_phase(struct ahc_softc *ahc);
 174typedef enum {
 175        AHCMSG_1B,
 176        AHCMSG_2B,
 177        AHCMSG_EXT
 178} ahc_msgtype;
 179static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 180                                     u_int msgval, int full);
 181static int              ahc_parse_msg(struct ahc_softc *ahc,
 182                                      struct ahc_devinfo *devinfo);
 183static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
 184                                              struct ahc_devinfo *devinfo);
 185static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 186                                                struct ahc_devinfo *devinfo);
 187static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 188static void             ahc_handle_devreset(struct ahc_softc *ahc,
 189                                            struct ahc_devinfo *devinfo,
 190                                            cam_status status, char *message,
 191                                            int verbose_level);
 192#ifdef AHC_TARGET_MODE
 193static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
 194                                               struct ahc_devinfo *devinfo,
 195                                               struct scb *scb);
 196#endif
 197
 198static bus_dmamap_callback_t    ahc_dmamap_cb;
 199static void             ahc_build_free_scb_list(struct ahc_softc *ahc);
 200static int              ahc_init_scbdata(struct ahc_softc *ahc);
 201static void             ahc_fini_scbdata(struct ahc_softc *ahc);
 202static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
 203                                            struct scb *prev_scb,
 204                                            struct scb *scb);
 205static int              ahc_qinfifo_count(struct ahc_softc *ahc);
 206static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 207                                                   u_int prev, u_int scbptr);
 208static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 209static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
 210                                     u_int scbpos, u_int prev);
 211static void             ahc_reset_current_bus(struct ahc_softc *ahc);
 212#ifdef AHC_DUMP_SEQ
 213static void             ahc_dumpseq(struct ahc_softc *ahc);
 214#endif
 215static int              ahc_loadseq(struct ahc_softc *ahc);
 216static int              ahc_check_patch(struct ahc_softc *ahc,
 217                                        const struct patch **start_patch,
 218                                        u_int start_instr, u_int *skip_addr);
 219static void             ahc_download_instr(struct ahc_softc *ahc,
 220                                           u_int instrptr, uint8_t *dconsts);
 221#ifdef AHC_TARGET_MODE
 222static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
 223                                               struct ahc_tmode_lstate *lstate,
 224                                               u_int initiator_id,
 225                                               u_int event_type,
 226                                               u_int event_arg);
 227static void             ahc_update_scsiid(struct ahc_softc *ahc,
 228                                          u_int targid_mask);
 229static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
 230                                              struct target_cmd *cmd);
 231#endif
 232
 233static u_int            ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl);
 234static void             ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl);
 235static void             ahc_busy_tcl(struct ahc_softc *ahc,
 236                                     u_int tcl, u_int busyid);
 237
 238/************************** SCB and SCB queue management **********************/
 239static void             ahc_run_untagged_queues(struct ahc_softc *ahc);
 240static void             ahc_run_untagged_queue(struct ahc_softc *ahc,
 241                                               struct scb_tailq *queue);
 242
 243/****************************** Initialization ********************************/
 244static void              ahc_alloc_scbs(struct ahc_softc *ahc);
 245static void              ahc_shutdown(void *arg);
 246
 247/*************************** Interrupt Services *******************************/
 248static void             ahc_clear_intstat(struct ahc_softc *ahc);
 249static void             ahc_run_qoutfifo(struct ahc_softc *ahc);
 250#ifdef AHC_TARGET_MODE
 251static void             ahc_run_tqinfifo(struct ahc_softc *ahc, int paused);
 252#endif
 253static void             ahc_handle_brkadrint(struct ahc_softc *ahc);
 254static void             ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat);
 255static void             ahc_handle_scsiint(struct ahc_softc *ahc,
 256                                           u_int intstat);
 257static void             ahc_clear_critical_section(struct ahc_softc *ahc);
 258
 259/***************************** Error Recovery *********************************/
 260static void             ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb);
 261static int              ahc_abort_scbs(struct ahc_softc *ahc, int target,
 262                                       char channel, int lun, u_int tag,
 263                                       role_t role, uint32_t status);
 264static void             ahc_calc_residual(struct ahc_softc *ahc,
 265                                          struct scb *scb);
 266
 267/*********************** Untagged Transaction Routines ************************/
 268static inline void      ahc_freeze_untagged_queues(struct ahc_softc *ahc);
 269static inline void      ahc_release_untagged_queues(struct ahc_softc *ahc);
 270
 271/*
 272 * Block our completion routine from starting the next untagged
 273 * transaction for this target or target lun.
 274 */
 275static inline void
 276ahc_freeze_untagged_queues(struct ahc_softc *ahc)
 277{
 278        if ((ahc->flags & AHC_SCB_BTT) == 0)
 279                ahc->untagged_queue_lock++;
 280}
 281
 282/*
 283 * Allow the next untagged transaction for this target or target lun
 284 * to be executed.  We use a counting semaphore to allow the lock
 285 * to be acquired recursively.  Once the count drops to zero, the
 286 * transaction queues will be run.
 287 */
 288static inline void
 289ahc_release_untagged_queues(struct ahc_softc *ahc)
 290{
 291        if ((ahc->flags & AHC_SCB_BTT) == 0) {
 292                ahc->untagged_queue_lock--;
 293                if (ahc->untagged_queue_lock == 0)
 294                        ahc_run_untagged_queues(ahc);
 295        }
 296}
 297
 298/************************* Sequencer Execution Control ************************/
 299/*
 300 * Work around any chip bugs related to halting sequencer execution.
 301 * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
 302 * reading a register that will set this signal and deassert it.
 303 * Without this workaround, if the chip is paused, by an interrupt or
 304 * manual pause while accessing scb ram, accesses to certain registers
 305 * will hang the system (infinite pci retries).
 306 */
 307static void
 308ahc_pause_bug_fix(struct ahc_softc *ahc)
 309{
 310        if ((ahc->features & AHC_ULTRA2) != 0)
 311                (void)ahc_inb(ahc, CCSCBCTL);
 312}
 313
 314/*
 315 * Determine whether the sequencer has halted code execution.
 316 * Returns non-zero status if the sequencer is stopped.
 317 */
 318int
 319ahc_is_paused(struct ahc_softc *ahc)
 320{
 321        return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
 322}
 323
 324/*
 325 * Request that the sequencer stop and wait, indefinitely, for it
 326 * to stop.  The sequencer will only acknowledge that it is paused
 327 * once it has reached an instruction boundary and PAUSEDIS is
 328 * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 329 * for critical sections.
 330 */
 331void
 332ahc_pause(struct ahc_softc *ahc)
 333{
 334        ahc_outb(ahc, HCNTRL, ahc->pause);
 335
 336        /*
 337         * Since the sequencer can disable pausing in a critical section, we
 338         * must loop until it actually stops.
 339         */
 340        while (ahc_is_paused(ahc) == 0)
 341                ;
 342
 343        ahc_pause_bug_fix(ahc);
 344}
 345
 346/*
 347 * Allow the sequencer to continue program execution.
 348 * We check here to ensure that no additional interrupt
 349 * sources that would cause the sequencer to halt have been
 350 * asserted.  If, for example, a SCSI bus reset is detected
 351 * while we are fielding a different, pausing, interrupt type,
 352 * we don't want to release the sequencer before going back
 353 * into our interrupt handler and dealing with this new
 354 * condition.
 355 */
 356void
 357ahc_unpause(struct ahc_softc *ahc)
 358{
 359        if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
 360                ahc_outb(ahc, HCNTRL, ahc->unpause);
 361}
 362
 363/************************** Memory mapping routines ***************************/
 364static struct ahc_dma_seg *
 365ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
 366{
 367        int sg_index;
 368
 369        sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
 370        /* sg_list_phys points to entry 1, not 0 */
 371        sg_index++;
 372
 373        return (&scb->sg_list[sg_index]);
 374}
 375
 376static uint32_t
 377ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
 378{
 379        int sg_index;
 380
 381        /* sg_list_phys points to entry 1, not 0 */
 382        sg_index = sg - &scb->sg_list[1];
 383
 384        return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
 385}
 386
 387static uint32_t
 388ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
 389{
 390        return (ahc->scb_data->hscb_busaddr
 391                + (sizeof(struct hardware_scb) * index));
 392}
 393
 394static void
 395ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
 396{
 397        ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
 398                        ahc->scb_data->hscb_dmamap,
 399                        /*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
 400                        /*len*/sizeof(*scb->hscb), op);
 401}
 402
 403void
 404ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
 405{
 406        if (scb->sg_count == 0)
 407                return;
 408
 409        ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
 410                        /*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
 411                                * sizeof(struct ahc_dma_seg),
 412                        /*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
 413}
 414
 415#ifdef AHC_TARGET_MODE
 416static uint32_t
 417ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
 418{
 419        return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
 420}
 421#endif
 422
 423/*********************** Miscellaneous Support Functions ***********************/
 424/*
 425 * Determine whether the sequencer reported a residual
 426 * for this SCB/transaction.
 427 */
 428static void
 429ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
 430{
 431        uint32_t sgptr;
 432
 433        sgptr = ahc_le32toh(scb->hscb->sgptr);
 434        if ((sgptr & SG_RESID_VALID) != 0)
 435                ahc_calc_residual(ahc, scb);
 436}
 437
 438/*
 439 * Return pointers to the transfer negotiation information
 440 * for the specified our_id/remote_id pair.
 441 */
 442struct ahc_initiator_tinfo *
 443ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
 444                    u_int remote_id, struct ahc_tmode_tstate **tstate)
 445{
 446        /*
 447         * Transfer data structures are stored from the perspective
 448         * of the target role.  Since the parameters for a connection
 449         * in the initiator role to a given target are the same as
 450         * when the roles are reversed, we pretend we are the target.
 451         */
 452        if (channel == 'B')
 453                our_id += 8;
 454        *tstate = ahc->enabled_targets[our_id];
 455        return (&(*tstate)->transinfo[remote_id]);
 456}
 457
 458uint16_t
 459ahc_inw(struct ahc_softc *ahc, u_int port)
 460{
 461        uint16_t r = ahc_inb(ahc, port+1) << 8;
 462        return r | ahc_inb(ahc, port);
 463}
 464
 465void
 466ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
 467{
 468        ahc_outb(ahc, port, value & 0xFF);
 469        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 470}
 471
 472uint32_t
 473ahc_inl(struct ahc_softc *ahc, u_int port)
 474{
 475        return ((ahc_inb(ahc, port))
 476              | (ahc_inb(ahc, port+1) << 8)
 477              | (ahc_inb(ahc, port+2) << 16)
 478              | (ahc_inb(ahc, port+3) << 24));
 479}
 480
 481void
 482ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
 483{
 484        ahc_outb(ahc, port, (value) & 0xFF);
 485        ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
 486        ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
 487        ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
 488}
 489
 490uint64_t
 491ahc_inq(struct ahc_softc *ahc, u_int port)
 492{
 493        return ((ahc_inb(ahc, port))
 494              | (ahc_inb(ahc, port+1) << 8)
 495              | (ahc_inb(ahc, port+2) << 16)
 496              | (((uint64_t)ahc_inb(ahc, port+3)) << 24)
 497              | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
 498              | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
 499              | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
 500              | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
 501}
 502
 503void
 504ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
 505{
 506        ahc_outb(ahc, port, value & 0xFF);
 507        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 508        ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
 509        ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
 510        ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
 511        ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
 512        ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
 513        ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
 514}
 515
 516/*
 517 * Get a free scb. If there are none, see if we can allocate a new SCB.
 518 */
 519struct scb *
 520ahc_get_scb(struct ahc_softc *ahc)
 521{
 522        struct scb *scb;
 523
 524        if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
 525                ahc_alloc_scbs(ahc);
 526                scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
 527                if (scb == NULL)
 528                        return (NULL);
 529        }
 530        SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
 531        return (scb);
 532}
 533
 534/*
 535 * Return an SCB resource to the free list.
 536 */
 537void
 538ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
 539{
 540        struct hardware_scb *hscb;
 541
 542        hscb = scb->hscb;
 543        /* Clean up for the next user */
 544        ahc->scb_data->scbindex[hscb->tag] = NULL;
 545        scb->flags = SCB_FREE;
 546        hscb->control = 0;
 547
 548        SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);
 549
 550        /* Notify the OSM that a resource is now available. */
 551        ahc_platform_scb_free(ahc, scb);
 552}
 553
 554struct scb *
 555ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
 556{
 557        struct scb* scb;
 558
 559        scb = ahc->scb_data->scbindex[tag];
 560        if (scb != NULL)
 561                ahc_sync_scb(ahc, scb,
 562                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 563        return (scb);
 564}
 565
 566static void
 567ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
 568{
 569        struct hardware_scb *q_hscb;
 570        u_int  saved_tag;
 571
 572        /*
 573         * Our queuing method is a bit tricky.  The card
 574         * knows in advance which HSCB to download, and we
 575         * can't disappoint it.  To achieve this, the next
 576         * SCB to download is saved off in ahc->next_queued_scb.
 577         * When we are called to queue "an arbitrary scb",
 578         * we copy the contents of the incoming HSCB to the one
 579         * the sequencer knows about, swap HSCB pointers and
 580         * finally assign the SCB to the tag indexed location
 581         * in the scb_array.  This makes sure that we can still
 582         * locate the correct SCB by SCB_TAG.
 583         */
 584        q_hscb = ahc->next_queued_scb->hscb;
 585        saved_tag = q_hscb->tag;
 586        memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 587        if ((scb->flags & SCB_CDB32_PTR) != 0) {
 588                q_hscb->shared_data.cdb_ptr =
 589                    ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
 590                              + offsetof(struct hardware_scb, cdb32));
 591        }
 592        q_hscb->tag = saved_tag;
 593        q_hscb->next = scb->hscb->tag;
 594
 595        /* Now swap HSCB pointers. */
 596        ahc->next_queued_scb->hscb = scb->hscb;
 597        scb->hscb = q_hscb;
 598
 599        /* Now define the mapping from tag to SCB in the scbindex */
 600        ahc->scb_data->scbindex[scb->hscb->tag] = scb;
 601}
 602
 603/*
 604 * Tell the sequencer about a new transaction to execute.
 605 */
 606void
 607ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
 608{
 609        ahc_swap_with_next_hscb(ahc, scb);
 610
 611        if (scb->hscb->tag == SCB_LIST_NULL
 612         || scb->hscb->next == SCB_LIST_NULL)
 613                panic("Attempt to queue invalid SCB tag %x:%x\n",
 614                      scb->hscb->tag, scb->hscb->next);
 615
 616        /*
 617         * Setup data "oddness".
 618         */
 619        scb->hscb->lun &= LID;
 620        if (ahc_get_transfer_length(scb) & 0x1)
 621                scb->hscb->lun |= SCB_XFERLEN_ODD;
 622
 623        /*
 624         * Keep a history of SCBs we've downloaded in the qinfifo.
 625         */
 626        ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
 627
 628        /*
 629         * Make sure our data is consistent from the
 630         * perspective of the adapter.
 631         */
 632        ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 633
 634        /* Tell the adapter about the newly queued SCB */
 635        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
 636                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
 637        } else {
 638                if ((ahc->features & AHC_AUTOPAUSE) == 0)
 639                        ahc_pause(ahc);
 640                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
 641                if ((ahc->features & AHC_AUTOPAUSE) == 0)
 642                        ahc_unpause(ahc);
 643        }
 644}
 645
 646struct scsi_sense_data *
 647ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
 648{
 649        int offset;
 650
 651        offset = scb - ahc->scb_data->scbarray;
 652        return (&ahc->scb_data->sense[offset]);
 653}
 654
 655static uint32_t
 656ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
 657{
 658        int offset;
 659
 660        offset = scb - ahc->scb_data->scbarray;
 661        return (ahc->scb_data->sense_busaddr
 662              + (offset * sizeof(struct scsi_sense_data)));
 663}
 664
 665/************************** Interrupt Processing ******************************/
 666static void
 667ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
 668{
 669        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 670                        /*offset*/0, /*len*/256, op);
 671}
 672
 673static void
 674ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
 675{
 676#ifdef AHC_TARGET_MODE
 677        if ((ahc->flags & AHC_TARGETROLE) != 0) {
 678                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 679                                ahc->shared_data_dmamap,
 680                                ahc_targetcmd_offset(ahc, 0),
 681                                sizeof(struct target_cmd) * AHC_TMODE_CMDS,
 682                                op);
 683        }
 684#endif
 685}
 686
 687/*
 688 * See if the firmware has posted any completed commands
 689 * into our in-core command complete fifos.
 690 */
 691#define AHC_RUN_QOUTFIFO 0x1
 692#define AHC_RUN_TQINFIFO 0x2
 693static u_int
 694ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
 695{
 696        u_int retval;
 697
 698        retval = 0;
 699        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 700                        /*offset*/ahc->qoutfifonext, /*len*/1,
 701                        BUS_DMASYNC_POSTREAD);
 702        if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
 703                retval |= AHC_RUN_QOUTFIFO;
 704#ifdef AHC_TARGET_MODE
 705        if ((ahc->flags & AHC_TARGETROLE) != 0
 706         && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
 707                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 708                                ahc->shared_data_dmamap,
 709                                ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
 710                                /*len*/sizeof(struct target_cmd),
 711                                BUS_DMASYNC_POSTREAD);
 712                if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
 713                        retval |= AHC_RUN_TQINFIFO;
 714        }
 715#endif
 716        return (retval);
 717}
 718
 719/*
 720 * Catch an interrupt from the adapter
 721 */
 722int
 723ahc_intr(struct ahc_softc *ahc)
 724{
 725        u_int   intstat;
 726
 727        if ((ahc->pause & INTEN) == 0) {
 728                /*
 729                 * Our interrupt is not enabled on the chip
 730                 * and may be disabled for re-entrancy reasons,
 731                 * so just return.  This is likely just a shared
 732                 * interrupt.
 733                 */
 734                return (0);
 735        }
 736        /*
 737         * Instead of directly reading the interrupt status register,
 738         * infer the cause of the interrupt by checking our in-core
 739         * completion queues.  This avoids a costly PCI bus read in
 740         * most cases.
 741         */
 742        if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
 743         && (ahc_check_cmdcmpltqueues(ahc) != 0))
 744                intstat = CMDCMPLT;
 745        else {
 746                intstat = ahc_inb(ahc, INTSTAT);
 747        }
 748
 749        if ((intstat & INT_PEND) == 0) {
 750#if AHC_PCI_CONFIG > 0
 751                if (ahc->unsolicited_ints > 500) {
 752                        ahc->unsolicited_ints = 0;
 753                        if ((ahc->chip & AHC_PCI) != 0
 754                         && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
 755                                ahc->bus_intr(ahc);
 756                }
 757#endif
 758                ahc->unsolicited_ints++;
 759                return (0);
 760        }
 761        ahc->unsolicited_ints = 0;
 762
 763        if (intstat & CMDCMPLT) {
 764                ahc_outb(ahc, CLRINT, CLRCMDINT);
 765
 766                /*
 767                 * Ensure that the chip sees that we've cleared
 768                 * this interrupt before we walk the output fifo.
 769                 * Otherwise, we may, due to posted bus writes,
 770                 * clear the interrupt after we finish the scan,
 771                 * and after the sequencer has added new entries
 772                 * and asserted the interrupt again.
 773                 */
 774                ahc_flush_device_writes(ahc);
 775                ahc_run_qoutfifo(ahc);
 776#ifdef AHC_TARGET_MODE
 777                if ((ahc->flags & AHC_TARGETROLE) != 0)
 778                        ahc_run_tqinfifo(ahc, /*paused*/FALSE);
 779#endif
 780        }
 781
 782        /*
 783         * Handle statuses that may invalidate our cached
 784         * copy of INTSTAT separately.
 785         */
 786        if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
 787                /* Hot eject.  Do nothing */
 788        } else if (intstat & BRKADRINT) {
 789                ahc_handle_brkadrint(ahc);
 790        } else if ((intstat & (SEQINT|SCSIINT)) != 0) {
 791
 792                ahc_pause_bug_fix(ahc);
 793
 794                if ((intstat & SEQINT) != 0)
 795                        ahc_handle_seqint(ahc, intstat);
 796
 797                if ((intstat & SCSIINT) != 0)
 798                        ahc_handle_scsiint(ahc, intstat);
 799        }
 800        return (1);
 801}
 802
 803/************************* Sequencer Execution Control ************************/
 804/*
 805 * Restart the sequencer program from address zero
 806 */
 807static void
 808ahc_restart(struct ahc_softc *ahc)
 809{
 810        uint8_t sblkctl;
 811
 812        ahc_pause(ahc);
 813
 814        /* No more pending messages. */
 815        ahc_clear_msg_state(ahc);
 816
 817        ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
 818        ahc_outb(ahc, MSG_OUT, NOP);    /* No message to send */
 819        ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 820        ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 821        ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 822        ahc_outb(ahc, SAVED_LUN, 0xFF);
 823
 824        /*
 825         * Ensure that the sequencer's idea of TQINPOS
 826         * matches our own.  The sequencer increments TQINPOS
 827         * only after it sees a DMA complete and a reset could
 828         * occur before the increment leaving the kernel to believe
 829         * the command arrived but the sequencer to not.
 830         */
 831        ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 832
 833        /* Always allow reselection */
 834        ahc_outb(ahc, SCSISEQ,
 835                 ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 836        if ((ahc->features & AHC_CMD_CHAN) != 0) {
 837                /* Ensure that no DMA operations are in progress */
 838                ahc_outb(ahc, CCSCBCNT, 0);
 839                ahc_outb(ahc, CCSGCTL, 0);
 840                ahc_outb(ahc, CCSCBCTL, 0);
 841        }
 842        /*
 843         * If we were in the process of DMA'ing SCB data into
 844         * an SCB, replace that SCB on the free list.  This prevents
 845         * an SCB leak.
 846         */
 847        if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 848                ahc_add_curscb_to_free_list(ahc);
 849                ahc_outb(ahc, SEQ_FLAGS2,
 850                         ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 851        }
 852
 853        /*
 854         * Clear any pending sequencer interrupt.  It is no
 855         * longer relevant since we're resetting the Program
 856         * Counter.
 857         */
 858        ahc_outb(ahc, CLRINT, CLRSEQINT);
 859
 860        ahc_outb(ahc, MWI_RESIDUAL, 0);
 861        ahc_outb(ahc, SEQCTL, ahc->seqctl);
 862        ahc_outb(ahc, SEQADDR0, 0);
 863        ahc_outb(ahc, SEQADDR1, 0);
 864
 865        /*
 866         * Take the LED out of diagnostic mode on PM resume, too
 867         */
 868        sblkctl = ahc_inb(ahc, SBLKCTL);
 869        ahc_outb(ahc, SBLKCTL, (sblkctl & ~(DIAGLEDEN|DIAGLEDON)));
 870
 871        ahc_unpause(ahc);
 872}
 873
 874/************************* Input/Output Queues ********************************/
 875static void
 876ahc_run_qoutfifo(struct ahc_softc *ahc)
 877{
 878        struct scb *scb;
 879        u_int  scb_index;
 880
 881        ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 882        while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 883
 884                scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 885                if ((ahc->qoutfifonext & 0x03) == 0x03) {
 886                        u_int modnext;
 887
 888                        /*
 889                         * Clear 32bits of QOUTFIFO at a time
 890                         * so that we don't clobber an incoming
 891                         * byte DMA to the array on architectures
 892                         * that only support 32bit load and store
 893                         * operations.
 894                         */
 895                        modnext = ahc->qoutfifonext & ~0x3;
 896                        *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 897                        ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 898                                        ahc->shared_data_dmamap,
 899                                        /*offset*/modnext, /*len*/4,
 900                                        BUS_DMASYNC_PREREAD);
 901                }
 902                ahc->qoutfifonext++;
 903
 904                scb = ahc_lookup_scb(ahc, scb_index);
 905                if (scb == NULL) {
 906                        printk("%s: WARNING no command for scb %d "
 907                               "(cmdcmplt)\nQOUTPOS = %d\n",
 908                               ahc_name(ahc), scb_index,
 909                               (ahc->qoutfifonext - 1) & 0xFF);
 910                        continue;
 911                }
 912
 913                /*
 914                 * Save off the residual
 915                 * if there is one.
 916                 */
 917                ahc_update_residual(ahc, scb);
 918                ahc_done(ahc, scb);
 919        }
 920}
 921
 922static void
 923ahc_run_untagged_queues(struct ahc_softc *ahc)
 924{
 925        int i;
 926
 927        for (i = 0; i < 16; i++)
 928                ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 929}
 930
 931static void
 932ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 933{
 934        struct scb *scb;
 935
 936        if (ahc->untagged_queue_lock != 0)
 937                return;
 938
 939        if ((scb = TAILQ_FIRST(queue)) != NULL
 940         && (scb->flags & SCB_ACTIVE) == 0) {
 941                scb->flags |= SCB_ACTIVE;
 942                ahc_queue_scb(ahc, scb);
 943        }
 944}
 945
 946/************************* Interrupt Handling *********************************/
 947static void
 948ahc_handle_brkadrint(struct ahc_softc *ahc)
 949{
 950        /*
 951         * We upset the sequencer :-(
 952         * Lookup the error message
 953         */
 954        int i;
 955        int error;
 956
 957        error = ahc_inb(ahc, ERROR);
 958        for (i = 0; error != 1 && i < num_errors; i++)
 959                error >>= 1;
 960        printk("%s: brkadrint, %s at seqaddr = 0x%x\n",
 961               ahc_name(ahc), ahc_hard_errors[i].errmesg,
 962               ahc_inb(ahc, SEQADDR0) |
 963               (ahc_inb(ahc, SEQADDR1) << 8));
 964
 965        ahc_dump_card_state(ahc);
 966
 967        /* Tell everyone that this HBA is no longer available */
 968        ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 969                       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 970                       CAM_NO_HBA);
 971
 972        /* Disable all interrupt sources by resetting the controller */
 973        ahc_shutdown(ahc);
 974}
 975
 976static void
 977ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 978{
 979        struct scb *scb;
 980        struct ahc_devinfo devinfo;
 981
 982        ahc_fetch_devinfo(ahc, &devinfo);
 983
 984        /*
 985         * Clear the upper byte that holds SEQINT status
 986         * codes and clear the SEQINT bit. We will unpause
 987         * the sequencer, if appropriate, after servicing
 988         * the request.
 989         */
 990        ahc_outb(ahc, CLRINT, CLRSEQINT);
 991        switch (intstat & SEQINT_MASK) {
 992        case BAD_STATUS:
 993        {
 994                u_int  scb_index;
 995                struct hardware_scb *hscb;
 996
 997                /*
 998                 * Set the default return value to 0 (don't
 999                 * send sense).  The sense code will change
1000                 * this if needed.
1001                 */
1002                ahc_outb(ahc, RETURN_1, 0);
1003
1004                /*
1005                 * The sequencer will notify us when a command
1006                 * has an error that would be of interest to
1007                 * the kernel.  This allows us to leave the sequencer
1008                 * running in the common case of command completes
1009                 * without error.  The sequencer will already have
1010                 * dma'd the SCB back up to us, so we can reference
1011                 * the in kernel copy directly.
1012                 */
1013                scb_index = ahc_inb(ahc, SCB_TAG);
1014                scb = ahc_lookup_scb(ahc, scb_index);
1015                if (scb == NULL) {
1016                        ahc_print_devinfo(ahc, &devinfo);
1017                        printk("ahc_intr - referenced scb "
1018                               "not valid during seqint 0x%x scb(%d)\n",
1019                               intstat, scb_index);
1020                        ahc_dump_card_state(ahc);
1021                        panic("for safety");
1022                        goto unpause;
1023                }
1024
1025                hscb = scb->hscb;
1026
1027                /* Don't want to clobber the original sense code */
1028                if ((scb->flags & SCB_SENSE) != 0) {
1029                        /*
1030                         * Clear the SCB_SENSE Flag and have
1031                         * the sequencer do a normal command
1032                         * complete.
1033                         */
1034                        scb->flags &= ~SCB_SENSE;
1035                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1036                        break;
1037                }
1038                ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
1039                /* Freeze the queue until the client sees the error. */
1040                ahc_freeze_devq(ahc, scb);
1041                ahc_freeze_scb(scb);
1042                ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
1043                switch (hscb->shared_data.status.scsi_status) {
1044                case SAM_STAT_GOOD:
1045                        printk("%s: Interrupted for status of 0???\n",
1046                               ahc_name(ahc));
1047                        break;
1048                case SAM_STAT_COMMAND_TERMINATED:
1049                case SAM_STAT_CHECK_CONDITION:
1050                {
1051                        struct ahc_dma_seg *sg;
1052                        struct scsi_sense *sc;
1053                        struct ahc_initiator_tinfo *targ_info;
1054                        struct ahc_tmode_tstate *tstate;
1055                        struct ahc_transinfo *tinfo;
1056#ifdef AHC_DEBUG
1057                        if (ahc_debug & AHC_SHOW_SENSE) {
1058                                ahc_print_path(ahc, scb);
1059                                printk("SCB %d: requests Check Status\n",
1060                                       scb->hscb->tag);
1061                        }
1062#endif
1063
1064                        if (ahc_perform_autosense(scb) == 0)
1065                                break;
1066
1067                        targ_info = ahc_fetch_transinfo(ahc,
1068                                                        devinfo.channel,
1069                                                        devinfo.our_scsiid,
1070                                                        devinfo.target,
1071                                                        &tstate);
1072                        tinfo = &targ_info->curr;
1073                        sg = scb->sg_list;
1074                        sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
1075                        /*
1076                         * Save off the residual if there is one.
1077                         */
1078                        ahc_update_residual(ahc, scb);
1079#ifdef AHC_DEBUG
1080                        if (ahc_debug & AHC_SHOW_SENSE) {
1081                                ahc_print_path(ahc, scb);
1082                                printk("Sending Sense\n");
1083                        }
1084#endif
1085                        sg->addr = ahc_get_sense_bufaddr(ahc, scb);
1086                        sg->len = ahc_get_sense_bufsize(ahc, scb);
1087                        sg->len |= AHC_DMA_LAST_SEG;
1088
1089                        /* Fixup byte order */
1090                        sg->addr = ahc_htole32(sg->addr);
1091                        sg->len = ahc_htole32(sg->len);
1092
1093                        sc->opcode = REQUEST_SENSE;
1094                        sc->byte2 = 0;
1095                        if (tinfo->protocol_version <= SCSI_REV_2
1096                         && SCB_GET_LUN(scb) < 8)
1097                                sc->byte2 = SCB_GET_LUN(scb) << 5;
1098                        sc->unused[0] = 0;
1099                        sc->unused[1] = 0;
1100                        sc->length = sg->len;
1101                        sc->control = 0;
1102
1103                        /*
1104                         * We can't allow the target to disconnect.
1105                         * This will be an untagged transaction and
1106                         * having the target disconnect will make this
1107                         * transaction indestinguishable from outstanding
1108                         * tagged transactions.
1109                         */
1110                        hscb->control = 0;
1111
1112                        /*
1113                         * This request sense could be because the
1114                         * the device lost power or in some other
1115                         * way has lost our transfer negotiations.
1116                         * Renegotiate if appropriate.  Unit attention
1117                         * errors will be reported before any data
1118                         * phases occur.
1119                         */
1120                        if (ahc_get_residual(scb)
1121                            == ahc_get_transfer_length(scb)) {
1122                                ahc_update_neg_request(ahc, &devinfo,
1123                                                       tstate, targ_info,
1124                                                       AHC_NEG_IF_NON_ASYNC);
1125                        }
1126                        if (tstate->auto_negotiate & devinfo.target_mask) {
1127                                hscb->control |= MK_MESSAGE;
1128                                scb->flags &= ~SCB_NEGOTIATE;
1129                                scb->flags |= SCB_AUTO_NEGOTIATE;
1130                        }
1131                        hscb->cdb_len = sizeof(*sc);
1132                        hscb->dataptr = sg->addr;
1133                        hscb->datacnt = sg->len;
1134                        hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
1135                        hscb->sgptr = ahc_htole32(hscb->sgptr);
1136                        scb->sg_count = 1;
1137                        scb->flags |= SCB_SENSE;
1138                        ahc_qinfifo_requeue_tail(ahc, scb);
1139                        ahc_outb(ahc, RETURN_1, SEND_SENSE);
1140                        /*
1141                         * Ensure we have enough time to actually
1142                         * retrieve the sense.
1143                         */
1144                        ahc_scb_timer_reset(scb, 5 * 1000000);
1145                        break;
1146                }
1147                default:
1148                        break;
1149                }
1150                break;
1151        }
1152        case NO_MATCH:
1153        {
1154                /* Ensure we don't leave the selection hardware on */
1155                ahc_outb(ahc, SCSISEQ,
1156                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1157
1158                printk("%s:%c:%d: no active SCB for reconnecting "
1159                       "target - issuing BUS DEVICE RESET\n",
1160                       ahc_name(ahc), devinfo.channel, devinfo.target);
1161                printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1162                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1163                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1164                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1165                printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1166                       "SINDEX == 0x%x\n",
1167                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1168                       ahc_index_busy_tcl(ahc,
1169                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1170                                      ahc_inb(ahc, SAVED_LUN))),
1171                       ahc_inb(ahc, SINDEX));
1172                printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1173                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1174                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1175                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1176                       ahc_inb(ahc, SCB_CONTROL));
1177                printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1178                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1179                printk("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
1180                printk("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
1181                ahc_dump_card_state(ahc);
1182                ahc->msgout_buf[0] = TARGET_RESET;
1183                ahc->msgout_len = 1;
1184                ahc->msgout_index = 0;
1185                ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1186                ahc_outb(ahc, MSG_OUT, HOST_MSG);
1187                ahc_assert_atn(ahc);
1188                break;
1189        }
1190        case SEND_REJECT:
1191        {
1192                u_int rejbyte = ahc_inb(ahc, ACCUM);
1193                printk("%s:%c:%d: Warning - unknown message received from "
1194                       "target (0x%x).  Rejecting\n",
1195                       ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
1196                break;
1197        }
1198        case PROTO_VIOLATION:
1199        {
1200                ahc_handle_proto_violation(ahc);
1201                break;
1202        }
1203        case IGN_WIDE_RES:
1204                ahc_handle_ign_wide_residue(ahc, &devinfo);
1205                break;
1206        case PDATA_REINIT:
1207                ahc_reinitialize_dataptrs(ahc);
1208                break;
1209        case BAD_PHASE:
1210        {
1211                u_int lastphase;
1212
1213                lastphase = ahc_inb(ahc, LASTPHASE);
1214                printk("%s:%c:%d: unknown scsi bus phase %x, "
1215                       "lastphase = 0x%x.  Attempting to continue\n",
1216                       ahc_name(ahc), devinfo.channel, devinfo.target,
1217                       lastphase, ahc_inb(ahc, SCSISIGI));
1218                break;
1219        }
1220        case MISSED_BUSFREE:
1221        {
1222                u_int lastphase;
1223
1224                lastphase = ahc_inb(ahc, LASTPHASE);
1225                printk("%s:%c:%d: Missed busfree. "
1226                       "Lastphase = 0x%x, Curphase = 0x%x\n",
1227                       ahc_name(ahc), devinfo.channel, devinfo.target,
1228                       lastphase, ahc_inb(ahc, SCSISIGI));
1229                ahc_restart(ahc);
1230                return;
1231        }
1232        case HOST_MSG_LOOP:
1233        {
1234                /*
1235                 * The sequencer has encountered a message phase
1236                 * that requires host assistance for completion.
1237                 * While handling the message phase(s), we will be
1238                 * notified by the sequencer after each byte is
1239                 * transferred so we can track bus phase changes.
1240                 *
1241                 * If this is the first time we've seen a HOST_MSG_LOOP
1242                 * interrupt, initialize the state of the host message
1243                 * loop.
1244                 */
1245                if (ahc->msg_type == MSG_TYPE_NONE) {
1246                        struct scb *scb;
1247                        u_int scb_index;
1248                        u_int bus_phase;
1249
1250                        bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1251                        if (bus_phase != P_MESGIN
1252                         && bus_phase != P_MESGOUT) {
1253                                printk("ahc_intr: HOST_MSG_LOOP bad "
1254                                       "phase 0x%x\n",
1255                                      bus_phase);
1256                                /*
1257                                 * Probably transitioned to bus free before
1258                                 * we got here.  Just punt the message.
1259                                 */
1260                                ahc_clear_intstat(ahc);
1261                                ahc_restart(ahc);
1262                                return;
1263                        }
1264
1265                        scb_index = ahc_inb(ahc, SCB_TAG);
1266                        scb = ahc_lookup_scb(ahc, scb_index);
1267                        if (devinfo.role == ROLE_INITIATOR) {
1268                                if (bus_phase == P_MESGOUT) {
1269                                        if (scb == NULL)
1270                                                panic("HOST_MSG_LOOP with "
1271                                                      "invalid SCB %x\n",
1272                                                      scb_index);
1273
1274                                        ahc_setup_initiator_msgout(ahc,
1275                                                                   &devinfo,
1276                                                                   scb);
1277                                } else {
1278                                        ahc->msg_type =
1279                                            MSG_TYPE_INITIATOR_MSGIN;
1280                                        ahc->msgin_index = 0;
1281                                }
1282                        }
1283#ifdef AHC_TARGET_MODE
1284                        else {
1285                                if (bus_phase == P_MESGOUT) {
1286                                        ahc->msg_type =
1287                                            MSG_TYPE_TARGET_MSGOUT;
1288                                        ahc->msgin_index = 0;
1289                                } else
1290                                        ahc_setup_target_msgin(ahc,
1291                                                               &devinfo,
1292                                                               scb);
1293                        }
1294#endif
1295                }
1296
1297                ahc_handle_message_phase(ahc);
1298                break;
1299        }
1300        case PERR_DETECTED:
1301        {
1302                /*
1303                 * If we've cleared the parity error interrupt
1304                 * but the sequencer still believes that SCSIPERR
1305                 * is true, it must be that the parity error is
1306                 * for the currently presented byte on the bus,
1307                 * and we are not in a phase (data-in) where we will
1308                 * eventually ack this byte.  Ack the byte and
1309                 * throw it away in the hope that the target will
1310                 * take us to message out to deliver the appropriate
1311                 * error message.
1312                 */
1313                if ((intstat & SCSIINT) == 0
1314                 && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
1315
1316                        if ((ahc->features & AHC_DT) == 0) {
1317                                u_int curphase;
1318
1319                                /*
1320                                 * The hardware will only let you ack bytes
1321                                 * if the expected phase in SCSISIGO matches
1322                                 * the current phase.  Make sure this is
1323                                 * currently the case.
1324                                 */
1325                                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1326                                ahc_outb(ahc, LASTPHASE, curphase);
1327                                ahc_outb(ahc, SCSISIGO, curphase);
1328                        }
1329                        if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
1330                                int wait;
1331
1332                                /*
1333                                 * In a data phase.  Faster to bitbucket
1334                                 * the data than to individually ack each
1335                                 * byte.  This is also the only strategy
1336                                 * that will work with AUTOACK enabled.
1337                                 */
1338                                ahc_outb(ahc, SXFRCTL1,
1339                                         ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
1340                                wait = 5000;
1341                                while (--wait != 0) {
1342                                        if ((ahc_inb(ahc, SCSISIGI)
1343                                          & (CDI|MSGI)) != 0)
1344                                                break;
1345                                        ahc_delay(100);
1346                                }
1347                                ahc_outb(ahc, SXFRCTL1,
1348                                         ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
1349                                if (wait == 0) {
1350                                        struct  scb *scb;
1351                                        u_int   scb_index;
1352
1353                                        ahc_print_devinfo(ahc, &devinfo);
1354                                        printk("Unable to clear parity error.  "
1355                                               "Resetting bus.\n");
1356                                        scb_index = ahc_inb(ahc, SCB_TAG);
1357                                        scb = ahc_lookup_scb(ahc, scb_index);
1358                                        if (scb != NULL)
1359                                                ahc_set_transaction_status(scb,
1360                                                    CAM_UNCOR_PARITY);
1361                                        ahc_reset_channel(ahc, devinfo.channel,
1362                                                          /*init reset*/TRUE);
1363                                }
1364                        } else {
1365                                ahc_inb(ahc, SCSIDATL);
1366                        }
1367                }
1368                break;
1369        }
1370        case DATA_OVERRUN:
1371        {
1372                /*
1373                 * When the sequencer detects an overrun, it
1374                 * places the controller in "BITBUCKET" mode
1375                 * and allows the target to complete its transfer.
1376                 * Unfortunately, none of the counters get updated
1377                 * when the controller is in this mode, so we have
1378                 * no way of knowing how large the overrun was.
1379                 */
1380                u_int scbindex = ahc_inb(ahc, SCB_TAG);
1381                u_int lastphase = ahc_inb(ahc, LASTPHASE);
1382                u_int i;
1383
1384                scb = ahc_lookup_scb(ahc, scbindex);
1385                for (i = 0; i < num_phases; i++) {
1386                        if (lastphase == ahc_phase_table[i].phase)
1387                                break;
1388                }
1389                ahc_print_path(ahc, scb);
1390                printk("data overrun detected %s."
1391                       "  Tag == 0x%x.\n",
1392                       ahc_phase_table[i].phasemsg,
1393                       scb->hscb->tag);
1394                ahc_print_path(ahc, scb);
1395                printk("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
1396                       ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
1397                       ahc_get_transfer_length(scb), scb->sg_count);
1398                if (scb->sg_count > 0) {
1399                        for (i = 0; i < scb->sg_count; i++) {
1400
1401                                printk("sg[%d] - Addr 0x%x%x : Length %d\n",
1402                                       i,
1403                                       (ahc_le32toh(scb->sg_list[i].len) >> 24
1404                                        & SG_HIGH_ADDR_BITS),
1405                                       ahc_le32toh(scb->sg_list[i].addr),
1406                                       ahc_le32toh(scb->sg_list[i].len)
1407                                       & AHC_SG_LEN_MASK);
1408                        }
1409                }
1410                /*
1411                 * Set this and it will take effect when the
1412                 * target does a command complete.
1413                 */
1414                ahc_freeze_devq(ahc, scb);
1415                if ((scb->flags & SCB_SENSE) == 0) {
1416                        ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1417                } else {
1418                        scb->flags &= ~SCB_SENSE;
1419                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1420                }
1421                ahc_freeze_scb(scb);
1422
1423                if ((ahc->features & AHC_ULTRA2) != 0) {
1424                        /*
1425                         * Clear the channel in case we return
1426                         * to data phase later.
1427                         */
1428                        ahc_outb(ahc, SXFRCTL0,
1429                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1430                        ahc_outb(ahc, SXFRCTL0,
1431                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1432                }
1433                if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
1434                        u_int dscommand1;
1435
1436                        /* Ensure HHADDR is 0 for future DMA operations. */
1437                        dscommand1 = ahc_inb(ahc, DSCOMMAND1);
1438                        ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
1439                        ahc_outb(ahc, HADDR, 0);
1440                        ahc_outb(ahc, DSCOMMAND1, dscommand1);
1441                }
1442                break;
1443        }
1444        case MKMSG_FAILED:
1445        {
1446                u_int scbindex;
1447
1448                printk("%s:%c:%d:%d: Attempt to issue message failed\n",
1449                       ahc_name(ahc), devinfo.channel, devinfo.target,
1450                       devinfo.lun);
1451                scbindex = ahc_inb(ahc, SCB_TAG);
1452                scb = ahc_lookup_scb(ahc, scbindex);
1453                if (scb != NULL
1454                 && (scb->flags & SCB_RECOVERY_SCB) != 0)
1455                        /*
1456                         * Ensure that we didn't put a second instance of this
1457                         * SCB into the QINFIFO.
1458                         */
1459                        ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
1460                                           SCB_GET_CHANNEL(ahc, scb),
1461                                           SCB_GET_LUN(scb), scb->hscb->tag,
1462                                           ROLE_INITIATOR, /*status*/0,
1463                                           SEARCH_REMOVE);
1464                break;
1465        }
1466        case NO_FREE_SCB:
1467        {
1468                printk("%s: No free or disconnected SCBs\n", ahc_name(ahc));
1469                ahc_dump_card_state(ahc);
1470                panic("for safety");
1471                break;
1472        }
1473        case SCB_MISMATCH:
1474        {
1475                u_int scbptr;
1476
1477                scbptr = ahc_inb(ahc, SCBPTR);
1478                printk("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
1479                       scbptr, ahc_inb(ahc, ARG_1),
1480                       ahc->scb_data->hscbs[scbptr].tag);
1481                ahc_dump_card_state(ahc);
1482                panic("for safety");
1483                break;
1484        }
1485        case OUT_OF_RANGE:
1486        {
1487                printk("%s: BTT calculation out of range\n", ahc_name(ahc));
1488                printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1489                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1490                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1491                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1492                printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1493                       "SINDEX == 0x%x\n, A == 0x%x\n",
1494                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1495                       ahc_index_busy_tcl(ahc,
1496                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1497                                      ahc_inb(ahc, SAVED_LUN))),
1498                       ahc_inb(ahc, SINDEX),
1499                       ahc_inb(ahc, ACCUM));
1500                printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1501                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1502                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1503                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1504                       ahc_inb(ahc, SCB_CONTROL));
1505                printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1506                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1507                ahc_dump_card_state(ahc);
1508                panic("for safety");
1509                break;
1510        }
1511        default:
1512                printk("ahc_intr: seqint, "
1513                       "intstat == 0x%x, scsisigi = 0x%x\n",
1514                       intstat, ahc_inb(ahc, SCSISIGI));
1515                break;
1516        }
1517unpause:
1518        /*
1519         *  The sequencer is paused immediately on
1520         *  a SEQINT, so we should restart it when
1521         *  we're done.
1522         */
1523        ahc_unpause(ahc);
1524}
1525
1526static void
1527ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
1528{
1529        u_int   scb_index;
1530        u_int   status0;
1531        u_int   status;
1532        struct  scb *scb;
1533        char    cur_channel;
1534        char    intr_channel;
1535
1536        if ((ahc->features & AHC_TWIN) != 0
1537         && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
1538                cur_channel = 'B';
1539        else
1540                cur_channel = 'A';
1541        intr_channel = cur_channel;
1542
1543        if ((ahc->features & AHC_ULTRA2) != 0)
1544                status0 = ahc_inb(ahc, SSTAT0) & IOERR;
1545        else
1546                status0 = 0;
1547        status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1548        if (status == 0 && status0 == 0) {
1549                if ((ahc->features & AHC_TWIN) != 0) {
1550                        /* Try the other channel */
1551                        ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1552                        status = ahc_inb(ahc, SSTAT1)
1553                               & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1554                        intr_channel = (cur_channel == 'A') ? 'B' : 'A';
1555                }
1556                if (status == 0) {
1557                        printk("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
1558                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
1559                        ahc_unpause(ahc);
1560                        return;
1561                }
1562        }
1563
1564        /* Make sure the sequencer is in a safe location. */
1565        ahc_clear_critical_section(ahc);
1566
1567        scb_index = ahc_inb(ahc, SCB_TAG);
1568        scb = ahc_lookup_scb(ahc, scb_index);
1569        if (scb != NULL
1570         && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1571                scb = NULL;
1572
1573        if ((ahc->features & AHC_ULTRA2) != 0
1574         && (status0 & IOERR) != 0) {
1575                int now_lvd;
1576
1577                now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1578                printk("%s: Transceiver State Has Changed to %s mode\n",
1579                       ahc_name(ahc), now_lvd ? "LVD" : "SE");
1580                ahc_outb(ahc, CLRSINT0, CLRIOERR);
1581                /*
1582                 * When transitioning to SE mode, the reset line
1583                 * glitches, triggering an arbitration bug in some
1584                 * Ultra2 controllers.  This bug is cleared when we
1585                 * assert the reset line.  Since a reset glitch has
1586                 * already occurred with this transition and a
1587                 * transceiver state change is handled just like
1588                 * a bus reset anyway, asserting the reset line
1589                 * ourselves is safe.
1590                 */
1591                ahc_reset_channel(ahc, intr_channel,
1592                                 /*Initiate Reset*/now_lvd == 0);
1593        } else if ((status & SCSIRSTI) != 0) {
1594                printk("%s: Someone reset channel %c\n",
1595                        ahc_name(ahc), intr_channel);
1596                if (intr_channel != cur_channel)
1597                        ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1598                ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1599        } else if ((status & SCSIPERR) != 0) {
1600                /*
1601                 * Determine the bus phase and queue an appropriate message.
1602                 * SCSIPERR is latched true as soon as a parity error
1603                 * occurs.  If the sequencer acked the transfer that
1604                 * caused the parity error and the currently presented
1605                 * transfer on the bus has correct parity, SCSIPERR will
1606                 * be cleared by CLRSCSIPERR.  Use this to determine if
1607                 * we should look at the last phase the sequencer recorded,
1608                 * or the current phase presented on the bus.
1609                 */
1610                struct  ahc_devinfo devinfo;
1611                u_int   mesg_out;
1612                u_int   curphase;
1613                u_int   errorphase;
1614                u_int   lastphase;
1615                u_int   scsirate;
1616                u_int   i;
1617                u_int   sstat2;
1618                int     silent;
1619
1620                lastphase = ahc_inb(ahc, LASTPHASE);
1621                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1622                sstat2 = ahc_inb(ahc, SSTAT2);
1623                ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1624                /*
1625                 * For all phases save DATA, the sequencer won't
1626                 * automatically ack a byte that has a parity error
1627                 * in it.  So the only way that the current phase
1628                 * could be 'data-in' is if the parity error is for
1629                 * an already acked byte in the data phase.  During
1630                 * synchronous data-in transfers, we may actually
1631                 * ack bytes before latching the current phase in
1632                 * LASTPHASE, leading to the discrepancy between
1633                 * curphase and lastphase.
1634                 */
1635                if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1636                 || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1637                        errorphase = curphase;
1638                else
1639                        errorphase = lastphase;
1640
1641                for (i = 0; i < num_phases; i++) {
1642                        if (errorphase == ahc_phase_table[i].phase)
1643                                break;
1644                }
1645                mesg_out = ahc_phase_table[i].mesg_out;
1646                silent = FALSE;
1647                if (scb != NULL) {
1648                        if (SCB_IS_SILENT(scb))
1649                                silent = TRUE;
1650                        else
1651                                ahc_print_path(ahc, scb);
1652                        scb->flags |= SCB_TRANSMISSION_ERROR;
1653                } else
1654                        printk("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1655                               SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1656                scsirate = ahc_inb(ahc, SCSIRATE);
1657                if (silent == FALSE) {
1658                        printk("parity error detected %s. "
1659                               "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1660                               ahc_phase_table[i].phasemsg,
1661                               ahc_inw(ahc, SEQADDR0),
1662                               scsirate);
1663                        if ((ahc->features & AHC_DT) != 0) {
1664                                if ((sstat2 & CRCVALERR) != 0)
1665                                        printk("\tCRC Value Mismatch\n");
1666                                if ((sstat2 & CRCENDERR) != 0)
1667                                        printk("\tNo terminal CRC packet "
1668                                               "received\n");
1669                                if ((sstat2 & CRCREQERR) != 0)
1670                                        printk("\tIllegal CRC packet "
1671                                               "request\n");
1672                                if ((sstat2 & DUAL_EDGE_ERR) != 0)
1673                                        printk("\tUnexpected %sDT Data Phase\n",
1674                                               (scsirate & SINGLE_EDGE)
1675                                             ? "" : "non-");
1676                        }
1677                }
1678
1679                if ((ahc->features & AHC_DT) != 0
1680                 && (sstat2 & DUAL_EDGE_ERR) != 0) {
1681                        /*
1682                         * This error applies regardless of
1683                         * data direction, so ignore the value
1684                         * in the phase table.
1685                         */
1686                        mesg_out = INITIATOR_ERROR;
1687                }
1688
1689                /*
1690                 * We've set the hardware to assert ATN if we
1691                 * get a parity error on "in" phases, so all we
1692                 * need to do is stuff the message buffer with
1693                 * the appropriate message.  "In" phases have set
1694                 * mesg_out to something other than MSG_NOP.
1695                 */
1696                if (mesg_out != NOP) {
1697                        if (ahc->msg_type != MSG_TYPE_NONE)
1698                                ahc->send_msg_perror = TRUE;
1699                        else
1700                                ahc_outb(ahc, MSG_OUT, mesg_out);
1701                }
1702                /*
1703                 * Force a renegotiation with this target just in
1704                 * case we are out of sync for some external reason
1705                 * unknown (or unreported) by the target.
1706                 */
1707                ahc_fetch_devinfo(ahc, &devinfo);
1708                ahc_force_renegotiation(ahc, &devinfo);
1709
1710                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1711                ahc_unpause(ahc);
1712        } else if ((status & SELTO) != 0) {
1713                u_int   scbptr;
1714
1715                /* Stop the selection */
1716                ahc_outb(ahc, SCSISEQ, 0);
1717
1718                /* No more pending messages */
1719                ahc_clear_msg_state(ahc);
1720
1721                /* Clear interrupt state */
1722                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1723                ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1724
1725                /*
1726                 * Although the driver does not care about the
1727                 * 'Selection in Progress' status bit, the busy
1728                 * LED does.  SELINGO is only cleared by a successful
1729                 * selection, so we must manually clear it to insure
1730                 * the LED turns off just incase no future successful
1731                 * selections occur (e.g. no devices on the bus).
1732                 */
1733                ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1734
1735                scbptr = ahc_inb(ahc, WAITING_SCBH);
1736                ahc_outb(ahc, SCBPTR, scbptr);
1737                scb_index = ahc_inb(ahc, SCB_TAG);
1738
1739                scb = ahc_lookup_scb(ahc, scb_index);
1740                if (scb == NULL) {
1741                        printk("%s: ahc_intr - referenced scb not "
1742                               "valid during SELTO scb(%d, %d)\n",
1743                               ahc_name(ahc), scbptr, scb_index);
1744                        ahc_dump_card_state(ahc);
1745                } else {
1746                        struct ahc_devinfo devinfo;
1747#ifdef AHC_DEBUG
1748                        if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1749                                ahc_print_path(ahc, scb);
1750                                printk("Saw Selection Timeout for SCB 0x%x\n",
1751                                       scb_index);
1752                        }
1753#endif
1754                        ahc_scb_devinfo(ahc, &devinfo, scb);
1755                        ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1756                        ahc_freeze_devq(ahc, scb);
1757
1758                        /*
1759                         * Cancel any pending transactions on the device
1760                         * now that it seems to be missing.  This will
1761                         * also revert us to async/narrow transfers until
1762                         * we can renegotiate with the device.
1763                         */
1764                        ahc_handle_devreset(ahc, &devinfo,
1765                                            CAM_SEL_TIMEOUT,
1766                                            "Selection Timeout",
1767                                            /*verbose_level*/1);
1768                }
1769                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1770                ahc_restart(ahc);
1771        } else if ((status & BUSFREE) != 0
1772                && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1773                struct  ahc_devinfo devinfo;
1774                u_int   lastphase;
1775                u_int   saved_scsiid;
1776                u_int   saved_lun;
1777                u_int   target;
1778                u_int   initiator_role_id;
1779                char    channel;
1780                int     printerror;
1781
1782                /*
1783                 * Clear our selection hardware as soon as possible.
1784                 * We may have an entry in the waiting Q for this target,
1785                 * that is affected by this busfree and we don't want to
1786                 * go about selecting the target while we handle the event.
1787                 */
1788                ahc_outb(ahc, SCSISEQ,
1789                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1790
1791                /*
1792                 * Disable busfree interrupts and clear the busfree
1793                 * interrupt status.  We do this here so that several
1794                 * bus transactions occur prior to clearing the SCSIINT
1795                 * latch.  It can take a bit for the clearing to take effect.
1796                 */
1797                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1798                ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1799
1800                /*
1801                 * Look at what phase we were last in.
1802                 * If its message out, chances are pretty good
1803                 * that the busfree was in response to one of
1804                 * our abort requests.
1805                 */
1806                lastphase = ahc_inb(ahc, LASTPHASE);
1807                saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1808                saved_lun = ahc_inb(ahc, SAVED_LUN);
1809                target = SCSIID_TARGET(ahc, saved_scsiid);
1810                initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1811                channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1812                ahc_compile_devinfo(&devinfo, initiator_role_id,
1813                                    target, saved_lun, channel, ROLE_INITIATOR);
1814                printerror = 1;
1815
1816                if (lastphase == P_MESGOUT) {
1817                        u_int tag;
1818
1819                        tag = SCB_LIST_NULL;
1820                        if (ahc_sent_msg(ahc, AHCMSG_1B, ABORT_TASK, TRUE)
1821                         || ahc_sent_msg(ahc, AHCMSG_1B, ABORT_TASK_SET, TRUE)) {
1822                                if (ahc->msgout_buf[ahc->msgout_index - 1]
1823                                 == ABORT_TASK)
1824                                        tag = scb->hscb->tag;
1825                                ahc_print_path(ahc, scb);
1826                                printk("SCB %d - Abort%s Completed.\n",
1827                                       scb->hscb->tag, tag == SCB_LIST_NULL ?
1828                                       "" : " Tag");
1829                                ahc_abort_scbs(ahc, target, channel,
1830                                               saved_lun, tag,
1831                                               ROLE_INITIATOR,
1832                                               CAM_REQ_ABORTED);
1833                                printerror = 0;
1834                        } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1835                                                TARGET_RESET, TRUE)) {
1836                                ahc_compile_devinfo(&devinfo,
1837                                                    initiator_role_id,
1838                                                    target,
1839                                                    CAM_LUN_WILDCARD,
1840                                                    channel,
1841                                                    ROLE_INITIATOR);
1842                                ahc_handle_devreset(ahc, &devinfo,
1843                                                    CAM_BDR_SENT,
1844                                                    "Bus Device Reset",
1845                                                    /*verbose_level*/0);
1846                                printerror = 0;
1847                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1848                                                EXTENDED_PPR, FALSE)) {
1849                                struct ahc_initiator_tinfo *tinfo;
1850                                struct ahc_tmode_tstate *tstate;
1851
1852                                /*
1853                                 * PPR Rejected.  Try non-ppr negotiation
1854                                 * and retry command.
1855                                 */
1856                                tinfo = ahc_fetch_transinfo(ahc,
1857                                                            devinfo.channel,
1858                                                            devinfo.our_scsiid,
1859                                                            devinfo.target,
1860                                                            &tstate);
1861                                tinfo->curr.transport_version = 2;
1862                                tinfo->goal.transport_version = 2;
1863                                tinfo->goal.ppr_options = 0;
1864                                ahc_qinfifo_requeue_tail(ahc, scb);
1865                                printerror = 0;
1866                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1867                                                EXTENDED_WDTR, FALSE)) {
1868                                /*
1869                                 * Negotiation Rejected.  Go-narrow and
1870                                 * retry command.
1871                                 */
1872                                ahc_set_width(ahc, &devinfo,
1873                                              MSG_EXT_WDTR_BUS_8_BIT,
1874                                              AHC_TRANS_CUR|AHC_TRANS_GOAL,
1875                                              /*paused*/TRUE);
1876                                ahc_qinfifo_requeue_tail(ahc, scb);
1877                                printerror = 0;
1878                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1879                                                EXTENDED_SDTR, FALSE)) {
1880                                /*
1881                                 * Negotiation Rejected.  Go-async and
1882                                 * retry command.
1883                                 */
1884                                ahc_set_syncrate(ahc, &devinfo,
1885                                                /*syncrate*/NULL,
1886                                                /*period*/0, /*offset*/0,
1887                                                /*ppr_options*/0,
1888                                                AHC_TRANS_CUR|AHC_TRANS_GOAL,
1889                                                /*paused*/TRUE);
1890                                ahc_qinfifo_requeue_tail(ahc, scb);
1891                                printerror = 0;
1892                        }
1893                }
1894                if (printerror != 0) {
1895                        u_int i;
1896
1897                        if (scb != NULL) {
1898                                u_int tag;
1899
1900                                if ((scb->hscb->control & TAG_ENB) != 0)
1901                                        tag = scb->hscb->tag;
1902                                else
1903                                        tag = SCB_LIST_NULL;
1904                                ahc_print_path(ahc, scb);
1905                                ahc_abort_scbs(ahc, target, channel,
1906                                               SCB_GET_LUN(scb), tag,
1907                                               ROLE_INITIATOR,
1908                                               CAM_UNEXP_BUSFREE);
1909                        } else {
1910                                /*
1911                                 * We had not fully identified this connection,
1912                                 * so we cannot abort anything.
1913                                 */
1914                                printk("%s: ", ahc_name(ahc));
1915                        }
1916                        for (i = 0; i < num_phases; i++) {
1917                                if (lastphase == ahc_phase_table[i].phase)
1918                                        break;
1919                        }
1920                        if (lastphase != P_BUSFREE) {
1921                                /*
1922                                 * Renegotiate with this device at the
1923                                 * next opportunity just in case this busfree
1924                                 * is due to a negotiation mismatch with the
1925                                 * device.
1926                                 */
1927                                ahc_force_renegotiation(ahc, &devinfo);
1928                        }
1929                        printk("Unexpected busfree %s\n"
1930                               "SEQADDR == 0x%x\n",
1931                               ahc_phase_table[i].phasemsg,
1932                               ahc_inb(ahc, SEQADDR0)
1933                                | (ahc_inb(ahc, SEQADDR1) << 8));
1934                }
1935                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1936                ahc_restart(ahc);
1937        } else {
1938                printk("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1939                       ahc_name(ahc), status);
1940                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1941        }
1942}
1943
1944/*
1945 * Force renegotiation to occur the next time we initiate
1946 * a command to the current device.
1947 */
1948static void
1949ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1950{
1951        struct  ahc_initiator_tinfo *targ_info;
1952        struct  ahc_tmode_tstate *tstate;
1953
1954        targ_info = ahc_fetch_transinfo(ahc,
1955                                        devinfo->channel,
1956                                        devinfo->our_scsiid,
1957                                        devinfo->target,
1958                                        &tstate);
1959        ahc_update_neg_request(ahc, devinfo, tstate,
1960                               targ_info, AHC_NEG_IF_NON_ASYNC);
1961}
1962
1963#define AHC_MAX_STEPS 2000
1964static void
1965ahc_clear_critical_section(struct ahc_softc *ahc)
1966{
1967        int     stepping;
1968        int     steps;
1969        u_int   simode0;
1970        u_int   simode1;
1971
1972        if (ahc->num_critical_sections == 0)
1973                return;
1974
1975        stepping = FALSE;
1976        steps = 0;
1977        simode0 = 0;
1978        simode1 = 0;
1979        for (;;) {
1980                struct  cs *cs;
1981                u_int   seqaddr;
1982                u_int   i;
1983
1984                seqaddr = ahc_inb(ahc, SEQADDR0)
1985                        | (ahc_inb(ahc, SEQADDR1) << 8);
1986
1987                /*
1988                 * Seqaddr represents the next instruction to execute,
1989                 * so we are really executing the instruction just
1990                 * before it.
1991                 */
1992                if (seqaddr != 0)
1993                        seqaddr -= 1;
1994                cs = ahc->critical_sections;
1995                for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1996                        if (cs->begin < seqaddr && cs->end >= seqaddr)
1997                                break;
1998                }
1999
2000                if (i == ahc->num_critical_sections)
2001                        break;
2002
2003                if (steps > AHC_MAX_STEPS) {
2004                        printk("%s: Infinite loop in critical section\n",
2005                               ahc_name(ahc));
2006                        ahc_dump_card_state(ahc);
2007                        panic("critical section loop");
2008                }
2009
2010                steps++;
2011                if (stepping == FALSE) {
2012
2013                        /*
2014                         * Disable all interrupt sources so that the
2015                         * sequencer will not be stuck by a pausing
2016                         * interrupt condition while we attempt to
2017                         * leave a critical section.
2018                         */
2019                        simode0 = ahc_inb(ahc, SIMODE0);
2020                        ahc_outb(ahc, SIMODE0, 0);
2021                        simode1 = ahc_inb(ahc, SIMODE1);
2022                        if ((ahc->features & AHC_DT) != 0)
2023                                /*
2024                                 * On DT class controllers, we
2025                                 * use the enhanced busfree logic.
2026                                 * Unfortunately we cannot re-enable
2027                                 * busfree detection within the
2028                                 * current connection, so we must
2029                                 * leave it on while single stepping.
2030                                 */
2031                                ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
2032                        else
2033                                ahc_outb(ahc, SIMODE1, 0);
2034                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2035                        ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
2036                        stepping = TRUE;
2037                }
2038                if ((ahc->features & AHC_DT) != 0) {
2039                        ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
2040                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2041                }
2042                ahc_outb(ahc, HCNTRL, ahc->unpause);
2043                while (!ahc_is_paused(ahc))
2044                        ahc_delay(200);
2045        }
2046        if (stepping) {
2047                ahc_outb(ahc, SIMODE0, simode0);
2048                ahc_outb(ahc, SIMODE1, simode1);
2049                ahc_outb(ahc, SEQCTL, ahc->seqctl);
2050        }
2051}
2052
2053/*
2054 * Clear any pending interrupt status.
2055 */
2056static void
2057ahc_clear_intstat(struct ahc_softc *ahc)
2058{
2059        /* Clear any interrupt conditions this may have caused */
2060        ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2061                                |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
2062                                CLRREQINIT);
2063        ahc_flush_device_writes(ahc);
2064        ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
2065        ahc_flush_device_writes(ahc);
2066        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2067        ahc_flush_device_writes(ahc);
2068}
2069
2070/**************************** Debugging Routines ******************************/
2071#ifdef AHC_DEBUG
2072uint32_t ahc_debug = AHC_DEBUG_OPTS;
2073#endif
2074
2075#if 0 /* unused */
2076static void
2077ahc_print_scb(struct scb *scb)
2078{
2079        int i;
2080
2081        struct hardware_scb *hscb = scb->hscb;
2082
2083        printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2084               (void *)scb,
2085               hscb->control,
2086               hscb->scsiid,
2087               hscb->lun,
2088               hscb->cdb_len);
2089        printk("Shared Data: ");
2090        for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
2091                printk("%#02x", hscb->shared_data.cdb[i]);
2092        printk("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
2093                ahc_le32toh(hscb->dataptr),
2094                ahc_le32toh(hscb->datacnt),
2095                ahc_le32toh(hscb->sgptr),
2096                hscb->tag);
2097        if (scb->sg_count > 0) {
2098                for (i = 0; i < scb->sg_count; i++) {
2099                        printk("sg[%d] - Addr 0x%x%x : Length %d\n",
2100                               i,
2101                               (ahc_le32toh(scb->sg_list[i].len) >> 24
2102                                & SG_HIGH_ADDR_BITS),
2103                               ahc_le32toh(scb->sg_list[i].addr),
2104                               ahc_le32toh(scb->sg_list[i].len));
2105                }
2106        }
2107}
2108#endif
2109
2110/************************* Transfer Negotiation *******************************/
2111/*
2112 * Allocate per target mode instance (ID we respond to as a target)
2113 * transfer negotiation data structures.
2114 */
2115static struct ahc_tmode_tstate *
2116ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
2117{
2118        struct ahc_tmode_tstate *master_tstate;
2119        struct ahc_tmode_tstate *tstate;
2120        int i;
2121
2122        master_tstate = ahc->enabled_targets[ahc->our_id];
2123        if (channel == 'B') {
2124                scsi_id += 8;
2125                master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
2126        }
2127        if (ahc->enabled_targets[scsi_id] != NULL
2128         && ahc->enabled_targets[scsi_id] != master_tstate)
2129                panic("%s: ahc_alloc_tstate - Target already allocated",
2130                      ahc_name(ahc));
2131        tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
2132        if (tstate == NULL)
2133                return (NULL);
2134
2135        /*
2136         * If we have allocated a master tstate, copy user settings from
2137         * the master tstate (taken from SRAM or the EEPROM) for this
2138         * channel, but reset our current and goal settings to async/narrow
2139         * until an initiator talks to us.
2140         */
2141        if (master_tstate != NULL) {
2142                memcpy(tstate, master_tstate, sizeof(*tstate));
2143                memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2144                tstate->ultraenb = 0;
2145                for (i = 0; i < AHC_NUM_TARGETS; i++) {
2146                        memset(&tstate->transinfo[i].curr, 0,
2147                              sizeof(tstate->transinfo[i].curr));
2148                        memset(&tstate->transinfo[i].goal, 0,
2149                              sizeof(tstate->transinfo[i].goal));
2150                }
2151        } else
2152                memset(tstate, 0, sizeof(*tstate));
2153        ahc->enabled_targets[scsi_id] = tstate;
2154        return (tstate);
2155}
2156
2157#ifdef AHC_TARGET_MODE
2158/*
2159 * Free per target mode instance (ID we respond to as a target)
2160 * transfer negotiation data structures.
2161 */
2162static void
2163ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
2164{
2165        struct ahc_tmode_tstate *tstate;
2166
2167        /*
2168         * Don't clean up our "master" tstate.
2169         * It has our default user settings.
2170         */
2171        if (((channel == 'B' && scsi_id == ahc->our_id_b)
2172          || (channel == 'A' && scsi_id == ahc->our_id))
2173         && force == FALSE)
2174                return;
2175
2176        if (channel == 'B')
2177                scsi_id += 8;
2178        tstate = ahc->enabled_targets[scsi_id];
2179        kfree(tstate);
2180        ahc->enabled_targets[scsi_id] = NULL;
2181}
2182#endif
2183
2184/*
2185 * Called when we have an active connection to a target on the bus,
2186 * this function finds the nearest syncrate to the input period limited
2187 * by the capabilities of the bus connectivity of and sync settings for
2188 * the target.
2189 */
2190static const struct ahc_syncrate *
2191ahc_devlimited_syncrate(struct ahc_softc *ahc,
2192                        struct ahc_initiator_tinfo *tinfo,
2193                        u_int *period, u_int *ppr_options, role_t role)
2194{
2195        struct  ahc_transinfo *transinfo;
2196        u_int   maxsync;
2197
2198        if ((ahc->features & AHC_ULTRA2) != 0) {
2199                if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
2200                 && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
2201                        maxsync = AHC_SYNCRATE_DT;
2202                } else {
2203                        maxsync = AHC_SYNCRATE_ULTRA;
2204                        /* Can't do DT on an SE bus */
2205                        *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2206                }
2207        } else if ((ahc->features & AHC_ULTRA) != 0) {
2208                maxsync = AHC_SYNCRATE_ULTRA;
2209        } else {
2210                maxsync = AHC_SYNCRATE_FAST;
2211        }
2212        /*
2213         * Never allow a value higher than our current goal
2214         * period otherwise we may allow a target initiated
2215         * negotiation to go above the limit as set by the
2216         * user.  In the case of an initiator initiated
2217         * sync negotiation, we limit based on the user
2218         * setting.  This allows the system to still accept
2219         * incoming negotiations even if target initiated
2220         * negotiation is not performed.
2221         */
2222        if (role == ROLE_TARGET)
2223                transinfo = &tinfo->user;
2224        else
2225                transinfo = &tinfo->goal;
2226        *ppr_options &= transinfo->ppr_options;
2227        if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2228                maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
2229                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2230        }
2231        if (transinfo->period == 0) {
2232                *period = 0;
2233                *ppr_options = 0;
2234                return (NULL);
2235        }
2236        *period = max(*period, (u_int)transinfo->period);
2237        return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
2238}
2239
2240/*
2241 * Look up the valid period to SCSIRATE conversion in our table.
2242 * Return the period and offset that should be sent to the target
2243 * if this was the beginning of an SDTR.
2244 */
2245const struct ahc_syncrate *
2246ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
2247                  u_int *ppr_options, u_int maxsync)
2248{
2249        const struct ahc_syncrate *syncrate;
2250
2251        if ((ahc->features & AHC_DT) == 0)
2252                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2253
2254        /* Skip all DT only entries if DT is not available */
2255        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2256         && maxsync < AHC_SYNCRATE_ULTRA2)
2257                maxsync = AHC_SYNCRATE_ULTRA2;
2258
2259        /* Now set the maxsync based on the card capabilities
2260         * DT is already done above */
2261        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2262            && maxsync < AHC_SYNCRATE_ULTRA)
2263                maxsync = AHC_SYNCRATE_ULTRA;
2264        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2265            && maxsync < AHC_SYNCRATE_FAST)
2266                maxsync = AHC_SYNCRATE_FAST;
2267
2268        for (syncrate = &ahc_syncrates[maxsync];
2269             syncrate->rate != NULL;
2270             syncrate++) {
2271
2272                /*
2273                 * The Ultra2 table doesn't go as low
2274                 * as for the Fast/Ultra cards.
2275                 */
2276                if ((ahc->features & AHC_ULTRA2) != 0
2277                 && (syncrate->sxfr_u2 == 0))
2278                        break;
2279
2280                if (*period <= syncrate->period) {
2281                        /*
2282                         * When responding to a target that requests
2283                         * sync, the requested rate may fall between
2284                         * two rates that we can output, but still be
2285                         * a rate that we can receive.  Because of this,
2286                         * we want to respond to the target with
2287                         * the same rate that it sent to us even
2288                         * if the period we use to send data to it
2289                         * is lower.  Only lower the response period
2290                         * if we must.
2291                         */
2292                        if (syncrate == &ahc_syncrates[maxsync])
2293                                *period = syncrate->period;
2294
2295                        /*
2296                         * At some speeds, we only support
2297                         * ST transfers.
2298                         */
2299                        if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
2300                                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2301                        break;
2302                }
2303        }
2304
2305        if ((*period == 0)
2306         || (syncrate->rate == NULL)
2307         || ((ahc->features & AHC_ULTRA2) != 0
2308          && (syncrate->sxfr_u2 == 0))) {
2309                /* Use asynchronous transfers. */
2310                *period = 0;
2311                syncrate = NULL;
2312                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2313        }
2314        return (syncrate);
2315}
2316
2317/*
2318 * Convert from an entry in our syncrate table to the SCSI equivalent
2319 * sync "period" factor.
2320 */
2321u_int
2322ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
2323{
2324        const struct ahc_syncrate *syncrate;
2325
2326        if ((ahc->features & AHC_ULTRA2) != 0)
2327                scsirate &= SXFR_ULTRA2;
2328        else
2329                scsirate &= SXFR;
2330
2331        /* now set maxsync based on card capabilities */
2332        if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
2333                maxsync = AHC_SYNCRATE_ULTRA2;
2334        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2335            && maxsync < AHC_SYNCRATE_ULTRA)
2336                maxsync = AHC_SYNCRATE_ULTRA;
2337        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2338            && maxsync < AHC_SYNCRATE_FAST)
2339                maxsync = AHC_SYNCRATE_FAST;
2340
2341
2342        syncrate = &ahc_syncrates[maxsync];
2343        while (syncrate->rate != NULL) {
2344
2345                if ((ahc->features & AHC_ULTRA2) != 0) {
2346                        if (syncrate->sxfr_u2 == 0)
2347                                break;
2348                        else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
2349                                return (syncrate->period);
2350                } else if (scsirate == (syncrate->sxfr & SXFR)) {
2351                                return (syncrate->period);
2352                }
2353                syncrate++;
2354        }
2355        return (0); /* async */
2356}
2357
2358/*
2359 * Truncate the given synchronous offset to a value the
2360 * current adapter type and syncrate are capable of.
2361 */
2362static void
2363ahc_validate_offset(struct ahc_softc *ahc,
2364                    struct ahc_initiator_tinfo *tinfo,
2365                    const struct ahc_syncrate *syncrate,
2366                    u_int *offset, int wide, role_t role)
2367{
2368        u_int maxoffset;
2369
2370        /* Limit offset to what we can do */
2371        if (syncrate == NULL) {
2372                maxoffset = 0;
2373        } else if ((ahc->features & AHC_ULTRA2) != 0) {
2374                maxoffset = MAX_OFFSET_ULTRA2;
2375        } else {
2376                if (wide)
2377                        maxoffset = MAX_OFFSET_16BIT;
2378                else
2379                        maxoffset = MAX_OFFSET_8BIT;
2380        }
2381        *offset = min(*offset, maxoffset);
2382        if (tinfo != NULL) {
2383                if (role == ROLE_TARGET)
2384                        *offset = min(*offset, (u_int)tinfo->user.offset);
2385                else
2386                        *offset = min(*offset, (u_int)tinfo->goal.offset);
2387        }
2388}
2389
2390/*
2391 * Truncate the given transfer width parameter to a value the
2392 * current adapter type is capable of.
2393 */
2394static void
2395ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
2396                   u_int *bus_width, role_t role)
2397{
2398        switch (*bus_width) {
2399        default:
2400                if (ahc->features & AHC_WIDE) {
2401                        /* Respond Wide */
2402                        *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2403                        break;
2404                }
2405                fallthrough;
2406        case MSG_EXT_WDTR_BUS_8_BIT:
2407                *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2408                break;
2409        }
2410        if (tinfo != NULL) {
2411                if (role == ROLE_TARGET)
2412                        *bus_width = min((u_int)tinfo->user.width, *bus_width);
2413                else
2414                        *bus_width = min((u_int)tinfo->goal.width, *bus_width);
2415        }
2416}
2417
2418/*
2419 * Update the bitmask of targets for which the controller should
2420 * negotiate with at the next convenient opportunity.  This currently
2421 * means the next time we send the initial identify messages for
2422 * a new transaction.
2423 */
2424int
2425ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2426                       struct ahc_tmode_tstate *tstate,
2427                       struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
2428{
2429        u_int auto_negotiate_orig;
2430
2431        auto_negotiate_orig = tstate->auto_negotiate;
2432        if (neg_type == AHC_NEG_ALWAYS) {
2433                /*
2434                 * Force our "current" settings to be
2435                 * unknown so that unless a bus reset
2436                 * occurs the need to renegotiate is
2437                 * recorded persistently.
2438                 */
2439                if ((ahc->features & AHC_WIDE) != 0)
2440                        tinfo->curr.width = AHC_WIDTH_UNKNOWN;
2441                tinfo->curr.period = AHC_PERIOD_UNKNOWN;
2442                tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
2443        }
2444        if (tinfo->curr.period != tinfo->goal.period
2445         || tinfo->curr.width != tinfo->goal.width
2446         || tinfo->curr.offset != tinfo->goal.offset
2447         || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2448         || (neg_type == AHC_NEG_IF_NON_ASYNC
2449          && (tinfo->goal.offset != 0
2450           || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2451           || tinfo->goal.ppr_options != 0)))
2452                tstate->auto_negotiate |= devinfo->target_mask;
2453        else
2454                tstate->auto_negotiate &= ~devinfo->target_mask;
2455
2456        return (auto_negotiate_orig != tstate->auto_negotiate);
2457}
2458
2459/*
2460 * Update the user/goal/curr tables of synchronous negotiation
2461 * parameters as well as, in the case of a current or active update,
2462 * any data structures on the host controller.  In the case of an
2463 * active update, the specified target is currently talking to us on
2464 * the bus, so the transfer parameter update must take effect
2465 * immediately.
2466 */
2467void
2468ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2469                 const struct ahc_syncrate *syncrate, u_int period,
2470                 u_int offset, u_int ppr_options, u_int type, int paused)
2471{
2472        struct  ahc_initiator_tinfo *tinfo;
2473        struct  ahc_tmode_tstate *tstate;
2474        u_int   old_period;
2475        u_int   old_offset;
2476        u_int   old_ppr;
2477        int     active;
2478        int     update_needed;
2479
2480        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2481        update_needed = 0;
2482
2483        if (syncrate == NULL) {
2484                period = 0;
2485                offset = 0;
2486        }
2487
2488        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2489                                    devinfo->target, &tstate);
2490
2491        if ((type & AHC_TRANS_USER) != 0) {
2492                tinfo->user.period = period;
2493                tinfo->user.offset = offset;
2494                tinfo->user.ppr_options = ppr_options;
2495        }
2496
2497        if ((type & AHC_TRANS_GOAL) != 0) {
2498                tinfo->goal.period = period;
2499                tinfo->goal.offset = offset;
2500                tinfo->goal.ppr_options = ppr_options;
2501        }
2502
2503        old_period = tinfo->curr.period;
2504        old_offset = tinfo->curr.offset;
2505        old_ppr    = tinfo->curr.ppr_options;
2506
2507        if ((type & AHC_TRANS_CUR) != 0
2508         && (old_period != period
2509          || old_offset != offset
2510          || old_ppr != ppr_options)) {
2511                u_int   scsirate;
2512
2513                update_needed++;
2514                scsirate = tinfo->scsirate;
2515                if ((ahc->features & AHC_ULTRA2) != 0) {
2516
2517                        scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
2518                        if (syncrate != NULL) {
2519                                scsirate |= syncrate->sxfr_u2;
2520                                if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
2521                                        scsirate |= ENABLE_CRC;
2522                                else
2523                                        scsirate |= SINGLE_EDGE;
2524                        }
2525                } else {
2526
2527                        scsirate &= ~(SXFR|SOFS);
2528                        /*
2529                         * Ensure Ultra mode is set properly for
2530                         * this target.
2531                         */
2532                        tstate->ultraenb &= ~devinfo->target_mask;
2533                        if (syncrate != NULL) {
2534                                if (syncrate->sxfr & ULTRA_SXFR) {
2535                                        tstate->ultraenb |=
2536                                                devinfo->target_mask;
2537                                }
2538                                scsirate |= syncrate->sxfr & SXFR;
2539                                scsirate |= offset & SOFS;
2540                        }
2541                        if (active) {
2542                                u_int sxfrctl0;
2543
2544                                sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
2545                                sxfrctl0 &= ~FAST20;
2546                                if (tstate->ultraenb & devinfo->target_mask)
2547                                        sxfrctl0 |= FAST20;
2548                                ahc_outb(ahc, SXFRCTL0, sxfrctl0);
2549                        }
2550                }
2551                if (active) {
2552                        ahc_outb(ahc, SCSIRATE, scsirate);
2553                        if ((ahc->features & AHC_ULTRA2) != 0)
2554                                ahc_outb(ahc, SCSIOFFSET, offset);
2555                }
2556
2557                tinfo->scsirate = scsirate;
2558                tinfo->curr.period = period;
2559                tinfo->curr.offset = offset;
2560                tinfo->curr.ppr_options = ppr_options;
2561
2562                ahc_send_async(ahc, devinfo->channel, devinfo->target,
2563                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2564                if (bootverbose) {
2565                        if (offset != 0) {
2566                                printk("%s: target %d synchronous at %sMHz%s, "
2567                                       "offset = 0x%x\n", ahc_name(ahc),
2568                                       devinfo->target, syncrate->rate,
2569                                       (ppr_options & MSG_EXT_PPR_DT_REQ)
2570                                       ? " DT" : "", offset);
2571                        } else {
2572                                printk("%s: target %d using "
2573                                       "asynchronous transfers\n",
2574                                       ahc_name(ahc), devinfo->target);
2575                        }
2576                }
2577        }
2578
2579        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2580                                                tinfo, AHC_NEG_TO_GOAL);
2581
2582        if (update_needed)
2583                ahc_update_pending_scbs(ahc);
2584}
2585
2586/*
2587 * Update the user/goal/curr tables of wide negotiation
2588 * parameters as well as, in the case of a current or active update,
2589 * any data structures on the host controller.  In the case of an
2590 * active update, the specified target is currently talking to us on
2591 * the bus, so the transfer parameter update must take effect
2592 * immediately.
2593 */
2594void
2595ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2596              u_int width, u_int type, int paused)
2597{
2598        struct  ahc_initiator_tinfo *tinfo;
2599        struct  ahc_tmode_tstate *tstate;
2600        u_int   oldwidth;
2601        int     active;
2602        int     update_needed;
2603
2604        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2605        update_needed = 0;
2606        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2607                                    devinfo->target, &tstate);
2608
2609        if ((type & AHC_TRANS_USER) != 0)
2610                tinfo->user.width = width;
2611
2612        if ((type & AHC_TRANS_GOAL) != 0)
2613                tinfo->goal.width = width;
2614
2615        oldwidth = tinfo->curr.width;
2616        if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2617                u_int   scsirate;
2618
2619                update_needed++;
2620                scsirate =  tinfo->scsirate;
2621                scsirate &= ~WIDEXFER;
2622                if (width == MSG_EXT_WDTR_BUS_16_BIT)
2623                        scsirate |= WIDEXFER;
2624
2625                tinfo->scsirate = scsirate;
2626
2627                if (active)
2628                        ahc_outb(ahc, SCSIRATE, scsirate);
2629
2630                tinfo->curr.width = width;
2631
2632                ahc_send_async(ahc, devinfo->channel, devinfo->target,
2633                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2634                if (bootverbose) {
2635                        printk("%s: target %d using %dbit transfers\n",
2636                               ahc_name(ahc), devinfo->target,
2637                               8 * (0x01 << width));
2638                }
2639        }
2640
2641        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2642                                                tinfo, AHC_NEG_TO_GOAL);
2643        if (update_needed)
2644                ahc_update_pending_scbs(ahc);
2645}
2646
2647/*
2648 * Update the current state of tagged queuing for a given target.
2649 */
2650static void
2651ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2652             struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2653{
2654        struct scsi_device *sdev = cmd->device;
2655
2656        ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2657        ahc_send_async(ahc, devinfo->channel, devinfo->target,
2658                       devinfo->lun, AC_TRANSFER_NEG);
2659}
2660
2661/*
2662 * When the transfer settings for a connection change, update any
2663 * in-transit SCBs to contain the new data so the hardware will
2664 * be set correctly during future (re)selections.
2665 */
2666static void
2667ahc_update_pending_scbs(struct ahc_softc *ahc)
2668{
2669        struct  scb *pending_scb;
2670        int     pending_scb_count;
2671        int     i;
2672        int     paused;
2673        u_int   saved_scbptr;
2674
2675        /*
2676         * Traverse the pending SCB list and ensure that all of the
2677         * SCBs there have the proper settings.
2678         */
2679        pending_scb_count = 0;
2680        LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2681                struct ahc_devinfo devinfo;
2682                struct hardware_scb *pending_hscb;
2683                struct ahc_initiator_tinfo *tinfo;
2684                struct ahc_tmode_tstate *tstate;
2685
2686                ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2687                tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2688                                            devinfo.our_scsiid,
2689                                            devinfo.target, &tstate);
2690                pending_hscb = pending_scb->hscb;
2691                pending_hscb->control &= ~ULTRAENB;
2692                if ((tstate->ultraenb & devinfo.target_mask) != 0)
2693                        pending_hscb->control |= ULTRAENB;
2694                pending_hscb->scsirate = tinfo->scsirate;
2695                pending_hscb->scsioffset = tinfo->curr.offset;
2696                if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2697                 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2698                        pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2699                        pending_hscb->control &= ~MK_MESSAGE;
2700                }
2701                ahc_sync_scb(ahc, pending_scb,
2702                             BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2703                pending_scb_count++;
2704        }
2705
2706        if (pending_scb_count == 0)
2707                return;
2708
2709        if (ahc_is_paused(ahc)) {
2710                paused = 1;
2711        } else {
2712                paused = 0;
2713                ahc_pause(ahc);
2714        }
2715
2716        saved_scbptr = ahc_inb(ahc, SCBPTR);
2717        /* Ensure that the hscbs down on the card match the new information */
2718        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2719                struct  hardware_scb *pending_hscb;
2720                u_int   control;
2721                u_int   scb_tag;
2722
2723                ahc_outb(ahc, SCBPTR, i);
2724                scb_tag = ahc_inb(ahc, SCB_TAG);
2725                pending_scb = ahc_lookup_scb(ahc, scb_tag);
2726                if (pending_scb == NULL)
2727                        continue;
2728
2729                pending_hscb = pending_scb->hscb;
2730                control = ahc_inb(ahc, SCB_CONTROL);
2731                control &= ~(ULTRAENB|MK_MESSAGE);
2732                control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2733                ahc_outb(ahc, SCB_CONTROL, control);
2734                ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2735                ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2736        }
2737        ahc_outb(ahc, SCBPTR, saved_scbptr);
2738
2739        if (paused == 0)
2740                ahc_unpause(ahc);
2741}
2742
2743/**************************** Pathing Information *****************************/
2744static void
2745ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2746{
2747        u_int   saved_scsiid;
2748        role_t  role;
2749        int     our_id;
2750
2751        if (ahc_inb(ahc, SSTAT0) & TARGET)
2752                role = ROLE_TARGET;
2753        else
2754                role = ROLE_INITIATOR;
2755
2756        if (role == ROLE_TARGET
2757            && (ahc->features & AHC_MULTI_TID) != 0
2758            && (ahc_inb(ahc, SEQ_FLAGS)
2759             & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2760                /* We were selected, so pull our id from TARGIDIN */
2761                our_id = ahc_inb(ahc, TARGIDIN) & OID;
2762        } else if ((ahc->features & AHC_ULTRA2) != 0)
2763                our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2764        else
2765                our_id = ahc_inb(ahc, SCSIID) & OID;
2766
2767        saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2768        ahc_compile_devinfo(devinfo,
2769                            our_id,
2770                            SCSIID_TARGET(ahc, saved_scsiid),
2771                            ahc_inb(ahc, SAVED_LUN),
2772                            SCSIID_CHANNEL(ahc, saved_scsiid),
2773                            role);
2774}
2775
2776static const struct ahc_phase_table_entry*
2777ahc_lookup_phase_entry(int phase)
2778{
2779        const struct ahc_phase_table_entry *entry;
2780        const struct ahc_phase_table_entry *last_entry;
2781
2782        /*
2783         * num_phases doesn't include the default entry which
2784         * will be returned if the phase doesn't match.
2785         */
2786        last_entry = &ahc_phase_table[num_phases];
2787        for (entry = ahc_phase_table; entry < last_entry; entry++) {
2788                if (phase == entry->phase)
2789                        break;
2790        }
2791        return (entry);
2792}
2793
2794void
2795ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2796                    u_int lun, char channel, role_t role)
2797{
2798        devinfo->our_scsiid = our_id;
2799        devinfo->target = target;
2800        devinfo->lun = lun;
2801        devinfo->target_offset = target;
2802        devinfo->channel = channel;
2803        devinfo->role = role;
2804        if (channel == 'B')
2805                devinfo->target_offset += 8;
2806        devinfo->target_mask = (0x01 << devinfo->target_offset);
2807}
2808
2809void
2810ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2811{
2812        printk("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2813               devinfo->target, devinfo->lun);
2814}
2815
2816static void
2817ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2818                struct scb *scb)
2819{
2820        role_t  role;
2821        int     our_id;
2822
2823        our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2824        role = ROLE_INITIATOR;
2825        if ((scb->flags & SCB_TARGET_SCB) != 0)
2826                role = ROLE_TARGET;
2827        ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2828                            SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2829}
2830
2831
2832/************************ Message Phase Processing ****************************/
2833static void
2834ahc_assert_atn(struct ahc_softc *ahc)
2835{
2836        u_int scsisigo;
2837
2838        scsisigo = ATNO;
2839        if ((ahc->features & AHC_DT) == 0)
2840                scsisigo |= ahc_inb(ahc, SCSISIGI);
2841        ahc_outb(ahc, SCSISIGO, scsisigo);
2842}
2843
2844/*
2845 * When an initiator transaction with the MK_MESSAGE flag either reconnects
2846 * or enters the initial message out phase, we are interrupted.  Fill our
2847 * outgoing message buffer with the appropriate message and beging handing
2848 * the message phase(s) manually.
2849 */
2850static void
2851ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2852                           struct scb *scb)
2853{
2854        /*
2855         * To facilitate adding multiple messages together,
2856         * each routine should increment the index and len
2857         * variables instead of setting them explicitly.
2858         */
2859        ahc->msgout_index = 0;
2860        ahc->msgout_len = 0;
2861
2862        if ((scb->flags & SCB_DEVICE_RESET) == 0
2863         && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2864                u_int identify_msg;
2865
2866                identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2867                if ((scb->hscb->control & DISCENB) != 0)
2868                        identify_msg |= MSG_IDENTIFY_DISCFLAG;
2869                ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2870                ahc->msgout_len++;
2871
2872                if ((scb->hscb->control & TAG_ENB) != 0) {
2873                        ahc->msgout_buf[ahc->msgout_index++] =
2874                            scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2875                        ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2876                        ahc->msgout_len += 2;
2877                }
2878        }
2879
2880        if (scb->flags & SCB_DEVICE_RESET) {
2881                ahc->msgout_buf[ahc->msgout_index++] = TARGET_RESET;
2882                ahc->msgout_len++;
2883                ahc_print_path(ahc, scb);
2884                printk("Bus Device Reset Message Sent\n");
2885                /*
2886                 * Clear our selection hardware in advance of
2887                 * the busfree.  We may have an entry in the waiting
2888                 * Q for this target, and we don't want to go about
2889                 * selecting while we handle the busfree and blow it
2890                 * away.
2891                 */
2892                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2893        } else if ((scb->flags & SCB_ABORT) != 0) {
2894                if ((scb->hscb->control & TAG_ENB) != 0)
2895                        ahc->msgout_buf[ahc->msgout_index++] = ABORT_TASK;
2896                else
2897                        ahc->msgout_buf[ahc->msgout_index++] = ABORT_TASK_SET;
2898                ahc->msgout_len++;
2899                ahc_print_path(ahc, scb);
2900                printk("Abort%s Message Sent\n",
2901                       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2902                /*
2903                 * Clear our selection hardware in advance of
2904                 * the busfree.  We may have an entry in the waiting
2905                 * Q for this target, and we don't want to go about
2906                 * selecting while we handle the busfree and blow it
2907                 * away.
2908                 */
2909                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2910        } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2911                ahc_build_transfer_msg(ahc, devinfo);
2912        } else {
2913                printk("ahc_intr: AWAITING_MSG for an SCB that "
2914                       "does not have a waiting message\n");
2915                printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2916                       devinfo->target_mask);
2917                panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2918                      "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2919                      ahc_inb(ahc, MSG_OUT), scb->flags);
2920        }
2921
2922        /*
2923         * Clear the MK_MESSAGE flag from the SCB so we aren't
2924         * asked to send this message again.
2925         */
2926        ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2927        scb->hscb->control &= ~MK_MESSAGE;
2928        ahc->msgout_index = 0;
2929        ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2930}
2931
2932/*
2933 * Build an appropriate transfer negotiation message for the
2934 * currently active target.
2935 */
2936static void
2937ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2938{
2939        /*
2940         * We need to initiate transfer negotiations.
2941         * If our current and goal settings are identical,
2942         * we want to renegotiate due to a check condition.
2943         */
2944        struct  ahc_initiator_tinfo *tinfo;
2945        struct  ahc_tmode_tstate *tstate;
2946        const struct ahc_syncrate *rate;
2947        int     dowide;
2948        int     dosync;
2949        int     doppr;
2950        u_int   period;
2951        u_int   ppr_options;
2952        u_int   offset;
2953
2954        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2955                                    devinfo->target, &tstate);
2956        /*
2957         * Filter our period based on the current connection.
2958         * If we can't perform DT transfers on this segment (not in LVD
2959         * mode for instance), then our decision to issue a PPR message
2960         * may change.
2961         */
2962        period = tinfo->goal.period;
2963        offset = tinfo->goal.offset;
2964        ppr_options = tinfo->goal.ppr_options;
2965        /* Target initiated PPR is not allowed in the SCSI spec */
2966        if (devinfo->role == ROLE_TARGET)
2967                ppr_options = 0;
2968        rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2969                                       &ppr_options, devinfo->role);
2970        dowide = tinfo->curr.width != tinfo->goal.width;
2971        dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2972        /*
2973         * Only use PPR if we have options that need it, even if the device
2974         * claims to support it.  There might be an expander in the way
2975         * that doesn't.
2976         */
2977        doppr = ppr_options != 0;
2978
2979        if (!dowide && !dosync && !doppr) {
2980                dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2981                dosync = tinfo->goal.offset != 0;
2982        }
2983
2984        if (!dowide && !dosync && !doppr) {
2985                /*
2986                 * Force async with a WDTR message if we have a wide bus,
2987                 * or just issue an SDTR with a 0 offset.
2988                 */
2989                if ((ahc->features & AHC_WIDE) != 0)
2990                        dowide = 1;
2991                else
2992                        dosync = 1;
2993
2994                if (bootverbose) {
2995                        ahc_print_devinfo(ahc, devinfo);
2996                        printk("Ensuring async\n");
2997                }
2998        }
2999
3000        /* Target initiated PPR is not allowed in the SCSI spec */
3001        if (devinfo->role == ROLE_TARGET)
3002                doppr = 0;
3003
3004        /*
3005         * Both the PPR message and SDTR message require the
3006         * goal syncrate to be limited to what the target device
3007         * is capable of handling (based on whether an LVD->SE
3008         * expander is on the bus), so combine these two cases.
3009         * Regardless, guarantee that if we are using WDTR and SDTR
3010         * messages that WDTR comes first.
3011         */
3012        if (doppr || (dosync && !dowide)) {
3013
3014                offset = tinfo->goal.offset;
3015                ahc_validate_offset(ahc, tinfo, rate, &offset,
3016                                    doppr ? tinfo->goal.width
3017                                          : tinfo->curr.width,
3018                                    devinfo->role);
3019                if (doppr) {
3020                        ahc_construct_ppr(ahc, devinfo, period, offset,
3021                                          tinfo->goal.width, ppr_options);
3022                } else {
3023                        ahc_construct_sdtr(ahc, devinfo, period, offset);
3024                }
3025        } else {
3026                ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
3027        }
3028}
3029
3030/*
3031 * Build a synchronous negotiation message in our message
3032 * buffer based on the input parameters.
3033 */
3034static void
3035ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3036                   u_int period, u_int offset)
3037{
3038        if (offset == 0)
3039                period = AHC_ASYNC_XFER_PERIOD;
3040        ahc->msgout_index += spi_populate_sync_msg(
3041                        ahc->msgout_buf + ahc->msgout_index, period, offset);
3042        ahc->msgout_len += 5;
3043        if (bootverbose) {
3044                printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3045                       ahc_name(ahc), devinfo->channel, devinfo->target,
3046                       devinfo->lun, period, offset);
3047        }
3048}
3049
3050/*
3051 * Build a wide negotiation message in our message
3052 * buffer based on the input parameters.
3053 */
3054static void
3055ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3056                   u_int bus_width)
3057{
3058        ahc->msgout_index += spi_populate_width_msg(
3059                        ahc->msgout_buf + ahc->msgout_index, bus_width);
3060        ahc->msgout_len += 4;
3061        if (bootverbose) {
3062                printk("(%s:%c:%d:%d): Sending WDTR %x\n",
3063                       ahc_name(ahc), devinfo->channel, devinfo->target,
3064                       devinfo->lun, bus_width);
3065        }
3066}
3067
3068/*
3069 * Build a parallel protocol request message in our message
3070 * buffer based on the input parameters.
3071 */
3072static void
3073ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3074                  u_int period, u_int offset, u_int bus_width,
3075                  u_int ppr_options)
3076{
3077        if (offset == 0)
3078                period = AHC_ASYNC_XFER_PERIOD;
3079        ahc->msgout_index += spi_populate_ppr_msg(
3080                        ahc->msgout_buf + ahc->msgout_index, period, offset,
3081                        bus_width, ppr_options);
3082        ahc->msgout_len += 8;
3083        if (bootverbose) {
3084                printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3085                       "offset %x, ppr_options %x\n", ahc_name(ahc),
3086                       devinfo->channel, devinfo->target, devinfo->lun,
3087                       bus_width, period, offset, ppr_options);
3088        }
3089}
3090
3091/*
3092 * Clear any active message state.
3093 */
3094static void
3095ahc_clear_msg_state(struct ahc_softc *ahc)
3096{
3097        ahc->msgout_len = 0;
3098        ahc->msgin_index = 0;
3099        ahc->msg_type = MSG_TYPE_NONE;
3100        if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
3101                /*
3102                 * The target didn't care to respond to our
3103                 * message request, so clear ATN.
3104                 */
3105                ahc_outb(ahc, CLRSINT1, CLRATNO);
3106        }
3107        ahc_outb(ahc, MSG_OUT, NOP);
3108        ahc_outb(ahc, SEQ_FLAGS2,
3109                 ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3110}
3111
3112static void
3113ahc_handle_proto_violation(struct ahc_softc *ahc)
3114{
3115        struct  ahc_devinfo devinfo;
3116        struct  scb *scb;
3117        u_int   scbid;
3118        u_int   seq_flags;
3119        u_int   curphase;
3120        u_int   lastphase;
3121        int     found;
3122
3123        ahc_fetch_devinfo(ahc, &devinfo);
3124        scbid = ahc_inb(ahc, SCB_TAG);
3125        scb = ahc_lookup_scb(ahc, scbid);
3126        seq_flags = ahc_inb(ahc, SEQ_FLAGS);
3127        curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3128        lastphase = ahc_inb(ahc, LASTPHASE);
3129        if ((seq_flags & NOT_IDENTIFIED) != 0) {
3130
3131                /*
3132                 * The reconnecting target either did not send an
3133                 * identify message, or did, but we didn't find an SCB
3134                 * to match.
3135                 */
3136                ahc_print_devinfo(ahc, &devinfo);
3137                printk("Target did not send an IDENTIFY message. "
3138                       "LASTPHASE = 0x%x.\n", lastphase);
3139                scb = NULL;
3140        } else if (scb == NULL) {
3141                /*
3142                 * We don't seem to have an SCB active for this
3143                 * transaction.  Print an error and reset the bus.
3144                 */
3145                ahc_print_devinfo(ahc, &devinfo);
3146                printk("No SCB found during protocol violation\n");
3147                goto proto_violation_reset;
3148        } else {
3149                ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3150                if ((seq_flags & NO_CDB_SENT) != 0) {
3151                        ahc_print_path(ahc, scb);
3152                        printk("No or incomplete CDB sent to device.\n");
3153                } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
3154                        /*
3155                         * The target never bothered to provide status to
3156                         * us prior to completing the command.  Since we don't
3157                         * know the disposition of this command, we must attempt
3158                         * to abort it.  Assert ATN and prepare to send an abort
3159                         * message.
3160                         */
3161                        ahc_print_path(ahc, scb);
3162                        printk("Completed command without status.\n");
3163                } else {
3164                        ahc_print_path(ahc, scb);
3165                        printk("Unknown protocol violation.\n");
3166                        ahc_dump_card_state(ahc);
3167                }
3168        }
3169        if ((lastphase & ~P_DATAIN_DT) == 0
3170         || lastphase == P_COMMAND) {
3171proto_violation_reset:
3172                /*
3173                 * Target either went directly to data/command
3174                 * phase or didn't respond to our ATN.
3175                 * The only safe thing to do is to blow
3176                 * it away with a bus reset.
3177                 */
3178                found = ahc_reset_channel(ahc, 'A', TRUE);
3179                printk("%s: Issued Channel %c Bus Reset. "
3180                       "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
3181        } else {
3182                /*
3183                 * Leave the selection hardware off in case
3184                 * this abort attempt will affect yet to
3185                 * be sent commands.
3186                 */
3187                ahc_outb(ahc, SCSISEQ,
3188                         ahc_inb(ahc, SCSISEQ) & ~ENSELO);
3189                ahc_assert_atn(ahc);
3190                ahc_outb(ahc, MSG_OUT, HOST_MSG);
3191                if (scb == NULL) {
3192                        ahc_print_devinfo(ahc, &devinfo);
3193                        ahc->msgout_buf[0] = ABORT_TASK;
3194                        ahc->msgout_len = 1;
3195                        ahc->msgout_index = 0;
3196                        ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3197                } else {
3198                        ahc_print_path(ahc, scb);
3199                        scb->flags |= SCB_ABORT;
3200                }
3201                printk("Protocol violation %s.  Attempting to abort.\n",
3202                       ahc_lookup_phase_entry(curphase)->phasemsg);
3203        }
3204}
3205
3206/*
3207 * Manual message loop handler.
3208 */
3209static void
3210ahc_handle_message_phase(struct ahc_softc *ahc)
3211{
3212        struct  ahc_devinfo devinfo;
3213        u_int   bus_phase;
3214        int     end_session;
3215
3216        ahc_fetch_devinfo(ahc, &devinfo);
3217        end_session = FALSE;
3218        bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3219
3220reswitch:
3221        switch (ahc->msg_type) {
3222        case MSG_TYPE_INITIATOR_MSGOUT:
3223        {
3224                int lastbyte;
3225                int phasemis;
3226                int msgdone;
3227
3228                if (ahc->msgout_len == 0)
3229                        panic("HOST_MSG_LOOP interrupt with no active message");
3230
3231#ifdef AHC_DEBUG
3232                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3233                        ahc_print_devinfo(ahc, &devinfo);
3234                        printk("INITIATOR_MSG_OUT");
3235                }
3236#endif
3237                phasemis = bus_phase != P_MESGOUT;
3238                if (phasemis) {
3239#ifdef AHC_DEBUG
3240                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3241                                printk(" PHASEMIS %s\n",
3242                                       ahc_lookup_phase_entry(bus_phase)
3243                                                             ->phasemsg);
3244                        }
3245#endif
3246                        if (bus_phase == P_MESGIN) {
3247                                /*
3248                                 * Change gears and see if
3249                                 * this messages is of interest to
3250                                 * us or should be passed back to
3251                                 * the sequencer.
3252                                 */
3253                                ahc_outb(ahc, CLRSINT1, CLRATNO);
3254                                ahc->send_msg_perror = FALSE;
3255                                ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3256                                ahc->msgin_index = 0;
3257                                goto reswitch;
3258                        }
3259                        end_session = TRUE;
3260                        break;
3261                }
3262
3263                if (ahc->send_msg_perror) {
3264                        ahc_outb(ahc, CLRSINT1, CLRATNO);
3265                        ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3266#ifdef AHC_DEBUG
3267                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3268                                printk(" byte 0x%x\n", ahc->send_msg_perror);
3269#endif
3270                        ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
3271                        break;
3272                }
3273
3274                msgdone = ahc->msgout_index == ahc->msgout_len;
3275                if (msgdone) {
3276                        /*
3277                         * The target has requested a retry.
3278                         * Re-assert ATN, reset our message index to
3279                         * 0, and try again.
3280                         */
3281                        ahc->msgout_index = 0;
3282                        ahc_assert_atn(ahc);
3283                }
3284
3285                lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
3286                if (lastbyte) {
3287                        /* Last byte is signified by dropping ATN */
3288                        ahc_outb(ahc, CLRSINT1, CLRATNO);
3289                }
3290
3291                /*
3292                 * Clear our interrupt status and present
3293                 * the next byte on the bus.
3294                 */
3295                ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3296#ifdef AHC_DEBUG
3297                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3298                        printk(" byte 0x%x\n",
3299                               ahc->msgout_buf[ahc->msgout_index]);
3300#endif
3301                ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3302                break;
3303        }
3304        case MSG_TYPE_INITIATOR_MSGIN:
3305        {
3306                int phasemis;
3307                int message_done;
3308
3309#ifdef AHC_DEBUG
3310                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3311                        ahc_print_devinfo(ahc, &devinfo);
3312                        printk("INITIATOR_MSG_IN");
3313                }
3314#endif
3315                phasemis = bus_phase != P_MESGIN;
3316                if (phasemis) {
3317#ifdef AHC_DEBUG
3318                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3319                                printk(" PHASEMIS %s\n",
3320                                       ahc_lookup_phase_entry(bus_phase)
3321                                                             ->phasemsg);
3322                        }
3323#endif
3324                        ahc->msgin_index = 0;
3325                        if (bus_phase == P_MESGOUT
3326                         && (ahc->send_msg_perror == TRUE
3327                          || (ahc->msgout_len != 0
3328                           && ahc->msgout_index == 0))) {
3329                                ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3330                                goto reswitch;
3331                        }
3332                        end_session = TRUE;
3333                        break;
3334                }
3335
3336                /* Pull the byte in without acking it */
3337                ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
3338#ifdef AHC_DEBUG
3339                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3340                        printk(" byte 0x%x\n",
3341                               ahc->msgin_buf[ahc->msgin_index]);
3342#endif
3343
3344                message_done = ahc_parse_msg(ahc, &devinfo);
3345
3346                if (message_done) {
3347                        /*
3348                         * Clear our incoming message buffer in case there
3349                         * is another message following this one.
3350                         */
3351                        ahc->msgin_index = 0;
3352
3353                        /*
3354                         * If this message illicited a response,
3355                         * assert ATN so the target takes us to the
3356                         * message out phase.
3357                         */
3358                        if (ahc->msgout_len != 0) {
3359#ifdef AHC_DEBUG
3360                                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3361                                        ahc_print_devinfo(ahc, &devinfo);
3362                                        printk("Asserting ATN for response\n");
3363                                }
3364#endif
3365                                ahc_assert_atn(ahc);
3366                        }
3367                } else
3368                        ahc->msgin_index++;
3369
3370                if (message_done == MSGLOOP_TERMINATED) {
3371                        end_session = TRUE;
3372                } else {
3373                        /* Ack the byte */
3374                        ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3375                        ahc_inb(ahc, SCSIDATL);
3376                }
3377                break;
3378        }
3379        case MSG_TYPE_TARGET_MSGIN:
3380        {
3381                int msgdone;
3382                int msgout_request;
3383
3384                if (ahc->msgout_len == 0)
3385                        panic("Target MSGIN with no active message");
3386
3387                /*
3388                 * If we interrupted a mesgout session, the initiator
3389                 * will not know this until our first REQ.  So, we
3390                 * only honor mesgout requests after we've sent our
3391                 * first byte.
3392                 */
3393                if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
3394                 && ahc->msgout_index > 0)
3395                        msgout_request = TRUE;
3396                else
3397                        msgout_request = FALSE;
3398
3399                if (msgout_request) {
3400
3401                        /*
3402                         * Change gears and see if
3403                         * this messages is of interest to
3404                         * us or should be passed back to
3405                         * the sequencer.
3406                         */
3407                        ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
3408                        ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
3409                        ahc->msgin_index = 0;
3410                        /* Dummy read to REQ for first byte */
3411                        ahc_inb(ahc, SCSIDATL);
3412                        ahc_outb(ahc, SXFRCTL0,
3413                                 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3414                        break;
3415                }
3416
3417                msgdone = ahc->msgout_index == ahc->msgout_len;
3418                if (msgdone) {
3419                        ahc_outb(ahc, SXFRCTL0,
3420                                 ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3421                        end_session = TRUE;
3422                        break;
3423                }
3424
3425                /*
3426                 * Present the next byte on the bus.
3427                 */
3428                ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3429                ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3430                break;
3431        }
3432        case MSG_TYPE_TARGET_MSGOUT:
3433        {
3434                int lastbyte;
3435                int msgdone;
3436
3437                /*
3438                 * The initiator signals that this is
3439                 * the last byte by dropping ATN.
3440                 */
3441                lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
3442
3443                /*
3444                 * Read the latched byte, but turn off SPIOEN first
3445                 * so that we don't inadvertently cause a REQ for the
3446                 * next byte.
3447                 */
3448                ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3449                ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
3450                msgdone = ahc_parse_msg(ahc, &devinfo);
3451                if (msgdone == MSGLOOP_TERMINATED) {
3452                        /*
3453                         * The message is *really* done in that it caused
3454                         * us to go to bus free.  The sequencer has already
3455                         * been reset at this point, so pull the ejection
3456                         * handle.
3457                         */
3458                        return;
3459                }
3460
3461                ahc->msgin_index++;
3462
3463                /*
3464                 * XXX Read spec about initiator dropping ATN too soon
3465                 *     and use msgdone to detect it.
3466                 */
3467                if (msgdone == MSGLOOP_MSGCOMPLETE) {
3468                        ahc->msgin_index = 0;
3469
3470                        /*
3471                         * If this message illicited a response, transition
3472                         * to the Message in phase and send it.
3473                         */
3474                        if (ahc->msgout_len != 0) {
3475                                ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
3476                                ahc_outb(ahc, SXFRCTL0,
3477                                         ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3478                                ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3479                                ahc->msgin_index = 0;
3480                                break;
3481                        }
3482                }
3483
3484                if (lastbyte)
3485                        end_session = TRUE;
3486                else {
3487                        /* Ask for the next byte. */
3488                        ahc_outb(ahc, SXFRCTL0,
3489                                 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3490                }
3491
3492                break;
3493        }
3494        default:
3495                panic("Unknown REQINIT message type");
3496        }
3497
3498        if (end_session) {
3499                ahc_clear_msg_state(ahc);
3500                ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
3501        } else
3502                ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
3503}
3504
3505/*
3506 * See if we sent a particular extended message to the target.
3507 * If "full" is true, return true only if the target saw the full
3508 * message.  If "full" is false, return true if the target saw at
3509 * least the first byte of the message.
3510 */
3511static int
3512ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
3513{
3514        int found;
3515        u_int index;
3516
3517        found = FALSE;
3518        index = 0;
3519
3520        while (index < ahc->msgout_len) {
3521                if (ahc->msgout_buf[index] == EXTENDED_MESSAGE) {
3522                        u_int end_index;
3523
3524                        end_index = index + 1 + ahc->msgout_buf[index + 1];
3525                        if (ahc->msgout_buf[index+2] == msgval
3526                         && type == AHCMSG_EXT) {
3527
3528                                if (full) {
3529                                        if (ahc->msgout_index > end_index)
3530                                                found = TRUE;
3531                                } else if (ahc->msgout_index > index)
3532                                        found = TRUE;
3533                        }
3534                        index = end_index;
3535                } else if (ahc->msgout_buf[index] >= SIMPLE_QUEUE_TAG
3536                        && ahc->msgout_buf[index] <= IGNORE_WIDE_RESIDUE) {
3537
3538                        /* Skip tag type and tag id or residue param*/
3539                        index += 2;
3540                } else {
3541                        /* Single byte message */
3542                        if (type == AHCMSG_1B
3543                         && ahc->msgout_buf[index] == msgval
3544                         && ahc->msgout_index > index)
3545                                found = TRUE;
3546                        index++;
3547                }
3548
3549                if (found)
3550                        break;
3551        }
3552        return (found);
3553}
3554
3555/*
3556 * Wait for a complete incoming message, parse it, and respond accordingly.
3557 */
3558static int
3559ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3560{
3561        struct  ahc_initiator_tinfo *tinfo;
3562        struct  ahc_tmode_tstate *tstate;
3563        int     reject;
3564        int     done;
3565        int     response;
3566        u_int   targ_scsirate;
3567
3568        done = MSGLOOP_IN_PROG;
3569        response = FALSE;
3570        reject = FALSE;
3571        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3572                                    devinfo->target, &tstate);
3573        targ_scsirate = tinfo->scsirate;
3574
3575        /*
3576         * Parse as much of the message as is available,
3577         * rejecting it if we don't support it.  When
3578         * the entire message is available and has been
3579         * handled, return MSGLOOP_MSGCOMPLETE, indicating
3580         * that we have parsed an entire message.
3581         *
3582         * In the case of extended messages, we accept the length
3583         * byte outright and perform more checking once we know the
3584         * extended message type.
3585         */
3586        switch (ahc->msgin_buf[0]) {
3587        case DISCONNECT:
3588        case SAVE_POINTERS:
3589        case COMMAND_COMPLETE:
3590        case RESTORE_POINTERS:
3591        case IGNORE_WIDE_RESIDUE:
3592                /*
3593                 * End our message loop as these are messages
3594                 * the sequencer handles on its own.
3595                 */
3596                done = MSGLOOP_TERMINATED;
3597                break;
3598        case MESSAGE_REJECT:
3599                response = ahc_handle_msg_reject(ahc, devinfo);
3600                fallthrough;
3601        case NOP:
3602                done = MSGLOOP_MSGCOMPLETE;
3603                break;
3604        case EXTENDED_MESSAGE:
3605        {
3606                /* Wait for enough of the message to begin validation */
3607                if (ahc->msgin_index < 2)
3608                        break;
3609                switch (ahc->msgin_buf[2]) {
3610                case EXTENDED_SDTR:
3611                {
3612                        const struct ahc_syncrate *syncrate;
3613                        u_int    period;
3614                        u_int    ppr_options;
3615                        u_int    offset;
3616                        u_int    saved_offset;
3617
3618                        if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3619                                reject = TRUE;
3620                                break;
3621                        }
3622
3623                        /*
3624                         * Wait until we have both args before validating
3625                         * and acting on this message.
3626                         *
3627                         * Add one to MSG_EXT_SDTR_LEN to account for
3628                         * the extended message preamble.
3629                         */
3630                        if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3631                                break;
3632
3633                        period = ahc->msgin_buf[3];
3634                        ppr_options = 0;
3635                        saved_offset = offset = ahc->msgin_buf[4];
3636                        syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3637                                                           &ppr_options,
3638                                                           devinfo->role);
3639                        ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3640                                            targ_scsirate & WIDEXFER,
3641                                            devinfo->role);
3642                        if (bootverbose) {
3643                                printk("(%s:%c:%d:%d): Received "
3644                                       "SDTR period %x, offset %x\n\t"
3645                                       "Filtered to period %x, offset %x\n",
3646                                       ahc_name(ahc), devinfo->channel,
3647                                       devinfo->target, devinfo->lun,
3648                                       ahc->msgin_buf[3], saved_offset,
3649                                       period, offset);
3650                        }
3651                        ahc_set_syncrate(ahc, devinfo,
3652                                         syncrate, period,
3653                                         offset, ppr_options,
3654                                         AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3655                                         /*paused*/TRUE);
3656
3657                        /*
3658                         * See if we initiated Sync Negotiation
3659                         * and didn't have to fall down to async
3660                         * transfers.
3661                         */
3662                        if (ahc_sent_msg(ahc, AHCMSG_EXT, EXTENDED_SDTR, TRUE)) {
3663                                /* We started it */
3664                                if (saved_offset != offset) {
3665                                        /* Went too low - force async */
3666                                        reject = TRUE;
3667                                }
3668                        } else {
3669                                /*
3670                                 * Send our own SDTR in reply
3671                                 */
3672                                if (bootverbose
3673                                 && devinfo->role == ROLE_INITIATOR) {
3674                                        printk("(%s:%c:%d:%d): Target "
3675                                               "Initiated SDTR\n",
3676                                               ahc_name(ahc), devinfo->channel,
3677                                               devinfo->target, devinfo->lun);
3678                                }
3679                                ahc->msgout_index = 0;
3680                                ahc->msgout_len = 0;
3681                                ahc_construct_sdtr(ahc, devinfo,
3682                                                   period, offset);
3683                                ahc->msgout_index = 0;
3684                                response = TRUE;
3685                        }
3686                        done = MSGLOOP_MSGCOMPLETE;
3687                        break;
3688                }
3689                case EXTENDED_WDTR:
3690                {
3691                        u_int bus_width;
3692                        u_int saved_width;
3693                        u_int sending_reply;
3694
3695                        sending_reply = FALSE;
3696                        if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3697                                reject = TRUE;
3698                                break;
3699                        }
3700
3701                        /*
3702                         * Wait until we have our arg before validating
3703                         * and acting on this message.
3704                         *
3705                         * Add one to MSG_EXT_WDTR_LEN to account for
3706                         * the extended message preamble.
3707                         */
3708                        if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3709                                break;
3710
3711                        bus_width = ahc->msgin_buf[3];
3712                        saved_width = bus_width;
3713                        ahc_validate_width(ahc, tinfo, &bus_width,
3714                                           devinfo->role);
3715                        if (bootverbose) {
3716                                printk("(%s:%c:%d:%d): Received WDTR "
3717                                       "%x filtered to %x\n",
3718                                       ahc_name(ahc), devinfo->channel,
3719                                       devinfo->target, devinfo->lun,
3720                                       saved_width, bus_width);
3721                        }
3722
3723                        if (ahc_sent_msg(ahc, AHCMSG_EXT, EXTENDED_WDTR, TRUE)) {
3724                                /*
3725                                 * Don't send a WDTR back to the
3726                                 * target, since we asked first.
3727                                 * If the width went higher than our
3728                                 * request, reject it.
3729                                 */
3730                                if (saved_width > bus_width) {
3731                                        reject = TRUE;
3732                                        printk("(%s:%c:%d:%d): requested %dBit "
3733                                               "transfers.  Rejecting...\n",
3734                                               ahc_name(ahc), devinfo->channel,
3735                                               devinfo->target, devinfo->lun,
3736                                               8 * (0x01 << bus_width));
3737                                        bus_width = 0;
3738                                }
3739                        } else {
3740                                /*
3741                                 * Send our own WDTR in reply
3742                                 */
3743                                if (bootverbose
3744                                 && devinfo->role == ROLE_INITIATOR) {
3745                                        printk("(%s:%c:%d:%d): Target "
3746                                               "Initiated WDTR\n",
3747                                               ahc_name(ahc), devinfo->channel,
3748                                               devinfo->target, devinfo->lun);
3749                                }
3750                                ahc->msgout_index = 0;
3751                                ahc->msgout_len = 0;
3752                                ahc_construct_wdtr(ahc, devinfo, bus_width);
3753                                ahc->msgout_index = 0;
3754                                response = TRUE;
3755                                sending_reply = TRUE;
3756                        }
3757                        /*
3758                         * After a wide message, we are async, but
3759                         * some devices don't seem to honor this portion
3760                         * of the spec.  Force a renegotiation of the
3761                         * sync component of our transfer agreement even
3762                         * if our goal is async.  By updating our width
3763                         * after forcing the negotiation, we avoid
3764                         * renegotiating for width.
3765                         */
3766                        ahc_update_neg_request(ahc, devinfo, tstate,
3767                                               tinfo, AHC_NEG_ALWAYS);
3768                        ahc_set_width(ahc, devinfo, bus_width,
3769                                      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3770                                      /*paused*/TRUE);
3771                        if (sending_reply == FALSE && reject == FALSE) {
3772
3773                                /*
3774                                 * We will always have an SDTR to send.
3775                                 */
3776                                ahc->msgout_index = 0;
3777                                ahc->msgout_len = 0;
3778                                ahc_build_transfer_msg(ahc, devinfo);
3779                                ahc->msgout_index = 0;
3780                                response = TRUE;
3781                        }
3782                        done = MSGLOOP_MSGCOMPLETE;
3783                        break;
3784                }
3785                case EXTENDED_PPR:
3786                {
3787                        const struct ahc_syncrate *syncrate;
3788                        u_int   period;
3789                        u_int   offset;
3790                        u_int   bus_width;
3791                        u_int   ppr_options;
3792                        u_int   saved_width;
3793                        u_int   saved_offset;
3794                        u_int   saved_ppr_options;
3795
3796                        if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3797                                reject = TRUE;
3798                                break;
3799                        }
3800
3801                        /*
3802                         * Wait until we have all args before validating
3803                         * and acting on this message.
3804                         *
3805                         * Add one to MSG_EXT_PPR_LEN to account for
3806                         * the extended message preamble.
3807                         */
3808                        if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3809                                break;
3810
3811                        period = ahc->msgin_buf[3];
3812                        offset = ahc->msgin_buf[5];
3813                        bus_width = ahc->msgin_buf[6];
3814                        saved_width = bus_width;
3815                        ppr_options = ahc->msgin_buf[7];
3816                        /*
3817                         * According to the spec, a DT only
3818                         * period factor with no DT option
3819                         * set implies async.
3820                         */
3821                        if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3822                         && period == 9)
3823                                offset = 0;
3824                        saved_ppr_options = ppr_options;
3825                        saved_offset = offset;
3826
3827                        /*
3828                         * Mask out any options we don't support
3829                         * on any controller.  Transfer options are
3830                         * only available if we are negotiating wide.
3831                         */
3832                        ppr_options &= MSG_EXT_PPR_DT_REQ;
3833                        if (bus_width == 0)
3834                                ppr_options = 0;
3835
3836                        ahc_validate_width(ahc, tinfo, &bus_width,
3837                                           devinfo->role);
3838                        syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3839                                                           &ppr_options,
3840                                                           devinfo->role);
3841                        ahc_validate_offset(ahc, tinfo, syncrate,
3842                                            &offset, bus_width,
3843                                            devinfo->role);
3844
3845                        if (ahc_sent_msg(ahc, AHCMSG_EXT, EXTENDED_PPR, TRUE)) {
3846                                /*
3847                                 * If we are unable to do any of the
3848                                 * requested options (we went too low),
3849                                 * then we'll have to reject the message.
3850                                 */
3851                                if (saved_width > bus_width
3852                                 || saved_offset != offset
3853                                 || saved_ppr_options != ppr_options) {
3854                                        reject = TRUE;
3855                                        period = 0;
3856                                        offset = 0;
3857                                        bus_width = 0;
3858                                        ppr_options = 0;
3859                                        syncrate = NULL;
3860                                }
3861                        } else {
3862                                if (devinfo->role != ROLE_TARGET)
3863                                        printk("(%s:%c:%d:%d): Target "
3864                                               "Initiated PPR\n",
3865                                               ahc_name(ahc), devinfo->channel,
3866                                               devinfo->target, devinfo->lun);
3867                                else
3868                                        printk("(%s:%c:%d:%d): Initiator "
3869                                               "Initiated PPR\n",
3870                                               ahc_name(ahc), devinfo->channel,
3871                                               devinfo->target, devinfo->lun);
3872                                ahc->msgout_index = 0;
3873                                ahc->msgout_len = 0;
3874                                ahc_construct_ppr(ahc, devinfo, period, offset,
3875                                                  bus_width, ppr_options);
3876                                ahc->msgout_index = 0;
3877                                response = TRUE;
3878                        }
3879                        if (bootverbose) {
3880                                printk("(%s:%c:%d:%d): Received PPR width %x, "
3881                                       "period %x, offset %x,options %x\n"
3882                                       "\tFiltered to width %x, period %x, "
3883                                       "offset %x, options %x\n",
3884                                       ahc_name(ahc), devinfo->channel,
3885                                       devinfo->target, devinfo->lun,
3886                                       saved_width, ahc->msgin_buf[3],
3887                                       saved_offset, saved_ppr_options,
3888                                       bus_width, period, offset, ppr_options);
3889                        }
3890                        ahc_set_width(ahc, devinfo, bus_width,
3891                                      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3892                                      /*paused*/TRUE);
3893                        ahc_set_syncrate(ahc, devinfo,
3894                                         syncrate, period,
3895                                         offset, ppr_options,
3896                                         AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3897                                         /*paused*/TRUE);
3898                        done = MSGLOOP_MSGCOMPLETE;
3899                        break;
3900                }
3901                default:
3902                        /* Unknown extended message.  Reject it. */
3903                        reject = TRUE;
3904                        break;
3905                }
3906                break;
3907        }
3908#ifdef AHC_TARGET_MODE
3909        case TARGET_RESET:
3910                ahc_handle_devreset(ahc, devinfo,
3911                                    CAM_BDR_SENT,
3912                                    "Bus Device Reset Received",
3913                                    /*verbose_level*/0);
3914                ahc_restart(ahc);
3915                done = MSGLOOP_TERMINATED;
3916                break;
3917        case ABORT_TASK:
3918        case ABORT_TASK_SET:
3919        case