linux/drivers/scsi/aic7xxx/aic79xx_core.c
<<
>>
Prefs
   1/*
   2 * Core routines and tables shareable across OS platforms.
   3 *
   4 * Copyright (c) 1994-2002 Justin T. Gibbs.
   5 * Copyright (c) 2000-2003 Adaptec Inc.
   6 * All rights reserved.
   7 *
   8 * Redistribution and use in source and binary forms, with or without
   9 * modification, are permitted provided that the following conditions
  10 * are met:
  11 * 1. Redistributions of source code must retain the above copyright
  12 *    notice, this list of conditions, and the following disclaimer,
  13 *    without modification.
  14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15 *    substantially similar to the "NO WARRANTY" disclaimer below
  16 *    ("Disclaimer") and any redistribution must be conditioned upon
  17 *    including a substantially similar Disclaimer requirement for further
  18 *    binary redistribution.
  19 * 3. Neither the names of the above-listed copyright holders nor the names
  20 *    of any contributors may be used to endorse or promote products derived
  21 *    from this software without specific prior written permission.
  22 *
  23 * Alternatively, this software may be distributed under the terms of the
  24 * GNU General Public License ("GPL") version 2 as published by the Free
  25 * Software Foundation.
  26 *
  27 * NO WARRANTY
  28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 * POSSIBILITY OF SUCH DAMAGES.
  39 *
  40 * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
  41 */
  42
  43#include "aic79xx_osm.h"
  44#include "aic79xx_inline.h"
  45#include "aicasm/aicasm_insformat.h"
  46
  47/***************************** Lookup Tables **********************************/
  48static const char *const ahd_chip_names[] =
  49{
  50        "NONE",
  51        "aic7901",
  52        "aic7902",
  53        "aic7901A"
  54};
  55
  56/*
  57 * Hardware error codes.
  58 */
  59struct ahd_hard_error_entry {
  60        uint8_t errno;
  61        const char *errmesg;
  62};
  63
  64static const struct ahd_hard_error_entry ahd_hard_errors[] = {
  65        { DSCTMOUT,     "Discard Timer has timed out" },
  66        { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  67        { SQPARERR,     "Sequencer Parity Error" },
  68        { DPARERR,      "Data-path Parity Error" },
  69        { MPARERR,      "Scratch or SCB Memory Parity Error" },
  70        { CIOPARERR,    "CIOBUS Parity Error" },
  71};
  72static const u_int num_errors = ARRAY_SIZE(ahd_hard_errors);
  73
  74static const struct ahd_phase_table_entry ahd_phase_table[] =
  75{
  76        { P_DATAOUT,    NOP,                    "in Data-out phase"     },
  77        { P_DATAIN,     INITIATOR_ERROR,        "in Data-in phase"      },
  78        { P_DATAOUT_DT, NOP,                    "in DT Data-out phase"  },
  79        { P_DATAIN_DT,  INITIATOR_ERROR,        "in DT Data-in phase"   },
  80        { P_COMMAND,    NOP,                    "in Command phase"      },
  81        { P_MESGOUT,    NOP,                    "in Message-out phase"  },
  82        { P_STATUS,     INITIATOR_ERROR,        "in Status phase"       },
  83        { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  84        { P_BUSFREE,    NOP,                    "while idle"            },
  85        { 0,            NOP,                    "in unknown phase"      }
  86};
  87
  88/*
  89 * In most cases we only wish to itterate over real phases, so
  90 * exclude the last element from the count.
  91 */
  92static const u_int num_phases = ARRAY_SIZE(ahd_phase_table) - 1;
  93
  94/* Our Sequencer Program */
  95#include "aic79xx_seq.h"
  96
  97/**************************** Function Declarations ***************************/
  98static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
  99static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 100                                                  u_int lqistat1);
 101static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 102                                               u_int busfreetime);
 103static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 104static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 105static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 106                                                struct ahd_devinfo *devinfo);
 107
 108static struct ahd_tmode_tstate*
 109                        ahd_alloc_tstate(struct ahd_softc *ahd,
 110                                         u_int scsi_id, char channel);
 111#ifdef AHD_TARGET_MODE
 112static void             ahd_free_tstate(struct ahd_softc *ahd,
 113                                        u_int scsi_id, char channel, int force);
 114#endif
 115static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 116                                                struct ahd_initiator_tinfo *,
 117                                                u_int *period,
 118                                                u_int *ppr_options,
 119                                                role_t role);
 120static void             ahd_update_neg_table(struct ahd_softc *ahd,
 121                                             struct ahd_devinfo *devinfo,
 122                                             struct ahd_transinfo *tinfo);
 123static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 124static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 125                                          struct ahd_devinfo *devinfo);
 126static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 127                                        struct ahd_devinfo *devinfo,
 128                                        struct scb *scb);
 129static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 130                                                   struct ahd_devinfo *devinfo,
 131                                                   struct scb *scb);
 132static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 133                                               struct ahd_devinfo *devinfo);
 134static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 135                                           struct ahd_devinfo *devinfo,
 136                                           u_int period, u_int offset);
 137static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 138                                           struct ahd_devinfo *devinfo,
 139                                           u_int bus_width);
 140static void             ahd_construct_ppr(struct ahd_softc *ahd,
 141                                          struct ahd_devinfo *devinfo,
 142                                          u_int period, u_int offset,
 143                                          u_int bus_width, u_int ppr_options);
 144static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 145static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 146typedef enum {
 147        AHDMSG_1B,
 148        AHDMSG_2B,
 149        AHDMSG_EXT
 150} ahd_msgtype;
 151static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 152                                     u_int msgval, int full);
 153static int              ahd_parse_msg(struct ahd_softc *ahd,
 154                                      struct ahd_devinfo *devinfo);
 155static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 156                                              struct ahd_devinfo *devinfo);
 157static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 158                                                struct ahd_devinfo *devinfo);
 159static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 160static void             ahd_handle_devreset(struct ahd_softc *ahd,
 161                                            struct ahd_devinfo *devinfo,
 162                                            u_int lun, cam_status status,
 163                                            char *message, int verbose_level);
 164#ifdef AHD_TARGET_MODE
 165static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 166                                               struct ahd_devinfo *devinfo,
 167                                               struct scb *scb);
 168#endif
 169
 170static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 171static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 172static bus_dmamap_callback_t
 173                        ahd_dmamap_cb;
 174static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 175static int              ahd_init_scbdata(struct ahd_softc *ahd);
 176static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 177static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 178static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 179static void             ahd_add_col_list(struct ahd_softc *ahd,
 180                                         struct scb *scb, u_int col_idx);
 181static void             ahd_rem_col_list(struct ahd_softc *ahd,
 182                                         struct scb *scb);
 183static void             ahd_chip_init(struct ahd_softc *ahd);
 184static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 185                                            struct scb *prev_scb,
 186                                            struct scb *scb);
 187static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 188static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 189                                            char channel, int lun, u_int tag,
 190                                            role_t role, uint32_t status,
 191                                            ahd_search_action action,
 192                                            u_int *list_head, u_int *list_tail,
 193                                            u_int tid);
 194static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 195                                            u_int tid_prev, u_int tid_cur,
 196                                            u_int tid_next);
 197static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 198                                                 u_int scbid);
 199static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 200                                     u_int prev, u_int next, u_int tid);
 201static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 202static void             ahd_stat_timer(struct timer_list *t);
 203#ifdef AHD_DUMP_SEQ
 204static void             ahd_dumpseq(struct ahd_softc *ahd);
 205#endif
 206static void             ahd_loadseq(struct ahd_softc *ahd);
 207static int              ahd_check_patch(struct ahd_softc *ahd,
 208                                        const struct patch **start_patch,
 209                                        u_int start_instr, u_int *skip_addr);
 210static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 211                                            u_int address);
 212static void             ahd_download_instr(struct ahd_softc *ahd,
 213                                           u_int instrptr, uint8_t *dconsts);
 214static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 215static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 216                                               struct scb *scb);
 217static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 218                                          struct scb *scb);
 219
 220#ifdef AHD_TARGET_MODE
 221static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 222                                               struct ahd_tmode_lstate *lstate,
 223                                               u_int initiator_id,
 224                                               u_int event_type,
 225                                               u_int event_arg);
 226static void             ahd_update_scsiid(struct ahd_softc *ahd,
 227                                          u_int targid_mask);
 228static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 229                                              struct target_cmd *cmd);
 230#endif
 231
 232static int              ahd_abort_scbs(struct ahd_softc *ahd, int target,
 233                                       char channel, int lun, u_int tag,
 234                                       role_t role, uint32_t status);
 235static void             ahd_alloc_scbs(struct ahd_softc *ahd);
 236static void             ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl,
 237                                     u_int scbid);
 238static void             ahd_calc_residual(struct ahd_softc *ahd,
 239                                          struct scb *scb);
 240static void             ahd_clear_critical_section(struct ahd_softc *ahd);
 241static void             ahd_clear_intstat(struct ahd_softc *ahd);
 242static void             ahd_enable_coalescing(struct ahd_softc *ahd,
 243                                              int enable);
 244static u_int            ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl);
 245static void             ahd_freeze_devq(struct ahd_softc *ahd,
 246                                        struct scb *scb);
 247static void             ahd_handle_scb_status(struct ahd_softc *ahd,
 248                                              struct scb *scb);
 249static const struct ahd_phase_table_entry* ahd_lookup_phase_entry(int phase);
 250static void             ahd_shutdown(void *arg);
 251static void             ahd_update_coalescing_values(struct ahd_softc *ahd,
 252                                                     u_int timer,
 253                                                     u_int maxcmds,
 254                                                     u_int mincmds);
 255static int              ahd_verify_vpd_cksum(struct vpd_config *vpd);
 256static int              ahd_wait_seeprom(struct ahd_softc *ahd);
 257static int              ahd_match_scb(struct ahd_softc *ahd, struct scb *scb,
 258                                      int target, char channel, int lun,
 259                                      u_int tag, role_t role);
 260
 261static void             ahd_reset_cmds_pending(struct ahd_softc *ahd);
 262
 263/*************************** Interrupt Services *******************************/
 264static void             ahd_run_qoutfifo(struct ahd_softc *ahd);
 265#ifdef AHD_TARGET_MODE
 266static void             ahd_run_tqinfifo(struct ahd_softc *ahd, int paused);
 267#endif
 268static void             ahd_handle_hwerrint(struct ahd_softc *ahd);
 269static void             ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat);
 270static void             ahd_handle_scsiint(struct ahd_softc *ahd,
 271                                           u_int intstat);
 272
 273/************************ Sequencer Execution Control *************************/
 274void
 275ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst)
 276{
 277        if (ahd->src_mode == src && ahd->dst_mode == dst)
 278                return;
 279#ifdef AHD_DEBUG
 280        if (ahd->src_mode == AHD_MODE_UNKNOWN
 281         || ahd->dst_mode == AHD_MODE_UNKNOWN)
 282                panic("Setting mode prior to saving it.\n");
 283        if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 284                printk("%s: Setting mode 0x%x\n", ahd_name(ahd),
 285                       ahd_build_mode_state(ahd, src, dst));
 286#endif
 287        ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst));
 288        ahd->src_mode = src;
 289        ahd->dst_mode = dst;
 290}
 291
 292static void
 293ahd_update_modes(struct ahd_softc *ahd)
 294{
 295        ahd_mode_state mode_ptr;
 296        ahd_mode src;
 297        ahd_mode dst;
 298
 299        mode_ptr = ahd_inb(ahd, MODE_PTR);
 300#ifdef AHD_DEBUG
 301        if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 302                printk("Reading mode 0x%x\n", mode_ptr);
 303#endif
 304        ahd_extract_mode_state(ahd, mode_ptr, &src, &dst);
 305        ahd_known_modes(ahd, src, dst);
 306}
 307
 308static void
 309ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode,
 310                 ahd_mode dstmode, const char *file, int line)
 311{
 312#ifdef AHD_DEBUG
 313        if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0
 314         || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) {
 315                panic("%s:%s:%d: Mode assertion failed.\n",
 316                       ahd_name(ahd), file, line);
 317        }
 318#endif
 319}
 320
 321#define AHD_ASSERT_MODES(ahd, source, dest) \
 322        ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__);
 323
 324ahd_mode_state
 325ahd_save_modes(struct ahd_softc *ahd)
 326{
 327        if (ahd->src_mode == AHD_MODE_UNKNOWN
 328         || ahd->dst_mode == AHD_MODE_UNKNOWN)
 329                ahd_update_modes(ahd);
 330
 331        return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode));
 332}
 333
 334void
 335ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state)
 336{
 337        ahd_mode src;
 338        ahd_mode dst;
 339
 340        ahd_extract_mode_state(ahd, state, &src, &dst);
 341        ahd_set_modes(ahd, src, dst);
 342}
 343
 344/*
 345 * Determine whether the sequencer has halted code execution.
 346 * Returns non-zero status if the sequencer is stopped.
 347 */
 348int
 349ahd_is_paused(struct ahd_softc *ahd)
 350{
 351        return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0);
 352}
 353
 354/*
 355 * Request that the sequencer stop and wait, indefinitely, for it
 356 * to stop.  The sequencer will only acknowledge that it is paused
 357 * once it has reached an instruction boundary and PAUSEDIS is
 358 * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 359 * for critical sections.
 360 */
 361void
 362ahd_pause(struct ahd_softc *ahd)
 363{
 364        ahd_outb(ahd, HCNTRL, ahd->pause);
 365
 366        /*
 367         * Since the sequencer can disable pausing in a critical section, we
 368         * must loop until it actually stops.
 369         */
 370        while (ahd_is_paused(ahd) == 0)
 371                ;
 372}
 373
 374/*
 375 * Allow the sequencer to continue program execution.
 376 * We check here to ensure that no additional interrupt
 377 * sources that would cause the sequencer to halt have been
 378 * asserted.  If, for example, a SCSI bus reset is detected
 379 * while we are fielding a different, pausing, interrupt type,
 380 * we don't want to release the sequencer before going back
 381 * into our interrupt handler and dealing with this new
 382 * condition.
 383 */
 384void
 385ahd_unpause(struct ahd_softc *ahd)
 386{
 387        /*
 388         * Automatically restore our modes to those saved
 389         * prior to the first change of the mode.
 390         */
 391        if (ahd->saved_src_mode != AHD_MODE_UNKNOWN
 392         && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) {
 393                if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0)
 394                        ahd_reset_cmds_pending(ahd);
 395                ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
 396        }
 397
 398        if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0)
 399                ahd_outb(ahd, HCNTRL, ahd->unpause);
 400
 401        ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN);
 402}
 403
 404/*********************** Scatter Gather List Handling *************************/
 405void *
 406ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb,
 407             void *sgptr, dma_addr_t addr, bus_size_t len, int last)
 408{
 409        scb->sg_count++;
 410        if (sizeof(dma_addr_t) > 4
 411         && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 412                struct ahd_dma64_seg *sg;
 413
 414                sg = (struct ahd_dma64_seg *)sgptr;
 415                sg->addr = ahd_htole64(addr);
 416                sg->len = ahd_htole32(len | (last ? AHD_DMA_LAST_SEG : 0));
 417                return (sg + 1);
 418        } else {
 419                struct ahd_dma_seg *sg;
 420
 421                sg = (struct ahd_dma_seg *)sgptr;
 422                sg->addr = ahd_htole32(addr & 0xFFFFFFFF);
 423                sg->len = ahd_htole32(len | ((addr >> 8) & 0x7F000000)
 424                                    | (last ? AHD_DMA_LAST_SEG : 0));
 425                return (sg + 1);
 426        }
 427}
 428
 429static void
 430ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb)
 431{
 432        /* XXX Handle target mode SCBs. */
 433        scb->crc_retry_count = 0;
 434        if ((scb->flags & SCB_PACKETIZED) != 0) {
 435                /* XXX what about ACA??  It is type 4, but TAG_TYPE == 0x3. */
 436                scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE;
 437        } else {
 438                if (ahd_get_transfer_length(scb) & 0x01)
 439                        scb->hscb->task_attribute = SCB_XFERLEN_ODD;
 440                else
 441                        scb->hscb->task_attribute = 0;
 442        }
 443
 444        if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR
 445         || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0)
 446                scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr =
 447                    ahd_htole32(scb->sense_busaddr);
 448}
 449
 450static void
 451ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb)
 452{
 453        /*
 454         * Copy the first SG into the "current" data ponter area.
 455         */
 456        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 457                struct ahd_dma64_seg *sg;
 458
 459                sg = (struct ahd_dma64_seg *)scb->sg_list;
 460                scb->hscb->dataptr = sg->addr;
 461                scb->hscb->datacnt = sg->len;
 462        } else {
 463                struct ahd_dma_seg *sg;
 464                uint32_t *dataptr_words;
 465
 466                sg = (struct ahd_dma_seg *)scb->sg_list;
 467                dataptr_words = (uint32_t*)&scb->hscb->dataptr;
 468                dataptr_words[0] = sg->addr;
 469                dataptr_words[1] = 0;
 470                if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) {
 471                        uint64_t high_addr;
 472
 473                        high_addr = ahd_le32toh(sg->len) & 0x7F000000;
 474                        scb->hscb->dataptr |= ahd_htole64(high_addr << 8);
 475                }
 476                scb->hscb->datacnt = sg->len;
 477        }
 478        /*
 479         * Note where to find the SG entries in bus space.
 480         * We also set the full residual flag which the
 481         * sequencer will clear as soon as a data transfer
 482         * occurs.
 483         */
 484        scb->hscb->sgptr = ahd_htole32(scb->sg_list_busaddr|SG_FULL_RESID);
 485}
 486
 487static void
 488ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb)
 489{
 490        scb->hscb->sgptr = ahd_htole32(SG_LIST_NULL);
 491        scb->hscb->dataptr = 0;
 492        scb->hscb->datacnt = 0;
 493}
 494
 495/************************** Memory mapping routines ***************************/
 496static void *
 497ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr)
 498{
 499        dma_addr_t sg_offset;
 500
 501        /* sg_list_phys points to entry 1, not 0 */
 502        sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd));
 503        return ((uint8_t *)scb->sg_list + sg_offset);
 504}
 505
 506static uint32_t
 507ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg)
 508{
 509        dma_addr_t sg_offset;
 510
 511        /* sg_list_phys points to entry 1, not 0 */
 512        sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list)
 513                  - ahd_sg_size(ahd);
 514
 515        return (scb->sg_list_busaddr + sg_offset);
 516}
 517
 518static void
 519ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op)
 520{
 521        ahd_dmamap_sync(ahd, ahd->scb_data.hscb_dmat,
 522                        scb->hscb_map->dmamap,
 523                        /*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr,
 524                        /*len*/sizeof(*scb->hscb), op);
 525}
 526
 527void
 528ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op)
 529{
 530        if (scb->sg_count == 0)
 531                return;
 532
 533        ahd_dmamap_sync(ahd, ahd->scb_data.sg_dmat,
 534                        scb->sg_map->dmamap,
 535                        /*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd),
 536                        /*len*/ahd_sg_size(ahd) * scb->sg_count, op);
 537}
 538
 539static void
 540ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op)
 541{
 542        ahd_dmamap_sync(ahd, ahd->scb_data.sense_dmat,
 543                        scb->sense_map->dmamap,
 544                        /*offset*/scb->sense_busaddr,
 545                        /*len*/AHD_SENSE_BUFSIZE, op);
 546}
 547
 548#ifdef AHD_TARGET_MODE
 549static uint32_t
 550ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index)
 551{
 552        return (((uint8_t *)&ahd->targetcmds[index])
 553               - (uint8_t *)ahd->qoutfifo);
 554}
 555#endif
 556
 557/*********************** Miscellaneous Support Functions ***********************/
 558/*
 559 * Return pointers to the transfer negotiation information
 560 * for the specified our_id/remote_id pair.
 561 */
 562struct ahd_initiator_tinfo *
 563ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id,
 564                    u_int remote_id, struct ahd_tmode_tstate **tstate)
 565{
 566        /*
 567         * Transfer data structures are stored from the perspective
 568         * of the target role.  Since the parameters for a connection
 569         * in the initiator role to a given target are the same as
 570         * when the roles are reversed, we pretend we are the target.
 571         */
 572        if (channel == 'B')
 573                our_id += 8;
 574        *tstate = ahd->enabled_targets[our_id];
 575        return (&(*tstate)->transinfo[remote_id]);
 576}
 577
 578uint16_t
 579ahd_inw(struct ahd_softc *ahd, u_int port)
 580{
 581        /*
 582         * Read high byte first as some registers increment
 583         * or have other side effects when the low byte is
 584         * read.
 585         */
 586        uint16_t r = ahd_inb(ahd, port+1) << 8;
 587        return r | ahd_inb(ahd, port);
 588}
 589
 590void
 591ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
 592{
 593        /*
 594         * Write low byte first to accommodate registers
 595         * such as PRGMCNT where the order maters.
 596         */
 597        ahd_outb(ahd, port, value & 0xFF);
 598        ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 599}
 600
 601uint32_t
 602ahd_inl(struct ahd_softc *ahd, u_int port)
 603{
 604        return ((ahd_inb(ahd, port))
 605              | (ahd_inb(ahd, port+1) << 8)
 606              | (ahd_inb(ahd, port+2) << 16)
 607              | (ahd_inb(ahd, port+3) << 24));
 608}
 609
 610void
 611ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
 612{
 613        ahd_outb(ahd, port, (value) & 0xFF);
 614        ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
 615        ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
 616        ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
 617}
 618
 619uint64_t
 620ahd_inq(struct ahd_softc *ahd, u_int port)
 621{
 622        return ((ahd_inb(ahd, port))
 623              | (ahd_inb(ahd, port+1) << 8)
 624              | (ahd_inb(ahd, port+2) << 16)
 625              | (ahd_inb(ahd, port+3) << 24)
 626              | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
 627              | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
 628              | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
 629              | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
 630}
 631
 632void
 633ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
 634{
 635        ahd_outb(ahd, port, value & 0xFF);
 636        ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 637        ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
 638        ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
 639        ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
 640        ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
 641        ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
 642        ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
 643}
 644
 645u_int
 646ahd_get_scbptr(struct ahd_softc *ahd)
 647{
 648        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 649                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 650        return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
 651}
 652
 653void
 654ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
 655{
 656        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 657                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 658        ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
 659        ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
 660}
 661
 662#if 0 /* unused */
 663static u_int
 664ahd_get_hnscb_qoff(struct ahd_softc *ahd)
 665{
 666        return (ahd_inw_atomic(ahd, HNSCB_QOFF));
 667}
 668#endif
 669
 670static void
 671ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
 672{
 673        ahd_outw_atomic(ahd, HNSCB_QOFF, value);
 674}
 675
 676#if 0 /* unused */
 677static u_int
 678ahd_get_hescb_qoff(struct ahd_softc *ahd)
 679{
 680        return (ahd_inb(ahd, HESCB_QOFF));
 681}
 682#endif
 683
 684static void
 685ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
 686{
 687        ahd_outb(ahd, HESCB_QOFF, value);
 688}
 689
 690static u_int
 691ahd_get_snscb_qoff(struct ahd_softc *ahd)
 692{
 693        u_int oldvalue;
 694
 695        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 696        oldvalue = ahd_inw(ahd, SNSCB_QOFF);
 697        ahd_outw(ahd, SNSCB_QOFF, oldvalue);
 698        return (oldvalue);
 699}
 700
 701static void
 702ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
 703{
 704        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 705        ahd_outw(ahd, SNSCB_QOFF, value);
 706}
 707
 708#if 0 /* unused */
 709static u_int
 710ahd_get_sescb_qoff(struct ahd_softc *ahd)
 711{
 712        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 713        return (ahd_inb(ahd, SESCB_QOFF));
 714}
 715#endif
 716
 717static void
 718ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
 719{
 720        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 721        ahd_outb(ahd, SESCB_QOFF, value);
 722}
 723
 724#if 0 /* unused */
 725static u_int
 726ahd_get_sdscb_qoff(struct ahd_softc *ahd)
 727{
 728        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 729        return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
 730}
 731#endif
 732
 733static void
 734ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
 735{
 736        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 737        ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
 738        ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
 739}
 740
 741u_int
 742ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
 743{
 744        u_int value;
 745
 746        /*
 747         * Workaround PCI-X Rev A. hardware bug.
 748         * After a host read of SCB memory, the chip
 749         * may become confused into thinking prefetch
 750         * was required.  This starts the discard timer
 751         * running and can cause an unexpected discard
 752         * timer interrupt.  The work around is to read
 753         * a normal register prior to the exhaustion of
 754         * the discard timer.  The mode pointer register
 755         * has no side effects and so serves well for
 756         * this purpose.
 757         *
 758         * Razor #528
 759         */
 760        value = ahd_inb(ahd, offset);
 761        if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
 762                ahd_inb(ahd, MODE_PTR);
 763        return (value);
 764}
 765
 766u_int
 767ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
 768{
 769        return (ahd_inb_scbram(ahd, offset)
 770              | (ahd_inb_scbram(ahd, offset+1) << 8));
 771}
 772
 773static uint32_t
 774ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
 775{
 776        return (ahd_inw_scbram(ahd, offset)
 777              | (ahd_inw_scbram(ahd, offset+2) << 16));
 778}
 779
 780static uint64_t
 781ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
 782{
 783        return (ahd_inl_scbram(ahd, offset)
 784              | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
 785}
 786
 787struct scb *
 788ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
 789{
 790        struct scb* scb;
 791
 792        if (tag >= AHD_SCB_MAX)
 793                return (NULL);
 794        scb = ahd->scb_data.scbindex[tag];
 795        if (scb != NULL)
 796                ahd_sync_scb(ahd, scb,
 797                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 798        return (scb);
 799}
 800
 801static void
 802ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
 803{
 804        struct   hardware_scb *q_hscb;
 805        struct   map_node *q_hscb_map;
 806        uint32_t saved_hscb_busaddr;
 807
 808        /*
 809         * Our queuing method is a bit tricky.  The card
 810         * knows in advance which HSCB (by address) to download,
 811         * and we can't disappoint it.  To achieve this, the next
 812         * HSCB to download is saved off in ahd->next_queued_hscb.
 813         * When we are called to queue "an arbitrary scb",
 814         * we copy the contents of the incoming HSCB to the one
 815         * the sequencer knows about, swap HSCB pointers and
 816         * finally assign the SCB to the tag indexed location
 817         * in the scb_array.  This makes sure that we can still
 818         * locate the correct SCB by SCB_TAG.
 819         */
 820        q_hscb = ahd->next_queued_hscb;
 821        q_hscb_map = ahd->next_queued_hscb_map;
 822        saved_hscb_busaddr = q_hscb->hscb_busaddr;
 823        memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 824        q_hscb->hscb_busaddr = saved_hscb_busaddr;
 825        q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
 826
 827        /* Now swap HSCB pointers. */
 828        ahd->next_queued_hscb = scb->hscb;
 829        ahd->next_queued_hscb_map = scb->hscb_map;
 830        scb->hscb = q_hscb;
 831        scb->hscb_map = q_hscb_map;
 832
 833        /* Now define the mapping from tag to SCB in the scbindex */
 834        ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
 835}
 836
 837/*
 838 * Tell the sequencer about a new transaction to execute.
 839 */
 840void
 841ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
 842{
 843        ahd_swap_with_next_hscb(ahd, scb);
 844
 845        if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
 846                panic("Attempt to queue invalid SCB tag %x\n",
 847                      SCB_GET_TAG(scb));
 848
 849        /*
 850         * Keep a history of SCBs we've downloaded in the qinfifo.
 851         */
 852        ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
 853        ahd->qinfifonext++;
 854
 855        if (scb->sg_count != 0)
 856                ahd_setup_data_scb(ahd, scb);
 857        else
 858                ahd_setup_noxfer_scb(ahd, scb);
 859        ahd_setup_scb_common(ahd, scb);
 860
 861        /*
 862         * Make sure our data is consistent from the
 863         * perspective of the adapter.
 864         */
 865        ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 866
 867#ifdef AHD_DEBUG
 868        if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
 869                uint64_t host_dataptr;
 870
 871                host_dataptr = ahd_le64toh(scb->hscb->dataptr);
 872                printk("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
 873                       ahd_name(ahd),
 874                       SCB_GET_TAG(scb), scb->hscb->scsiid,
 875                       ahd_le32toh(scb->hscb->hscb_busaddr),
 876                       (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
 877                       (u_int)(host_dataptr & 0xFFFFFFFF),
 878                       ahd_le32toh(scb->hscb->datacnt));
 879        }
 880#endif
 881        /* Tell the adapter about the newly queued SCB */
 882        ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
 883}
 884
 885/************************** Interrupt Processing ******************************/
 886static void
 887ahd_sync_qoutfifo(struct ahd_softc *ahd, int op)
 888{
 889        ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 890                        /*offset*/0,
 891                        /*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op);
 892}
 893
 894static void
 895ahd_sync_tqinfifo(struct ahd_softc *ahd, int op)
 896{
 897#ifdef AHD_TARGET_MODE
 898        if ((ahd->flags & AHD_TARGETROLE) != 0) {
 899                ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 900                                ahd->shared_data_map.dmamap,
 901                                ahd_targetcmd_offset(ahd, 0),
 902                                sizeof(struct target_cmd) * AHD_TMODE_CMDS,
 903                                op);
 904        }
 905#endif
 906}
 907
 908/*
 909 * See if the firmware has posted any completed commands
 910 * into our in-core command complete fifos.
 911 */
 912#define AHD_RUN_QOUTFIFO 0x1
 913#define AHD_RUN_TQINFIFO 0x2
 914static u_int
 915ahd_check_cmdcmpltqueues(struct ahd_softc *ahd)
 916{
 917        u_int retval;
 918
 919        retval = 0;
 920        ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 921                        /*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo),
 922                        /*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD);
 923        if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag
 924          == ahd->qoutfifonext_valid_tag)
 925                retval |= AHD_RUN_QOUTFIFO;
 926#ifdef AHD_TARGET_MODE
 927        if ((ahd->flags & AHD_TARGETROLE) != 0
 928         && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) {
 929                ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 930                                ahd->shared_data_map.dmamap,
 931                                ahd_targetcmd_offset(ahd, ahd->tqinfifofnext),
 932                                /*len*/sizeof(struct target_cmd),
 933                                BUS_DMASYNC_POSTREAD);
 934                if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0)
 935                        retval |= AHD_RUN_TQINFIFO;
 936        }
 937#endif
 938        return (retval);
 939}
 940
 941/*
 942 * Catch an interrupt from the adapter
 943 */
 944int
 945ahd_intr(struct ahd_softc *ahd)
 946{
 947        u_int   intstat;
 948
 949        if ((ahd->pause & INTEN) == 0) {
 950                /*
 951                 * Our interrupt is not enabled on the chip
 952                 * and may be disabled for re-entrancy reasons,
 953                 * so just return.  This is likely just a shared
 954                 * interrupt.
 955                 */
 956                return (0);
 957        }
 958
 959        /*
 960         * Instead of directly reading the interrupt status register,
 961         * infer the cause of the interrupt by checking our in-core
 962         * completion queues.  This avoids a costly PCI bus read in
 963         * most cases.
 964         */
 965        if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0
 966         && (ahd_check_cmdcmpltqueues(ahd) != 0))
 967                intstat = CMDCMPLT;
 968        else
 969                intstat = ahd_inb(ahd, INTSTAT);
 970
 971        if ((intstat & INT_PEND) == 0)
 972                return (0);
 973
 974        if (intstat & CMDCMPLT) {
 975                ahd_outb(ahd, CLRINT, CLRCMDINT);
 976
 977                /*
 978                 * Ensure that the chip sees that we've cleared
 979                 * this interrupt before we walk the output fifo.
 980                 * Otherwise, we may, due to posted bus writes,
 981                 * clear the interrupt after we finish the scan,
 982                 * and after the sequencer has added new entries
 983                 * and asserted the interrupt again.
 984                 */
 985                if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 986                        if (ahd_is_paused(ahd)) {
 987                                /*
 988                                 * Potentially lost SEQINT.
 989                                 * If SEQINTCODE is non-zero,
 990                                 * simulate the SEQINT.
 991                                 */
 992                                if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT)
 993                                        intstat |= SEQINT;
 994                        }
 995                } else {
 996                        ahd_flush_device_writes(ahd);
 997                }
 998                ahd_run_qoutfifo(ahd);
 999                ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++;
1000                ahd->cmdcmplt_total++;
1001#ifdef AHD_TARGET_MODE
1002                if ((ahd->flags & AHD_TARGETROLE) != 0)
1003                        ahd_run_tqinfifo(ahd, /*paused*/FALSE);
1004#endif
1005        }
1006
1007        /*
1008         * Handle statuses that may invalidate our cached
1009         * copy of INTSTAT separately.
1010         */
1011        if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) {
1012                /* Hot eject.  Do nothing */
1013        } else if (intstat & HWERRINT) {
1014                ahd_handle_hwerrint(ahd);
1015        } else if ((intstat & (PCIINT|SPLTINT)) != 0) {
1016                ahd->bus_intr(ahd);
1017        } else {
1018
1019                if ((intstat & SEQINT) != 0)
1020                        ahd_handle_seqint(ahd, intstat);
1021
1022                if ((intstat & SCSIINT) != 0)
1023                        ahd_handle_scsiint(ahd, intstat);
1024        }
1025        return (1);
1026}
1027
1028/******************************** Private Inlines *****************************/
1029static inline void
1030ahd_assert_atn(struct ahd_softc *ahd)
1031{
1032        ahd_outb(ahd, SCSISIGO, ATNO);
1033}
1034
1035/*
1036 * Determine if the current connection has a packetized
1037 * agreement.  This does not necessarily mean that we
1038 * are currently in a packetized transfer.  We could
1039 * just as easily be sending or receiving a message.
1040 */
1041static int
1042ahd_currently_packetized(struct ahd_softc *ahd)
1043{
1044        ahd_mode_state   saved_modes;
1045        int              packetized;
1046
1047        saved_modes = ahd_save_modes(ahd);
1048        if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
1049                /*
1050                 * The packetized bit refers to the last
1051                 * connection, not the current one.  Check
1052                 * for non-zero LQISTATE instead.
1053                 */
1054                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1055                packetized = ahd_inb(ahd, LQISTATE) != 0;
1056        } else {
1057                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1058                packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
1059        }
1060        ahd_restore_modes(ahd, saved_modes);
1061        return (packetized);
1062}
1063
1064static inline int
1065ahd_set_active_fifo(struct ahd_softc *ahd)
1066{
1067        u_int active_fifo;
1068
1069        AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1070        active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
1071        switch (active_fifo) {
1072        case 0:
1073        case 1:
1074                ahd_set_modes(ahd, active_fifo, active_fifo);
1075                return (1);
1076        default:
1077                return (0);
1078        }
1079}
1080
1081static inline void
1082ahd_unbusy_tcl(struct ahd_softc *ahd, u_int tcl)
1083{
1084        ahd_busy_tcl(ahd, tcl, SCB_LIST_NULL);
1085}
1086
1087/*
1088 * Determine whether the sequencer reported a residual
1089 * for this SCB/transaction.
1090 */
1091static inline void
1092ahd_update_residual(struct ahd_softc *ahd, struct scb *scb)
1093{
1094        uint32_t sgptr;
1095
1096        sgptr = ahd_le32toh(scb->hscb->sgptr);
1097        if ((sgptr & SG_STATUS_VALID) != 0)
1098                ahd_calc_residual(ahd, scb);
1099}
1100
1101static inline void
1102ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb)
1103{
1104        uint32_t sgptr;
1105
1106        sgptr = ahd_le32toh(scb->hscb->sgptr);
1107        if ((sgptr & SG_STATUS_VALID) != 0)
1108                ahd_handle_scb_status(ahd, scb);
1109        else
1110                ahd_done(ahd, scb);
1111}
1112
1113
1114/************************* Sequencer Execution Control ************************/
1115/*
1116 * Restart the sequencer program from address zero
1117 */
1118static void
1119ahd_restart(struct ahd_softc *ahd)
1120{
1121
1122        ahd_pause(ahd);
1123
1124        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1125
1126        /* No more pending messages */
1127        ahd_clear_msg_state(ahd);
1128        ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
1129        ahd_outb(ahd, MSG_OUT, NOP);    /* No message to send */
1130        ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
1131        ahd_outb(ahd, SEQINTCTL, 0);
1132        ahd_outb(ahd, LASTPHASE, P_BUSFREE);
1133        ahd_outb(ahd, SEQ_FLAGS, 0);
1134        ahd_outb(ahd, SAVED_SCSIID, 0xFF);
1135        ahd_outb(ahd, SAVED_LUN, 0xFF);
1136
1137        /*
1138         * Ensure that the sequencer's idea of TQINPOS
1139         * matches our own.  The sequencer increments TQINPOS
1140         * only after it sees a DMA complete and a reset could
1141         * occur before the increment leaving the kernel to believe
1142         * the command arrived but the sequencer to not.
1143         */
1144        ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
1145
1146        /* Always allow reselection */
1147        ahd_outb(ahd, SCSISEQ1,
1148                 ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
1149        ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1150
1151        /*
1152         * Clear any pending sequencer interrupt.  It is no
1153         * longer relevant since we're resetting the Program
1154         * Counter.
1155         */
1156        ahd_outb(ahd, CLRINT, CLRSEQINT);
1157
1158        ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
1159        ahd_unpause(ahd);
1160}
1161
1162static void
1163ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
1164{
1165        ahd_mode_state   saved_modes;
1166
1167#ifdef AHD_DEBUG
1168        if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
1169                printk("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
1170#endif
1171        saved_modes = ahd_save_modes(ahd);
1172        ahd_set_modes(ahd, fifo, fifo);
1173        ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
1174        if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1175                ahd_outb(ahd, CCSGCTL, CCSGRESET);
1176        ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1177        ahd_outb(ahd, SG_STATE, 0);
1178        ahd_restore_modes(ahd, saved_modes);
1179}
1180
1181/************************* Input/Output Queues ********************************/
1182/*
1183 * Flush and completed commands that are sitting in the command
1184 * complete queues down on the chip but have yet to be dma'ed back up.
1185 */
1186static void
1187ahd_flush_qoutfifo(struct ahd_softc *ahd)
1188{
1189        struct          scb *scb;
1190        ahd_mode_state  saved_modes;
1191        u_int           saved_scbptr;
1192        u_int           ccscbctl;
1193        u_int           scbid;
1194        u_int           next_scbid;
1195
1196        saved_modes = ahd_save_modes(ahd);
1197
1198        /*
1199         * Flush the good status FIFO for completed packetized commands.
1200         */
1201        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1202        saved_scbptr = ahd_get_scbptr(ahd);
1203        while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
1204                u_int fifo_mode;
1205                u_int i;
1206
1207                scbid = ahd_inw(ahd, GSFIFO);
1208                scb = ahd_lookup_scb(ahd, scbid);
1209                if (scb == NULL) {
1210                        printk("%s: Warning - GSFIFO SCB %d invalid\n",
1211                               ahd_name(ahd), scbid);
1212                        continue;
1213                }
1214                /*
1215                 * Determine if this transaction is still active in
1216                 * any FIFO.  If it is, we must flush that FIFO to
1217                 * the host before completing the  command.
1218                 */
1219                fifo_mode = 0;
1220rescan_fifos:
1221                for (i = 0; i < 2; i++) {
1222                        /* Toggle to the other mode. */
1223                        fifo_mode ^= 1;
1224                        ahd_set_modes(ahd, fifo_mode, fifo_mode);
1225
1226                        if (ahd_scb_active_in_fifo(ahd, scb) == 0)
1227                                continue;
1228
1229                        ahd_run_data_fifo(ahd, scb);
1230
1231                        /*
1232                         * Running this FIFO may cause a CFG4DATA for
1233                         * this same transaction to assert in the other
1234                         * FIFO or a new snapshot SAVEPTRS interrupt
1235                         * in this FIFO.  Even running a FIFO may not
1236                         * clear the transaction if we are still waiting
1237                         * for data to drain to the host. We must loop
1238                         * until the transaction is not active in either
1239                         * FIFO just to be sure.  Reset our loop counter
1240                         * so we will visit both FIFOs again before
1241                         * declaring this transaction finished.  We
1242                         * also delay a bit so that status has a chance
1243                         * to change before we look at this FIFO again.
1244                         */
1245                        ahd_delay(200);
1246                        goto rescan_fifos;
1247                }
1248                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1249                ahd_set_scbptr(ahd, scbid);
1250                if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
1251                 && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
1252                  || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
1253                      & SG_LIST_NULL) != 0)) {
1254                        u_int comp_head;
1255
1256                        /*
1257                         * The transfer completed with a residual.
1258                         * Place this SCB on the complete DMA list
1259                         * so that we update our in-core copy of the
1260                         * SCB before completing the command.
1261                         */
1262                        ahd_outb(ahd, SCB_SCSI_STATUS, 0);
1263                        ahd_outb(ahd, SCB_SGPTR,
1264                                 ahd_inb_scbram(ahd, SCB_SGPTR)
1265                                 | SG_STATUS_VALID);
1266                        ahd_outw(ahd, SCB_TAG, scbid);
1267                        ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
1268                        comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1269                        if (SCBID_IS_NULL(comp_head)) {
1270                                ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
1271                                ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1272                        } else {
1273                                u_int tail;
1274
1275                                tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
1276                                ahd_set_scbptr(ahd, tail);
1277                                ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
1278                                ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1279                                ahd_set_scbptr(ahd, scbid);
1280                        }
1281                } else
1282                        ahd_complete_scb(ahd, scb);
1283        }
1284        ahd_set_scbptr(ahd, saved_scbptr);
1285
1286        /*
1287         * Setup for command channel portion of flush.
1288         */
1289        ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1290
1291        /*
1292         * Wait for any inprogress DMA to complete and clear DMA state
1293         * if this is for an SCB in the qinfifo.
1294         */
1295        while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
1296
1297                if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
1298                        if ((ccscbctl & ARRDONE) != 0)
1299                                break;
1300                } else if ((ccscbctl & CCSCBDONE) != 0)
1301                        break;
1302                ahd_delay(200);
1303        }
1304        /*
1305         * We leave the sequencer to cleanup in the case of DMA's to
1306         * update the qoutfifo.  In all other cases (DMA's to the
1307         * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
1308         * we disable the DMA engine so that the sequencer will not
1309         * attempt to handle the DMA completion.
1310         */
1311        if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
1312                ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
1313
1314        /*
1315         * Complete any SCBs that just finished
1316         * being DMA'ed into the qoutfifo.
1317         */
1318        ahd_run_qoutfifo(ahd);
1319
1320        saved_scbptr = ahd_get_scbptr(ahd);
1321        /*
1322         * Manually update/complete any completed SCBs that are waiting to be
1323         * DMA'ed back up to the host.
1324         */
1325        scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1326        while (!SCBID_IS_NULL(scbid)) {
1327                uint8_t *hscb_ptr;
1328                u_int    i;
1329
1330                ahd_set_scbptr(ahd, scbid);
1331                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1332                scb = ahd_lookup_scb(ahd, scbid);
1333                if (scb == NULL) {
1334                        printk("%s: Warning - DMA-up and complete "
1335                               "SCB %d invalid\n", ahd_name(ahd), scbid);
1336                        continue;
1337                }
1338                hscb_ptr = (uint8_t *)scb->hscb;
1339                for (i = 0; i < sizeof(struct hardware_scb); i++)
1340                        *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
1341
1342                ahd_complete_scb(ahd, scb);
1343                scbid = next_scbid;
1344        }
1345        ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
1346        ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
1347
1348        scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
1349        while (!SCBID_IS_NULL(scbid)) {
1350
1351                ahd_set_scbptr(ahd, scbid);
1352                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1353                scb = ahd_lookup_scb(ahd, scbid);
1354                if (scb == NULL) {
1355                        printk("%s: Warning - Complete Qfrz SCB %d invalid\n",
1356                               ahd_name(ahd), scbid);
1357                        continue;
1358                }
1359
1360                ahd_complete_scb(ahd, scb);
1361                scbid = next_scbid;
1362        }
1363        ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
1364
1365        scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
1366        while (!SCBID_IS_NULL(scbid)) {
1367
1368                ahd_set_scbptr(ahd, scbid);
1369                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1370                scb = ahd_lookup_scb(ahd, scbid);
1371                if (scb == NULL) {
1372                        printk("%s: Warning - Complete SCB %d invalid\n",
1373                               ahd_name(ahd), scbid);
1374                        continue;
1375                }
1376
1377                ahd_complete_scb(ahd, scb);
1378                scbid = next_scbid;
1379        }
1380        ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
1381
1382        /*
1383         * Restore state.
1384         */
1385        ahd_set_scbptr(ahd, saved_scbptr);
1386        ahd_restore_modes(ahd, saved_modes);
1387        ahd->flags |= AHD_UPDATE_PEND_CMDS;
1388}
1389
1390/*
1391 * Determine if an SCB for a packetized transaction
1392 * is active in a FIFO.
1393 */
1394static int
1395ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
1396{
1397
1398        /*
1399         * The FIFO is only active for our transaction if
1400         * the SCBPTR matches the SCB's ID and the firmware
1401         * has installed a handler for the FIFO or we have
1402         * a pending SAVEPTRS or CFG4DATA interrupt.
1403         */
1404        if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
1405         || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
1406          && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
1407                return (0);
1408
1409        return (1);
1410}
1411
1412/*
1413 * Run a data fifo to completion for a transaction we know
1414 * has completed across the SCSI bus (good status has been
1415 * received).  We are already set to the correct FIFO mode
1416 * on entry to this routine.
1417 *
1418 * This function attempts to operate exactly as the firmware
1419 * would when running this FIFO.  Care must be taken to update
1420 * this routine any time the firmware's FIFO algorithm is
1421 * changed.
1422 */
1423static void
1424ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
1425{
1426        u_int seqintsrc;
1427
1428        seqintsrc = ahd_inb(ahd, SEQINTSRC);
1429        if ((seqintsrc & CFG4DATA) != 0) {
1430                uint32_t datacnt;
1431                uint32_t sgptr;
1432
1433                /*
1434                 * Clear full residual flag.
1435                 */
1436                sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
1437                ahd_outb(ahd, SCB_SGPTR, sgptr);
1438
1439                /*
1440                 * Load datacnt and address.
1441                 */
1442                datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
1443                if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
1444                        sgptr |= LAST_SEG;
1445                        ahd_outb(ahd, SG_STATE, 0);
1446                } else
1447                        ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1448                ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
1449                ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
1450                ahd_outb(ahd, SG_CACHE_PRE, sgptr);
1451                ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1452
1453                /*
1454                 * Initialize Residual Fields.
1455                 */
1456                ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
1457                ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
1458
1459                /*
1460                 * Mark the SCB as having a FIFO in use.
1461                 */
1462                ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1463                         ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
1464
1465                /*
1466                 * Install a "fake" handler for this FIFO.
1467                 */
1468                ahd_outw(ahd, LONGJMP_ADDR, 0);
1469
1470                /*
1471                 * Notify the hardware that we have satisfied
1472                 * this sequencer interrupt.
1473                 */
1474                ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
1475        } else if ((seqintsrc & SAVEPTRS) != 0) {
1476                uint32_t sgptr;
1477                uint32_t resid;
1478
1479                if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
1480                        /*
1481                         * Snapshot Save Pointers.  All that
1482                         * is necessary to clear the snapshot
1483                         * is a CLRCHN.
1484                         */
1485                        goto clrchn;
1486                }
1487
1488                /*
1489                 * Disable S/G fetch so the DMA engine
1490                 * is available to future users.
1491                 */
1492                if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1493                        ahd_outb(ahd, CCSGCTL, 0);
1494                ahd_outb(ahd, SG_STATE, 0);
1495
1496                /*
1497                 * Flush the data FIFO.  Strickly only
1498                 * necessary for Rev A parts.
1499                 */
1500                ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
1501
1502                /*
1503                 * Calculate residual.
1504                 */
1505                sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1506                resid = ahd_inl(ahd, SHCNT);
1507                resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
1508                ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
1509                if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
1510                        /*
1511                         * Must back up to the correct S/G element.
1512                         * Typically this just means resetting our
1513                         * low byte to the offset in the SG_CACHE,
1514                         * but if we wrapped, we have to correct
1515                         * the other bytes of the sgptr too.
1516                         */
1517                        if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
1518                         && (sgptr & 0x80) == 0)
1519                                sgptr -= 0x100;
1520                        sgptr &= ~0xFF;
1521                        sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
1522                               & SG_ADDR_MASK;
1523                        ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1524                        ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
1525                } else if ((resid & AHD_SG_LEN_MASK) == 0) {
1526                        ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
1527                                 sgptr | SG_LIST_NULL);
1528                }
1529                /*
1530                 * Save Pointers.
1531                 */
1532                ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
1533                ahd_outl(ahd, SCB_DATACNT, resid);
1534                ahd_outl(ahd, SCB_SGPTR, sgptr);
1535                ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
1536                ahd_outb(ahd, SEQIMODE,
1537                         ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
1538                /*
1539                 * If the data is to the SCSI bus, we are
1540                 * done, otherwise wait for FIFOEMP.
1541                 */
1542                if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
1543                        goto clrchn;
1544        } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
1545                uint32_t sgptr;
1546                uint64_t data_addr;
1547                uint32_t data_len;
1548                u_int    dfcntrl;
1549
1550                /*
1551                 * Disable S/G fetch so the DMA engine
1552                 * is available to future users.  We won't
1553                 * be using the DMA engine to load segments.
1554                 */
1555                if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
1556                        ahd_outb(ahd, CCSGCTL, 0);
1557                        ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1558                }
1559
1560                /*
1561                 * Wait for the DMA engine to notice that the
1562                 * host transfer is enabled and that there is
1563                 * space in the S/G FIFO for new segments before
1564                 * loading more segments.
1565                 */
1566                if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
1567                 && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
1568
1569                        /*
1570                         * Determine the offset of the next S/G
1571                         * element to load.
1572                         */
1573                        sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1574                        sgptr &= SG_PTR_MASK;
1575                        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
1576                                struct ahd_dma64_seg *sg;
1577
1578                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1579                                data_addr = sg->addr;
1580                                data_len = sg->len;
1581                                sgptr += sizeof(*sg);
1582                        } else {
1583                                struct  ahd_dma_seg *sg;
1584
1585                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1586                                data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
1587                                data_addr <<= 8;
1588                                data_addr |= sg->addr;
1589                                data_len = sg->len;
1590                                sgptr += sizeof(*sg);
1591                        }
1592
1593                        /*
1594                         * Update residual information.
1595                         */
1596                        ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
1597                        ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1598
1599                        /*
1600                         * Load the S/G.
1601                         */
1602                        if (data_len & AHD_DMA_LAST_SEG) {
1603                                sgptr |= LAST_SEG;
1604                                ahd_outb(ahd, SG_STATE, 0);
1605                        }
1606                        ahd_outq(ahd, HADDR, data_addr);
1607                        ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
1608                        ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
1609
1610                        /*
1611                         * Advertise the segment to the hardware.
1612                         */
1613                        dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
1614                        if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
1615                                /*
1616                                 * Use SCSIENWRDIS so that SCSIEN
1617                                 * is never modified by this
1618                                 * operation.
1619                                 */
1620                                dfcntrl |= SCSIENWRDIS;
1621                        }
1622                        ahd_outb(ahd, DFCNTRL, dfcntrl);
1623                }
1624        } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
1625
1626                /*
1627                 * Transfer completed to the end of SG list
1628                 * and has flushed to the host.
1629                 */
1630                ahd_outb(ahd, SCB_SGPTR,
1631                         ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
1632                goto clrchn;
1633        } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
1634clrchn:
1635                /*
1636                 * Clear any handler for this FIFO, decrement
1637                 * the FIFO use count for the SCB, and release
1638                 * the FIFO.
1639                 */
1640                ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1641                ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1642                         ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
1643                ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
1644        }
1645}
1646
1647/*
1648 * Look for entries in the QoutFIFO that have completed.
1649 * The valid_tag completion field indicates the validity
1650 * of the entry - the valid value toggles each time through
1651 * the queue. We use the sg_status field in the completion
1652 * entry to avoid referencing the hscb if the completion
1653 * occurred with no errors and no residual.  sg_status is
1654 * a copy of the first byte (little endian) of the sgptr
1655 * hscb field.
1656 */
1657static void
1658ahd_run_qoutfifo(struct ahd_softc *ahd)
1659{
1660        struct ahd_completion *completion;
1661        struct scb *scb;
1662        u_int  scb_index;
1663
1664        if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
1665                panic("ahd_run_qoutfifo recursion");
1666        ahd->flags |= AHD_RUNNING_QOUTFIFO;
1667        ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
1668        for (;;) {
1669                completion = &ahd->qoutfifo[ahd->qoutfifonext];
1670
1671                if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
1672                        break;
1673
1674                scb_index = ahd_le16toh(completion->tag);
1675                scb = ahd_lookup_scb(ahd, scb_index);
1676                if (scb == NULL) {
1677                        printk("%s: WARNING no command for scb %d "
1678                               "(cmdcmplt)\nQOUTPOS = %d\n",
1679                               ahd_name(ahd), scb_index,
1680                               ahd->qoutfifonext);
1681                        ahd_dump_card_state(ahd);
1682                } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
1683                        ahd_handle_scb_status(ahd, scb);
1684                } else {
1685                        ahd_done(ahd, scb);
1686                }
1687
1688                ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
1689                if (ahd->qoutfifonext == 0)
1690                        ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
1691        }
1692        ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
1693}
1694
1695/************************* Interrupt Handling *********************************/
1696static void
1697ahd_handle_hwerrint(struct ahd_softc *ahd)
1698{
1699        /*
1700         * Some catastrophic hardware error has occurred.
1701         * Print it for the user and disable the controller.
1702         */
1703        int i;
1704        int error;
1705
1706        error = ahd_inb(ahd, ERROR);
1707        for (i = 0; i < num_errors; i++) {
1708                if ((error & ahd_hard_errors[i].errno) != 0)
1709                        printk("%s: hwerrint, %s\n",
1710                               ahd_name(ahd), ahd_hard_errors[i].errmesg);
1711        }
1712
1713        ahd_dump_card_state(ahd);
1714        panic("BRKADRINT");
1715
1716        /* Tell everyone that this HBA is no longer available */
1717        ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
1718                       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
1719                       CAM_NO_HBA);
1720
1721        /* Tell the system that this controller has gone away. */
1722        ahd_free(ahd);
1723}
1724
1725#ifdef AHD_DEBUG
1726static void
1727ahd_dump_sglist(struct scb *scb)
1728{
1729        int i;
1730
1731        if (scb->sg_count > 0) {
1732                if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
1733                        struct ahd_dma64_seg *sg_list;
1734
1735                        sg_list = (struct ahd_dma64_seg*)scb->sg_list;
1736                        for (i = 0; i < scb->sg_count; i++) {
1737                                uint64_t addr;
1738
1739                                addr = ahd_le64toh(sg_list[i].addr);
1740                                printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1741                                       i,
1742                                       (uint32_t)((addr >> 32) & 0xFFFFFFFF),
1743                                       (uint32_t)(addr & 0xFFFFFFFF),
1744                                       sg_list[i].len & AHD_SG_LEN_MASK,
1745                                       (sg_list[i].len & AHD_DMA_LAST_SEG)
1746                                     ? " Last" : "");
1747                        }
1748                } else {
1749                        struct ahd_dma_seg *sg_list;
1750
1751                        sg_list = (struct ahd_dma_seg*)scb->sg_list;
1752                        for (i = 0; i < scb->sg_count; i++) {
1753                                uint32_t len;
1754
1755                                len = ahd_le32toh(sg_list[i].len);
1756                                printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1757                                       i,
1758                                       (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
1759                                       ahd_le32toh(sg_list[i].addr),
1760                                       len & AHD_SG_LEN_MASK,
1761                                       len & AHD_DMA_LAST_SEG ? " Last" : "");
1762                        }
1763                }
1764        }
1765}
1766#endif  /*  AHD_DEBUG  */
1767
1768static void
1769ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
1770{
1771        u_int seqintcode;
1772
1773        /*
1774         * Save the sequencer interrupt code and clear the SEQINT
1775         * bit. We will unpause the sequencer, if appropriate,
1776         * after servicing the request.
1777         */
1778        seqintcode = ahd_inb(ahd, SEQINTCODE);
1779        ahd_outb(ahd, CLRINT, CLRSEQINT);
1780        if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
1781                /*
1782                 * Unpause the sequencer and let it clear
1783                 * SEQINT by writing NO_SEQINT to it.  This
1784                 * will cause the sequencer to be paused again,
1785                 * which is the expected state of this routine.
1786                 */
1787                ahd_unpause(ahd);
1788                while (!ahd_is_paused(ahd))
1789                        ;
1790                ahd_outb(ahd, CLRINT, CLRSEQINT);
1791        }
1792        ahd_update_modes(ahd);
1793#ifdef AHD_DEBUG
1794        if ((ahd_debug & AHD_SHOW_MISC) != 0)
1795                printk("%s: Handle Seqint Called for code %d\n",
1796                       ahd_name(ahd), seqintcode);
1797#endif
1798        switch (seqintcode) {
1799        case ENTERING_NONPACK:
1800        {
1801                struct  scb *scb;
1802                u_int   scbid;
1803
1804                AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1805                                 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1806                scbid = ahd_get_scbptr(ahd);
1807                scb = ahd_lookup_scb(ahd, scbid);
1808                if (scb == NULL) {
1809                        /*
1810                         * Somehow need to know if this
1811                         * is from a selection or reselection.
1812                         * From that, we can determine target
1813                         * ID so we at least have an I_T nexus.
1814                         */
1815                } else {
1816                        ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1817                        ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
1818                        ahd_outb(ahd, SEQ_FLAGS, 0x0);
1819                }
1820                if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
1821                 && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
1822                        /*
1823                         * Phase change after read stream with
1824                         * CRC error with P0 asserted on last
1825                         * packet.
1826                         */
1827#ifdef AHD_DEBUG
1828                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1829                                printk("%s: Assuming LQIPHASE_NLQ with "
1830                                       "P0 assertion\n", ahd_name(ahd));
1831#endif
1832                }
1833#ifdef AHD_DEBUG
1834                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1835                        printk("%s: Entering NONPACK\n", ahd_name(ahd));
1836#endif
1837                break;
1838        }
1839        case INVALID_SEQINT:
1840                printk("%s: Invalid Sequencer interrupt occurred, "
1841                       "resetting channel.\n",
1842                       ahd_name(ahd));
1843#ifdef AHD_DEBUG
1844                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1845                        ahd_dump_card_state(ahd);
1846#endif
1847                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1848                break;
1849        case STATUS_OVERRUN:
1850        {
1851                struct  scb *scb;
1852                u_int   scbid;
1853
1854                scbid = ahd_get_scbptr(ahd);
1855                scb = ahd_lookup_scb(ahd, scbid);
1856                if (scb != NULL)
1857                        ahd_print_path(ahd, scb);
1858                else
1859                        printk("%s: ", ahd_name(ahd));
1860                printk("SCB %d Packetized Status Overrun", scbid);
1861                ahd_dump_card_state(ahd);
1862                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1863                break;
1864        }
1865        case CFG4ISTAT_INTR:
1866        {
1867                struct  scb *scb;
1868                u_int   scbid;
1869
1870                scbid = ahd_get_scbptr(ahd);
1871                scb = ahd_lookup_scb(ahd, scbid);
1872                if (scb == NULL) {
1873                        ahd_dump_card_state(ahd);
1874                        printk("CFG4ISTAT: Free SCB %d referenced", scbid);
1875                        panic("For safety");
1876                }
1877                ahd_outq(ahd, HADDR, scb->sense_busaddr);
1878                ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1879                ahd_outb(ahd, HCNT + 2, 0);
1880                ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1881                ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1882                break;
1883        }
1884        case ILLEGAL_PHASE:
1885        {
1886                u_int bus_phase;
1887
1888                bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1889                printk("%s: ILLEGAL_PHASE 0x%x\n",
1890                       ahd_name(ahd), bus_phase);
1891
1892                switch (bus_phase) {
1893                case P_DATAOUT:
1894                case P_DATAIN:
1895                case P_DATAOUT_DT:
1896                case P_DATAIN_DT:
1897                case P_MESGOUT:
1898                case P_STATUS:
1899                case P_MESGIN:
1900                        ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1901                        printk("%s: Issued Bus Reset.\n", ahd_name(ahd));
1902                        break;
1903                case P_COMMAND:
1904                {
1905                        struct  ahd_devinfo devinfo;
1906                        struct  scb *scb;
1907                        u_int   scbid;
1908
1909                        /*
1910                         * If a target takes us into the command phase
1911                         * assume that it has been externally reset and
1912                         * has thus lost our previous packetized negotiation
1913                         * agreement.  Since we have not sent an identify
1914                         * message and may not have fully qualified the
1915                         * connection, we change our command to TUR, assert
1916                         * ATN and ABORT the task when we go to message in
1917                         * phase.  The OSM will see the REQUEUE_REQUEST
1918                         * status and retry the command.
1919                         */
1920                        scbid = ahd_get_scbptr(ahd);
1921                        scb = ahd_lookup_scb(ahd, scbid);
1922                        if (scb == NULL) {
1923                                printk("Invalid phase with no valid SCB.  "
1924                                       "Resetting bus.\n");
1925                                ahd_reset_channel(ahd, 'A',
1926                                                  /*Initiate Reset*/TRUE);
1927                                break;
1928                        }
1929                        ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1930                                            SCB_GET_TARGET(ahd, scb),
1931                                            SCB_GET_LUN(scb),
1932                                            SCB_GET_CHANNEL(ahd, scb),
1933                                            ROLE_INITIATOR);
1934                        ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1935                                      AHD_TRANS_ACTIVE, /*paused*/TRUE);
1936                        ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1937                                         /*offset*/0, /*ppr_options*/0,
1938                                         AHD_TRANS_ACTIVE, /*paused*/TRUE);
1939                        /* Hand-craft TUR command */
1940                        ahd_outb(ahd, SCB_CDB_STORE, 0);
1941                        ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1942                        ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1943                        ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1944                        ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1945                        ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1946                        ahd_outb(ahd, SCB_CDB_LEN, 6);
1947                        scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1948                        scb->hscb->control |= MK_MESSAGE;
1949                        ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1950                        ahd_outb(ahd, MSG_OUT, HOST_MSG);
1951                        ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1952                        /*
1953                         * The lun is 0, regardless of the SCB's lun
1954                         * as we have not sent an identify message.
1955                         */
1956                        ahd_outb(ahd, SAVED_LUN, 0);
1957                        ahd_outb(ahd, SEQ_FLAGS, 0);
1958                        ahd_assert_atn(ahd);
1959                        scb->flags &= ~SCB_PACKETIZED;
1960                        scb->flags |= SCB_ABORT|SCB_EXTERNAL_RESET;
1961                        ahd_freeze_devq(ahd, scb);
1962                        ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1963                        ahd_freeze_scb(scb);
1964
1965                        /* Notify XPT */
1966                        ahd_send_async(ahd, devinfo.channel, devinfo.target,
1967                                       CAM_LUN_WILDCARD, AC_SENT_BDR);
1968
1969                        /*
1970                         * Allow the sequencer to continue with
1971                         * non-pack processing.
1972                         */
1973                        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1974                        ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1975                        if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1976                                ahd_outb(ahd, CLRLQOINT1, 0);
1977                        }
1978#ifdef AHD_DEBUG
1979                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1980                                ahd_print_path(ahd, scb);
1981                                printk("Unexpected command phase from "
1982                                       "packetized target\n");
1983                        }
1984#endif
1985                        break;
1986                }
1987                }
1988                break;
1989        }
1990        case CFG4OVERRUN:
1991        {
1992                struct  scb *scb;
1993                u_int   scb_index;
1994
1995#ifdef AHD_DEBUG
1996                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1997                        printk("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1998                               ahd_inb(ahd, MODE_PTR));
1999                }
2000#endif
2001                scb_index = ahd_get_scbptr(ahd);
2002                scb = ahd_lookup_scb(ahd, scb_index);
2003                if (scb == NULL) {
2004                        /*
2005                         * Attempt to transfer to an SCB that is
2006                         * not outstanding.
2007                         */
2008                        ahd_assert_atn(ahd);
2009                        ahd_outb(ahd, MSG_OUT, HOST_MSG);
2010                        ahd->msgout_buf[0] = ABORT_TASK;
2011                        ahd->msgout_len = 1;
2012                        ahd->msgout_index = 0;
2013                        ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2014                        /*
2015                         * Clear status received flag to prevent any
2016                         * attempt to complete this bogus SCB.
2017                         */
2018                        ahd_outb(ahd, SCB_CONTROL,
2019                                 ahd_inb_scbram(ahd, SCB_CONTROL)
2020                                 & ~STATUS_RCVD);
2021                }
2022                break;
2023        }
2024        case DUMP_CARD_STATE:
2025        {
2026                ahd_dump_card_state(ahd);
2027                break;
2028        }
2029        case PDATA_REINIT:
2030        {
2031#ifdef AHD_DEBUG
2032                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2033                        printk("%s: PDATA_REINIT - DFCNTRL = 0x%x "
2034                               "SG_CACHE_SHADOW = 0x%x\n",
2035                               ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
2036                               ahd_inb(ahd, SG_CACHE_SHADOW));
2037                }
2038#endif
2039                ahd_reinitialize_dataptrs(ahd);
2040                break;
2041        }
2042        case HOST_MSG_LOOP:
2043        {
2044                struct ahd_devinfo devinfo;
2045
2046                /*
2047                 * The sequencer has encountered a message phase
2048                 * that requires host assistance for completion.
2049                 * While handling the message phase(s), we will be
2050                 * notified by the sequencer after each byte is
2051                 * transferred so we can track bus phase changes.
2052                 *
2053                 * If this is the first time we've seen a HOST_MSG_LOOP
2054                 * interrupt, initialize the state of the host message
2055                 * loop.
2056                 */
2057                ahd_fetch_devinfo(ahd, &devinfo);
2058                if (ahd->msg_type == MSG_TYPE_NONE) {
2059                        struct scb *scb;
2060                        u_int scb_index;
2061                        u_int bus_phase;
2062
2063                        bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2064                        if (bus_phase != P_MESGIN
2065                         && bus_phase != P_MESGOUT) {
2066                                printk("ahd_intr: HOST_MSG_LOOP bad "
2067                                       "phase 0x%x\n", bus_phase);
2068                                /*
2069                                 * Probably transitioned to bus free before
2070                                 * we got here.  Just punt the message.
2071                                 */
2072                                ahd_dump_card_state(ahd);
2073                                ahd_clear_intstat(ahd);
2074                                ahd_restart(ahd);
2075                                return;
2076                        }
2077
2078                        scb_index = ahd_get_scbptr(ahd);
2079                        scb = ahd_lookup_scb(ahd, scb_index);
2080                        if (devinfo.role == ROLE_INITIATOR) {
2081                                if (bus_phase == P_MESGOUT)
2082                                        ahd_setup_initiator_msgout(ahd,
2083                                                                   &devinfo,
2084                                                                   scb);
2085                                else {
2086                                        ahd->msg_type =
2087                                            MSG_TYPE_INITIATOR_MSGIN;
2088                                        ahd->msgin_index = 0;
2089                                }
2090                        }
2091#ifdef AHD_TARGET_MODE
2092                        else {
2093                                if (bus_phase == P_MESGOUT) {
2094                                        ahd->msg_type =
2095                                            MSG_TYPE_TARGET_MSGOUT;
2096                                        ahd->msgin_index = 0;
2097                                } else
2098                                        ahd_setup_target_msgin(ahd,
2099                                                               &devinfo,
2100                                                               scb);
2101                        }
2102#endif
2103                }
2104
2105                ahd_handle_message_phase(ahd);
2106                break;
2107        }
2108        case NO_MATCH:
2109        {
2110                /* Ensure we don't leave the selection hardware on */
2111                AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
2112                ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2113
2114                printk("%s:%c:%d: no active SCB for reconnecting "
2115                       "target - issuing BUS DEVICE RESET\n",
2116                       ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
2117                printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
2118                       "REG0 == 0x%x ACCUM = 0x%x\n",
2119                       ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
2120                       ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
2121                printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
2122                       "SINDEX == 0x%x\n",
2123                       ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
2124                       ahd_find_busy_tcl(ahd,
2125                                         BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
2126                                                   ahd_inb(ahd, SAVED_LUN))),
2127                       ahd_inw(ahd, SINDEX));
2128                printk("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
2129                       "SCB_CONTROL == 0x%x\n",
2130                       ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
2131                       ahd_inb_scbram(ahd, SCB_LUN),
2132                       ahd_inb_scbram(ahd, SCB_CONTROL));
2133                printk("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
2134                       ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
2135                printk("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
2136                printk("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
2137                ahd_dump_card_state(ahd);
2138                ahd->msgout_buf[0] = TARGET_RESET;
2139                ahd->msgout_len = 1;
2140                ahd->msgout_index = 0;
2141                ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2142                ahd_outb(ahd, MSG_OUT, HOST_MSG);
2143                ahd_assert_atn(ahd);
2144                break;
2145        }
2146        case PROTO_VIOLATION:
2147        {
2148                ahd_handle_proto_violation(ahd);
2149                break;
2150        }
2151        case IGN_WIDE_RES:
2152        {
2153                struct ahd_devinfo devinfo;
2154
2155                ahd_fetch_devinfo(ahd, &devinfo);
2156                ahd_handle_ign_wide_residue(ahd, &devinfo);
2157                break;
2158        }
2159        case BAD_PHASE:
2160        {
2161                u_int lastphase;
2162
2163                lastphase = ahd_inb(ahd, LASTPHASE);
2164                printk("%s:%c:%d: unknown scsi bus phase %x, "
2165                       "lastphase = 0x%x.  Attempting to continue\n",
2166                       ahd_name(ahd), 'A',
2167                       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2168                       lastphase, ahd_inb(ahd, SCSISIGI));
2169                break;
2170        }
2171        case MISSED_BUSFREE:
2172        {
2173                u_int lastphase;
2174
2175                lastphase = ahd_inb(ahd, LASTPHASE);
2176                printk("%s:%c:%d: Missed busfree. "
2177                       "Lastphase = 0x%x, Curphase = 0x%x\n",
2178                       ahd_name(ahd), 'A',
2179                       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2180                       lastphase, ahd_inb(ahd, SCSISIGI));
2181                ahd_restart(ahd);
2182                return;
2183        }
2184        case DATA_OVERRUN:
2185        {
2186                /*
2187                 * When the sequencer detects an overrun, it
2188                 * places the controller in "BITBUCKET" mode
2189                 * and allows the target to complete its transfer.
2190                 * Unfortunately, none of the counters get updated
2191                 * when the controller is in this mode, so we have
2192                 * no way of knowing how large the overrun was.
2193                 */
2194                struct  scb *scb;
2195                u_int   scbindex;
2196#ifdef AHD_DEBUG
2197                u_int   lastphase;
2198#endif
2199
2200                scbindex = ahd_get_scbptr(ahd);
2201                scb = ahd_lookup_scb(ahd, scbindex);
2202#ifdef AHD_DEBUG
2203                lastphase = ahd_inb(ahd, LASTPHASE);
2204                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2205                        ahd_print_path(ahd, scb);
2206                        printk("data overrun detected %s.  Tag == 0x%x.\n",
2207                               ahd_lookup_phase_entry(lastphase)->phasemsg,
2208                               SCB_GET_TAG(scb));
2209                        ahd_print_path(ahd, scb);
2210                        printk("%s seen Data Phase.  Length = %ld.  "
2211                               "NumSGs = %d.\n",
2212                               ahd_inb(ahd, SEQ_FLAGS) & DPHASE
2213                               ? "Have" : "Haven't",
2214                               ahd_get_transfer_length(scb), scb->sg_count);
2215                        ahd_dump_sglist(scb);
2216                }
2217#endif
2218
2219                /*
2220                 * Set this and it will take effect when the
2221                 * target does a command complete.
2222                 */
2223                ahd_freeze_devq(ahd, scb);
2224                ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
2225                ahd_freeze_scb(scb);
2226                break;
2227        }
2228        case MKMSG_FAILED:
2229        {
2230                struct ahd_devinfo devinfo;
2231                struct scb *scb;
2232                u_int scbid;
2233
2234                ahd_fetch_devinfo(ahd, &devinfo);
2235                printk("%s:%c:%d:%d: Attempt to issue message failed\n",
2236                       ahd_name(ahd), devinfo.channel, devinfo.target,
2237                       devinfo.lun);
2238                scbid = ahd_get_scbptr(ahd);
2239                scb = ahd_lookup_scb(ahd, scbid);
2240                if (scb != NULL
2241                 && (scb->flags & SCB_RECOVERY_SCB) != 0)
2242                        /*
2243                         * Ensure that we didn't put a second instance of this
2244                         * SCB into the QINFIFO.
2245                         */
2246                        ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2247                                           SCB_GET_CHANNEL(ahd, scb),
2248                                           SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2249                                           ROLE_INITIATOR, /*status*/0,
2250                                           SEARCH_REMOVE);
2251                ahd_outb(ahd, SCB_CONTROL,
2252                         ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
2253                break;
2254        }
2255        case TASKMGMT_FUNC_COMPLETE:
2256        {
2257                u_int   scbid;
2258                struct  scb *scb;
2259
2260                scbid = ahd_get_scbptr(ahd);
2261                scb = ahd_lookup_scb(ahd, scbid);
2262                if (scb != NULL) {
2263                        u_int      lun;
2264                        u_int      tag;
2265                        cam_status error;
2266
2267                        ahd_print_path(ahd, scb);
2268                        printk("Task Management Func 0x%x Complete\n",
2269                               scb->hscb->task_management);
2270                        lun = CAM_LUN_WILDCARD;
2271                        tag = SCB_LIST_NULL;
2272
2273                        switch (scb->hscb->task_management) {
2274                        case SIU_TASKMGMT_ABORT_TASK:
2275                                tag = SCB_GET_TAG(scb);
2276                                fallthrough;
2277                        case SIU_TASKMGMT_ABORT_TASK_SET:
2278                        case SIU_TASKMGMT_CLEAR_TASK_SET:
2279                                lun = scb->hscb->lun;
2280                                error = CAM_REQ_ABORTED;
2281                                ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2282                                               'A', lun, tag, ROLE_INITIATOR,
2283                                               error);
2284                                break;
2285                        case SIU_TASKMGMT_LUN_RESET:
2286                                lun = scb->hscb->lun;
2287                                fallthrough;
2288                        case SIU_TASKMGMT_TARGET_RESET:
2289                        {
2290                                struct ahd_devinfo devinfo;
2291
2292                                ahd_scb_devinfo(ahd, &devinfo, scb);
2293                                error = CAM_BDR_SENT;
2294                                ahd_handle_devreset(ahd, &devinfo, lun,
2295                                                    CAM_BDR_SENT,
2296                                                    lun != CAM_LUN_WILDCARD
2297                                                    ? "Lun Reset"
2298                                                    : "Target Reset",
2299                                                    /*verbose_level*/0);
2300                                break;
2301                        }
2302                        default:
2303                                panic("Unexpected TaskMgmt Func\n");
2304                                break;
2305                        }
2306                }
2307                break;
2308        }
2309        case TASKMGMT_CMD_CMPLT_OKAY:
2310        {
2311                u_int   scbid;
2312                struct  scb *scb;
2313
2314                /*
2315                 * An ABORT TASK TMF failed to be delivered before
2316                 * the targeted command completed normally.
2317                 */
2318                scbid = ahd_get_scbptr(ahd);
2319                scb = ahd_lookup_scb(ahd, scbid);
2320                if (scb != NULL) {
2321                        /*
2322                         * Remove the second instance of this SCB from
2323                         * the QINFIFO if it is still there.
2324                         */
2325                        ahd_print_path(ahd, scb);
2326                        printk("SCB completes before TMF\n");
2327                        /*
2328                         * Handle losing the race.  Wait until any
2329                         * current selection completes.  We will then
2330                         * set the TMF back to zero in this SCB so that
2331                         * the sequencer doesn't bother to issue another
2332                         * sequencer interrupt for its completion.
2333                         */
2334                        while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
2335                            && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
2336                            && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
2337                                ;
2338                        ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
2339                        ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2340                                           SCB_GET_CHANNEL(ahd, scb),
2341                                           SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2342                                           ROLE_INITIATOR, /*status*/0,
2343                                           SEARCH_REMOVE);
2344                }
2345                break;
2346        }
2347        case TRACEPOINT0:
2348        case TRACEPOINT1:
2349        case TRACEPOINT2:
2350        case TRACEPOINT3:
2351                printk("%s: Tracepoint %d\n", ahd_name(ahd),
2352                       seqintcode - TRACEPOINT0);
2353                break;
2354        case NO_SEQINT:
2355                break;
2356        case SAW_HWERR:
2357                ahd_handle_hwerrint(ahd);
2358                break;
2359        default:
2360                printk("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
2361                       seqintcode);
2362                break;
2363        }
2364        /*
2365         *  The sequencer is paused immediately on
2366         *  a SEQINT, so we should restart it when
2367         *  we're done.
2368         */
2369        ahd_unpause(ahd);
2370}
2371
2372static void
2373ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
2374{
2375        struct scb      *scb;
2376        u_int            status0;
2377        u_int            status3;
2378        u_int            status;
2379        u_int            lqistat1;
2380        u_int            lqostat0;
2381        u_int            scbid;
2382        u_int            busfreetime;
2383
2384        ahd_update_modes(ahd);
2385        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2386
2387        status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
2388        status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
2389        status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
2390        lqistat1 = ahd_inb(ahd, LQISTAT1);
2391        lqostat0 = ahd_inb(ahd, LQOSTAT0);
2392        busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2393
2394        /*
2395         * Ignore external resets after a bus reset.
2396         */
2397        if (((status & SCSIRSTI) != 0) && (ahd->flags & AHD_BUS_RESET_ACTIVE)) {
2398                ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
2399                return;
2400        }
2401
2402        /*
2403         * Clear bus reset flag
2404         */
2405        ahd->flags &= ~AHD_BUS_RESET_ACTIVE;
2406
2407        if ((status0 & (SELDI|SELDO)) != 0) {
2408                u_int simode0;
2409
2410                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2411                simode0 = ahd_inb(ahd, SIMODE0);
2412                status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
2413                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2414        }
2415        scbid = ahd_get_scbptr(ahd);
2416        scb = ahd_lookup_scb(ahd, scbid);
2417        if (scb != NULL
2418         && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2419                scb = NULL;
2420
2421        if ((status0 & IOERR) != 0) {
2422                u_int now_lvd;
2423
2424                now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
2425                printk("%s: Transceiver State Has Changed to %s mode\n",
2426                       ahd_name(ahd), now_lvd ? "LVD" : "SE");
2427                ahd_outb(ahd, CLRSINT0, CLRIOERR);
2428                /*
2429                 * A change in I/O mode is equivalent to a bus reset.
2430                 */
2431                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2432                ahd_pause(ahd);
2433                ahd_setup_iocell_workaround(ahd);
2434                ahd_unpause(ahd);
2435        } else if ((status0 & OVERRUN) != 0) {
2436
2437                printk("%s: SCSI offset overrun detected.  Resetting bus.\n",
2438                       ahd_name(ahd));
2439                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2440        } else if ((status & SCSIRSTI) != 0) {
2441
2442                printk("%s: Someone reset channel A\n", ahd_name(ahd));
2443                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
2444        } else if ((status & SCSIPERR) != 0) {
2445
2446                /* Make sure the sequencer is in a safe location. */
2447                ahd_clear_critical_section(ahd);
2448
2449                ahd_handle_transmission_error(ahd);
2450        } else if (lqostat0 != 0) {
2451
2452                printk("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
2453                ahd_outb(ahd, CLRLQOINT0, lqostat0);
2454                if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2455                        ahd_outb(ahd, CLRLQOINT1, 0);
2456        } else if ((status & SELTO) != 0) {
2457                /* Stop the selection */
2458                ahd_outb(ahd, SCSISEQ0, 0);
2459
2460                /* Make sure the sequencer is in a safe location. */
2461                ahd_clear_critical_section(ahd);
2462
2463                /* No more pending messages */
2464                ahd_clear_msg_state(ahd);
2465
2466                /* Clear interrupt state */
2467                ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
2468
2469                /*
2470                 * Although the driver does not care about the
2471                 * 'Selection in Progress' status bit, the busy
2472                 * LED does.  SELINGO is only cleared by a successful
2473                 * selection, so we must manually clear it to insure
2474                 * the LED turns off just incase no future successful
2475                 * selections occur (e.g. no devices on the bus).
2476                 */
2477                ahd_outb(ahd, CLRSINT0, CLRSELINGO);
2478
2479                scbid = ahd_inw(ahd, WAITING_TID_HEAD);
2480                scb = ahd_lookup_scb(ahd, scbid);
2481                if (scb == NULL) {
2482                        printk("%s: ahd_intr - referenced scb not "
2483                               "valid during SELTO scb(0x%x)\n",
2484                               ahd_name(ahd), scbid);
2485                        ahd_dump_card_state(ahd);
2486                } else {
2487                        struct ahd_devinfo devinfo;
2488#ifdef AHD_DEBUG
2489                        if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
2490                                ahd_print_path(ahd, scb);
2491                                printk("Saw Selection Timeout for SCB 0x%x\n",
2492                                       scbid);
2493                        }
2494#endif
2495                        ahd_scb_devinfo(ahd, &devinfo, scb);
2496                        ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
2497                        ahd_freeze_devq(ahd, scb);
2498
2499                        /*
2500                         * Cancel any pending transactions on the device
2501                         * now that it seems to be missing.  This will
2502                         * also revert us to async/narrow transfers until
2503                         * we can renegotiate with the device.
2504                         */
2505                        ahd_handle_devreset(ahd, &devinfo,
2506                                            CAM_LUN_WILDCARD,
2507                                            CAM_SEL_TIMEOUT,
2508                                            "Selection Timeout",
2509                                            /*verbose_level*/1);
2510                }
2511                ahd_outb(ahd, CLRINT, CLRSCSIINT);
2512                ahd_iocell_first_selection(ahd);
2513                ahd_unpause(ahd);
2514        } else if ((status0 & (SELDI|SELDO)) != 0) {
2515
2516                ahd_iocell_first_selection(ahd);
2517                ahd_unpause(ahd);
2518        } else if (status3 != 0) {
2519                printk("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
2520                       ahd_name(ahd), status3);
2521                ahd_outb(ahd, CLRSINT3, status3);
2522        } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
2523
2524                /* Make sure the sequencer is in a safe location. */
2525                ahd_clear_critical_section(ahd);
2526
2527                ahd_handle_lqiphase_error(ahd, lqistat1);
2528        } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2529                /*
2530                 * This status can be delayed during some
2531                 * streaming operations.  The SCSIPHASE
2532                 * handler has already dealt with this case
2533                 * so just clear the error.
2534                 */
2535                ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
2536        } else if ((status & BUSFREE) != 0
2537                || (lqistat1 & LQOBUSFREE) != 0) {
2538                u_int lqostat1;
2539                int   restart;
2540                int   clear_fifo;
2541                int   packetized;
2542                u_int mode;
2543
2544                /*
2545                 * Clear our selection hardware as soon as possible.
2546                 * We may have an entry in the waiting Q for this target,
2547                 * that is affected by this busfree and we don't want to
2548                 * go about selecting the target while we handle the event.
2549                 */
2550                ahd_outb(ahd, SCSISEQ0, 0);
2551
2552                /* Make sure the sequencer is in a safe location. */
2553                ahd_clear_critical_section(ahd);
2554
2555                /*
2556                 * Determine what we were up to at the time of
2557                 * the busfree.
2558                 */
2559                mode = AHD_MODE_SCSI;
2560                busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2561                lqostat1 = ahd_inb(ahd, LQOSTAT1);
2562                switch (busfreetime) {
2563                case BUSFREE_DFF0:
2564                case BUSFREE_DFF1:
2565                {
2566                        mode = busfreetime == BUSFREE_DFF0
2567                             ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
2568                        ahd_set_modes(ahd, mode, mode);
2569                        scbid = ahd_get_scbptr(ahd);
2570                        scb = ahd_lookup_scb(ahd, scbid);
2571                        if (scb == NULL) {
2572                                printk("%s: Invalid SCB %d in DFF%d "
2573                                       "during unexpected busfree\n",
2574                                       ahd_name(ahd), scbid, mode);
2575                                packetized = 0;
2576                        } else
2577                                packetized = (scb->flags & SCB_PACKETIZED) != 0;
2578                        clear_fifo = 1;
2579                        break;
2580                }
2581                case BUSFREE_LQO:
2582                        clear_fifo = 0;
2583                        packetized = 1;
2584                        break;
2585                default:
2586                        clear_fifo = 0;
2587                        packetized =  (lqostat1 & LQOBUSFREE) != 0;
2588                        if (!packetized
2589                         && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
2590                         && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
2591                         && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
2592                          || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
2593                                /*
2594                                 * Assume packetized if we are not
2595                                 * on the bus in a non-packetized
2596                                 * capacity and any pending selection
2597                                 * was a packetized selection.
2598                                 */
2599                                packetized = 1;
2600                        break;
2601                }
2602
2603#ifdef AHD_DEBUG
2604                if ((ahd_debug & AHD_SHOW_MISC) != 0)
2605                        printk("Saw Busfree.  Busfreetime = 0x%x.\n",
2606                               busfreetime);
2607#endif
2608                /*
2609                 * Busfrees that occur in non-packetized phases are
2610                 * handled by the nonpkt_busfree handler.
2611                 */
2612                if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
2613                        restart = ahd_handle_pkt_busfree(ahd, busfreetime);
2614                } else {
2615                        packetized = 0;
2616                        restart = ahd_handle_nonpkt_busfree(ahd);
2617                }
2618                /*
2619                 * Clear the busfree interrupt status.  The setting of
2620                 * the interrupt is a pulse, so in a perfect world, we
2621                 * would not need to muck with the ENBUSFREE logic.  This
2622                 * would ensure that if the bus moves on to another
2623                 * connection, busfree protection is still in force.  If
2624                 * BUSFREEREV is broken, however, we must manually clear
2625                 * the ENBUSFREE if the busfree occurred during a non-pack
2626                 * connection so that we don't get false positives during
2627                 * future, packetized, connections.
2628                 */
2629                ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2630                if (packetized == 0
2631                 && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
2632                        ahd_outb(ahd, SIMODE1,
2633                                 ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
2634
2635                if (clear_fifo)
2636                        ahd_clear_fifo(ahd, mode);
2637
2638                ahd_clear_msg_state(ahd);
2639                ahd_outb(ahd, CLRINT, CLRSCSIINT);
2640                if (restart) {
2641                        ahd_restart(ahd);
2642                } else {
2643                        ahd_unpause(ahd);
2644                }
2645        } else {
2646                printk("%s: Missing case in ahd_handle_scsiint. status = %x\n",
2647                       ahd_name(ahd), status);
2648                ahd_dump_card_state(ahd);
2649                ahd_clear_intstat(ahd);
2650                ahd_unpause(ahd);
2651        }
2652}
2653
2654static void
2655ahd_handle_transmission_error(struct ahd_softc *ahd)
2656{
2657        struct  scb *scb;
2658        u_int   scbid;
2659        u_int   lqistat1;
2660        u_int   msg_out;
2661        u_int   curphase;
2662        u_int   lastphase;
2663        u_int   perrdiag;
2664        u_int   cur_col;
2665        int     silent;
2666
2667        scb = NULL;
2668        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2669        lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
2670        ahd_inb(ahd, LQISTAT2);
2671        if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
2672         && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
2673                u_int lqistate;
2674
2675                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2676                lqistate = ahd_inb(ahd, LQISTATE);
2677                if ((lqistate >= 0x1E && lqistate <= 0x24)
2678                 || (lqistate == 0x29)) {
2679#ifdef AHD_DEBUG
2680                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2681                                printk("%s: NLQCRC found via LQISTATE\n",
2682                                       ahd_name(ahd));
2683                        }
2684#endif
2685                        lqistat1 |= LQICRCI_NLQ;
2686                }
2687                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2688        }
2689
2690        ahd_outb(ahd, CLRLQIINT1, lqistat1);
2691        lastphase = ahd_inb(ahd, LASTPHASE);
2692        curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2693        perrdiag = ahd_inb(ahd, PERRDIAG);
2694        msg_out = INITIATOR_ERROR;
2695        ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
2696
2697        /*
2698         * Try to find the SCB associated with this error.
2699         */
2700        silent = FALSE;
2701        if (lqistat1 == 0
2702         || (lqistat1 & LQICRCI_NLQ) != 0) {
2703                if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
2704                        ahd_set_active_fifo(ahd);
2705                scbid = ahd_get_scbptr(ahd);
2706                scb = ahd_lookup_scb(ahd, scbid);
2707                if (scb != NULL && SCB_IS_SILENT(scb))
2708                        silent = TRUE;
2709        }
2710
2711        cur_col = 0;
2712        if (silent == FALSE) {
2713                printk("%s: Transmission error detected\n", ahd_name(ahd));
2714                ahd_lqistat1_print(lqistat1, &cur_col, 50);
2715                ahd_lastphase_print(lastphase, &cur_col, 50);
2716                ahd_scsisigi_print(curphase, &cur_col, 50);
2717                ahd_perrdiag_print(perrdiag, &cur_col, 50);
2718                printk("\n");
2719                ahd_dump_card_state(ahd);
2720        }
2721
2722        if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
2723                if (silent == FALSE) {
2724                        printk("%s: Gross protocol error during incoming "
2725                               "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
2726                               ahd_name(ahd), lqistat1);
2727                }
2728                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2729                return;
2730        } else if ((lqistat1 & LQICRCI_LQ) != 0) {
2731                /*
2732                 * A CRC error has been detected on an incoming LQ.
2733                 * The bus is currently hung on the last ACK.
2734                 * Hit LQIRETRY to release the last ack, and
2735                 * wait for the sequencer to determine that ATNO
2736                 * is asserted while in message out to take us
2737                 * to our host message loop.  No NONPACKREQ or
2738                 * LQIPHASE type errors will occur in this
2739                 * scenario.  After this first LQIRETRY, the LQI
2740                 * manager will be in ISELO where it will
2741                 * happily sit until another packet phase begins.
2742                 * Unexpected bus free detection is enabled
2743                 * through any phases that occur after we release
2744                 * this last ack until the LQI manager sees a
2745                 * packet phase.  This implies we may have to
2746                 * ignore a perfectly valid "unexected busfree"
2747                 * after our "initiator detected error" message is
2748                 * sent.  A busfree is the expected response after
2749                 * we tell the target that it's L_Q was corrupted.
2750                 * (SPI4R09 10.7.3.3.3)
2751                 */
2752                ahd_outb(ahd, LQCTL2, LQIRETRY);
2753                printk("LQIRetry for LQICRCI_LQ to release ACK\n");
2754        } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2755                /*
2756                 * We detected a CRC error in a NON-LQ packet.
2757                 * The hardware has varying behavior in this situation
2758                 * depending on whether this packet was part of a
2759                 * stream or not.
2760                 *
2761                 * PKT by PKT mode:
2762                 * The hardware has already acked the complete packet.
2763                 * If the target honors our outstanding ATN condition,
2764                 * we should be (or soon will be) in MSGOUT phase.
2765                 * This will trigger the LQIPHASE_LQ status bit as the
2766                 * hardware was expecting another LQ.  Unexpected
2767                 * busfree detection is enabled.  Once LQIPHASE_LQ is
2768                 * true (first entry into host message loop is much
2769                 * the same), we must clear LQIPHASE_LQ and hit
2770                 * LQIRETRY so the hardware is ready to handle
2771                 * a future LQ.  NONPACKREQ will not be asserted again
2772                 * once we hit LQIRETRY until another packet is
2773                 * processed.  The target may either go busfree
2774                 * or start another packet in response to our message.
2775                 *
2776                 * Read Streaming P0 asserted:
2777                 * If we raise ATN and the target completes the entire
2778                 * stream (P0 asserted during the last packet), the
2779                 * hardware will ack all data and return to the ISTART
2780                 * state.  When the target reponds to our ATN condition,
2781                 * LQIPHASE_LQ will be asserted.  We should respond to
2782                 * this with an LQIRETRY to prepare for any future
2783                 * packets.  NONPACKREQ will not be asserted again
2784                 * once we hit LQIRETRY until another packet is
2785                 * processed.  The target may either go busfree or
2786                 * start another packet in response to our message.
2787                 * Busfree detection is enabled.
2788                 *
2789                 * Read Streaming P0 not asserted:
2790                 * If we raise ATN and the target transitions to
2791                 * MSGOUT in or after a packet where P0 is not
2792                 * asserted, the hardware will assert LQIPHASE_NLQ.
2793                 * We should respond to the LQIPHASE_NLQ with an
2794                 * LQIRETRY.  Should the target stay in a non-pkt
2795                 * phase after we send our message, the hardware
2796                 * will assert LQIPHASE_LQ.  Recovery is then just as
2797                 * listed above for the read streaming with P0 asserted.
2798                 * Busfree detection is enabled.
2799                 */
2800                if (silent == FALSE)
2801                        printk("LQICRC_NLQ\n");
2802                if (scb == NULL) {
2803                        printk("%s: No SCB valid for LQICRC_NLQ.  "
2804                               "Resetting bus\n", ahd_name(ahd));
2805                        ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2806                        return;
2807                }
2808        } else if ((lqistat1 & LQIBADLQI) != 0) {
2809                printk("Need to handle BADLQI!\n");
2810                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2811                return;
2812        } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
2813                if ((curphase & ~P_DATAIN_DT) != 0) {
2814                        /* Ack the byte.  So we can continue. */
2815                        if (silent == FALSE)
2816                                printk("Acking %s to clear perror\n",
2817                                    ahd_lookup_phase_entry(curphase)->phasemsg);
2818                        ahd_inb(ahd, SCSIDAT);
2819                }
2820
2821                if (curphase == P_MESGIN)
2822                        msg_out = MSG_PARITY_ERROR;
2823        }
2824
2825        /*
2826         * We've set the hardware to assert ATN if we
2827         * get a parity error on "in" phases, so all we
2828         * need to do is stuff the message buffer with
2829         * the appropriate message.  "In" phases have set
2830         * mesg_out to something other than NOP.
2831         */
2832        ahd->send_msg_perror = msg_out;
2833        if (scb != NULL && msg_out == INITIATOR_ERROR)
2834                scb->flags |= SCB_TRANSMISSION_ERROR;
2835        ahd_outb(ahd, MSG_OUT, HOST_MSG);
2836        ahd_outb(ahd, CLRINT, CLRSCSIINT);
2837        ahd_unpause(ahd);
2838}
2839
2840static void
2841ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2842{
2843        /*
2844         * Clear the sources of the interrupts.
2845         */
2846        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2847        ahd_outb(ahd, CLRLQIINT1, lqistat1);
2848
2849        /*
2850         * If the "illegal" phase changes were in response
2851         * to our ATN to flag a CRC error, AND we ended up
2852         * on packet boundaries, clear the error, restart the
2853         * LQI manager as appropriate, and go on our merry
2854         * way toward sending the message.  Otherwise, reset
2855         * the bus to clear the error.
2856         */
2857        ahd_set_active_fifo(ahd);
2858        if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2859         && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2860                if ((lqistat1 & LQIPHASE_LQ) != 0) {
2861                        printk("LQIRETRY for LQIPHASE_LQ\n");
2862                        ahd_outb(ahd, LQCTL2, LQIRETRY);
2863                } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2864                        printk("LQIRETRY for LQIPHASE_NLQ\n");
2865                        ahd_outb(ahd, LQCTL2, LQIRETRY);
2866                } else
2867                        panic("ahd_handle_lqiphase_error: No phase errors\n");
2868                ahd_dump_card_state(ahd);
2869                ahd_outb(ahd, CLRINT, CLRSCSIINT);
2870                ahd_unpause(ahd);
2871        } else {
2872                printk("Resetting Channel for LQI Phase error\n");
2873                ahd_dump_card_state(ahd);
2874                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2875        }
2876}
2877
2878/*
2879 * Packetized unexpected or expected busfree.
2880 * Entered in mode based on busfreetime.
2881 */
2882static int
2883ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2884{
2885        u_int lqostat1;
2886
2887        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2888                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2889        lqostat1 = ahd_inb(ahd, LQOSTAT1);
2890        if ((lqostat1 & LQOBUSFREE) != 0) {
2891                struct scb *scb;
2892                u_int scbid;
2893                u_int saved_scbptr;
2894                u_int waiting_h;
2895                u_int waiting_t;
2896                u_int next;
2897
2898                /*
2899                 * The LQO manager detected an unexpected busfree
2900                 * either:
2901                 *
2902                 * 1) During an outgoing LQ.
2903                 * 2) After an outgoing LQ but before the first
2904                 *    REQ of the command packet.
2905                 * 3) During an outgoing command packet.
2906                 *
2907                 * In all cases, CURRSCB is pointing to the
2908                 * SCB that encountered the failure.  Clean
2909                 * up the queue, clear SELDO and LQOBUSFREE,
2910                 * and allow the sequencer to restart the select
2911                 * out at its lesure.
2912                 */
2913                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2914                scbid = ahd_inw(ahd, CURRSCB);
2915                scb = ahd_lookup_scb(ahd, scbid);
2916                if (scb == NULL)
2917                       panic("SCB not valid during LQOBUSFREE");
2918                /*
2919                 * Clear the status.
2920                 */
2921                ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2922                if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2923                        ahd_outb(ahd, CLRLQOINT1, 0);
2924                ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2925                ahd_flush_device_writes(ahd);
2926                ahd_outb(ahd, CLRSINT0, CLRSELDO);
2927
2928                /*
2929                 * Return the LQO manager to its idle loop.  It will
2930                 * not do this automatically if the busfree occurs
2931                 * after the first REQ of either the LQ or command
2932                 * packet or between the LQ and command packet.
2933                 */
2934                ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2935
2936                /*
2937                 * Update the waiting for selection queue so
2938                 * we restart on the correct SCB.
2939                 */
2940                waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2941                saved_scbptr = ahd_get_scbptr(ahd);
2942                if (waiting_h != scbid) {
2943
2944                        ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2945                        waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2946                        if (waiting_t == waiting_h) {
2947                                ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2948                                next = SCB_LIST_NULL;
2949                        } else {
2950                                ahd_set_scbptr(ahd, waiting_h);
2951                                next = ahd_inw_scbram(ahd, SCB_NEXT2);
2952                        }
2953                        ahd_set_scbptr(ahd, scbid);
2954                        ahd_outw(ahd, SCB_NEXT2, next);
2955                }
2956                ahd_set_scbptr(ahd, saved_scbptr);
2957                if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2958                        if (SCB_IS_SILENT(scb) == FALSE) {
2959                                ahd_print_path(ahd, scb);
2960                                printk("Probable outgoing LQ CRC error.  "
2961                                       "Retrying command\n");
2962                        }
2963                        scb->crc_retry_count++;
2964                } else {
2965                        ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
2966                        ahd_freeze_scb(scb);
2967                        ahd_freeze_devq(ahd, scb);
2968                }
2969                /* Return unpausing the sequencer. */
2970                return (0);
2971        } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2972                /*
2973                 * Ignore what are really parity errors that
2974                 * occur on the last REQ of a free running
2975                 * clock prior to going busfree.  Some drives
2976                 * do not properly active negate just before
2977                 * going busfree resulting in a parity glitch.
2978                 */
2979                ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2980#ifdef AHD_DEBUG
2981                if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2982                        printk("%s: Parity on last REQ detected "
2983                               "during busfree phase.\n",
2984                               ahd_name(ahd));
2985#endif
2986                /* Return unpausing the sequencer. */
2987                return (0);
2988        }
2989        if (ahd->src_mode != AHD_MODE_SCSI) {
2990                u_int   scbid;
2991                struct  scb *scb;
2992
2993                scbid = ahd_get_scbptr(ahd);
2994                scb = ahd_lookup_scb(ahd, scbid);
2995                ahd_print_path(ahd, scb);
2996                printk("Unexpected PKT busfree condition\n");
2997                ahd_dump_card_state(ahd);
2998                ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
2999                               SCB_GET_LUN(scb), SCB_GET_TAG(scb),
3000                               ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
3001
3002                /* Return restarting the sequencer. */
3003                return (1);
3004        }
3005        printk("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
3006        ahd_dump_card_state(ahd);
3007        /* Restart the sequencer. */
3008        return (1);
3009}
3010
3011/*
3012 * Non-packetized unexpected or expected busfree.
3013 */
3014static int
3015ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
3016{
3017        struct  ahd_devinfo devinfo;
3018        struct  scb *scb;
3019        u_int   lastphase;
3020        u_int   saved_scsiid;
3021        u_int   saved_lun;
3022        u_int   target;
3023        u_int   initiator_role_id;
3024        u_int   scbid;
3025        u_int   ppr_busfree;
3026        int     printerror;
3027
3028        /*
3029         * Look at what phase we were last in.  If its message out,
3030         * chances are pretty good that the busfree was in response
3031         * to one of our abort requests.
3032         */
3033        lastphase = ahd_inb(ahd, LASTPHASE);
3034        saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3035        saved_lun = ahd_inb(ahd, SAVED_LUN);
3036        target = SCSIID_TARGET(ahd, saved_scsiid);
3037        initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
3038        ahd_compile_devinfo(&devinfo, initiator_role_id,
3039                            target, saved_lun, 'A', ROLE_INITIATOR);
3040        printerror = 1;
3041
3042        scbid = ahd_get_scbptr(ahd);
3043        scb = ahd_lookup_scb(ahd, scbid);
3044        if (scb != NULL
3045         && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
3046                scb = NULL;
3047
3048        ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
3049        if (lastphase == P_MESGOUT) {
3050                u_int tag;
3051
3052                tag = SCB_LIST_NULL;
3053                if (ahd_sent_msg(ahd, AHDMSG_1B, ABORT_TASK, TRUE)
3054                 || ahd_sent_msg(ahd, AHDMSG_1B, ABORT_TASK_SET, TRUE)) {
3055                        int found;
3056                        int sent_msg;
3057
3058                        if (scb == NULL) {
3059                                ahd_print_devinfo(ahd, &devinfo);
3060                                printk("Abort for unidentified "
3061                                       "connection completed.\n");
3062                                /* restart the sequencer. */
3063                                return (1);
3064                        }
3065                        sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
3066                        ahd_print_path(ahd, scb);
3067                        printk("SCB %d - Abort%s Completed.\n",
3068                               SCB_GET_TAG(scb),
3069                               sent_msg == ABORT_TASK ? "" : " Tag");
3070
3071                        if (sent_msg == ABORT_TASK)
3072                                tag = SCB_GET_TAG(scb);
3073
3074                        if ((scb->flags & SCB_EXTERNAL_RESET) != 0) {
3075                                /*
3076                                 * This abort is in response to an
3077                                 * unexpected switch to command phase
3078                                 * for a packetized connection.  Since
3079                                 * the identify message was never sent,
3080                                 * "saved lun" is 0.  We really want to
3081                                 * abort only the SCB that encountered
3082                                 * this error, which could have a different
3083                                 * lun.  The SCB will be retried so the OS
3084                                 * will see the UA after renegotiating to
3085                                 * packetized.
3086                                 */
3087                                tag = SCB_GET_TAG(scb);
3088                                saved_lun = scb->hscb->lun;
3089                        }
3090                        found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
3091                                               tag, ROLE_INITIATOR,
3092                                               CAM_REQ_ABORTED);
3093                        printk("found == 0x%x\n", found);
3094                        printerror = 0;
3095                } else if (ahd_sent_msg(ahd, AHDMSG_1B,
3096                                        TARGET_RESET, TRUE)) {
3097                        ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
3098                                            CAM_BDR_SENT, "Bus Device Reset",
3099                                            /*verbose_level*/0);
3100                        printerror = 0;
3101                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, EXTENDED_PPR, FALSE)
3102                        && ppr_busfree == 0) {
3103                        struct ahd_initiator_tinfo *tinfo;
3104                        struct ahd_tmode_tstate *tstate;
3105
3106                        /*
3107                         * PPR Rejected.
3108                         *
3109                         * If the previous negotiation was packetized,
3110                         * this could be because the device has been
3111                         * reset without our knowledge.  Force our
3112                         * current negotiation to async and retry the
3113                         * negotiation.  Otherwise retry the command
3114                         * with non-ppr negotiation.
3115                         */
3116#ifdef AHD_DEBUG
3117                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3118                                printk("PPR negotiation rejected busfree.\n");
3119#endif
3120                        tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3121                                                    devinfo.our_scsiid,
3122                                                    devinfo.target, &tstate);
3123                        if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
3124                                ahd_set_width(ahd, &devinfo,
3125                                              MSG_EXT_WDTR_BUS_8_BIT,
3126                                              AHD_TRANS_CUR,
3127                                              /*paused*/TRUE);
3128                                ahd_set_syncrate(ahd, &devinfo,
3129                                                /*period*/0, /*offset*/0,
3130                                                /*ppr_options*/0,
3131                                                AHD_TRANS_CUR,
3132                                                /*paused*/TRUE);
3133                                /*
3134                                 * The expect PPR busfree handler below
3135                                 * will effect the retry and necessary
3136                                 * abort.
3137                                 */
3138                        } else {
3139                                tinfo->curr.transport_version = 2;
3140                                tinfo->goal.transport_version = 2;
3141                                tinfo->goal.ppr_options = 0;
3142                                if (scb != NULL) {
3143                                        /*
3144                                         * Remove any SCBs in the waiting
3145                                         * for selection queue that may
3146                                         * also be for this target so that
3147                                         * command ordering is preserved.
3148                                         */
3149                                        ahd_freeze_devq(ahd, scb);
3150                                        ahd_qinfifo_requeue_tail(ahd, scb);
3151                                }
3152                                printerror = 0;
3153                        }
3154                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, EXTENDED_WDTR, FALSE)
3155                        && ppr_busfree == 0) {
3156                        /*
3157                         * Negotiation Rejected.  Go-narrow and
3158                         * retry command.
3159                         */
3160#ifdef AHD_DEBUG
3161                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3162                                printk("WDTR negotiation rejected busfree.\n");
3163#endif
3164                        ahd_set_width(ahd, &devinfo,
3165                                      MSG_EXT_WDTR_BUS_8_BIT,
3166                                      AHD_TRANS_CUR|AHD_TRANS_GOAL,
3167                                      /*paused*/TRUE);
3168                        if (scb != NULL) {
3169                                /*
3170                                 * Remove any SCBs in the waiting for
3171                                 * selection queue that may also be for
3172                                 * this target so that command ordering
3173                                 * is preserved.
3174                                 */
3175                                ahd_freeze_devq(ahd, scb);
3176                                ahd_qinfifo_requeue_tail(ahd, scb);
3177                        }
3178                        printerror = 0;
3179                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, EXTENDED_SDTR, FALSE)
3180                        && ppr_busfree == 0) {
3181                        /*
3182                         * Negotiation Rejected.  Go-async and
3183                         * retry command.
3184                         */
3185#ifdef AHD_DEBUG
3186                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3187                                printk("SDTR negotiation rejected busfree.\n");
3188#endif
3189                        ahd_set_syncrate(ahd, &devinfo,
3190                                        /*period*/0, /*offset*/0,
3191                                        /*ppr_options*/0,
3192                                        AHD_TRANS_CUR|AHD_TRANS_GOAL,
3193                                        /*paused*/TRUE);
3194                        if (scb != NULL) {
3195                                /*
3196                                 * Remove any SCBs in the waiting for
3197                                 * selection queue that may also be for
3198                                 * this target so that command ordering
3199                                 * is preserved.
3200                                 */
3201                                ahd_freeze_devq(ahd, scb);
3202                                ahd_qinfifo_requeue_tail(ahd, scb);
3203                        }
3204                        printerror = 0;
3205                } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
3206                        && ahd_sent_msg(ahd, AHDMSG_1B,
3207                                         INITIATOR_ERROR, TRUE)) {
3208
3209#ifdef AHD_DEBUG
3210                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3211                                printk("Expected IDE Busfree\n");
3212#endif
3213                        printerror = 0;
3214                } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
3215                        && ahd_sent_msg(ahd, AHDMSG_1B,
3216                                        MESSAGE_REJECT, TRUE)) {
3217
3218#ifdef AHD_DEBUG
3219                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3220                                printk("Expected QAS Reject Busfree\n");
3221#endif
3222                        printerror = 0;
3223                }
3224        }
3225
3226        /*
3227         * The busfree required flag is honored at the end of
3228         * the message phases.  We check it last in case we
3229         * had to send some other message that caused a busfree.
3230         */
3231        if (scb != NULL && printerror != 0
3232         && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
3233         && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
3234
3235                ahd_freeze_devq(ahd, scb);
3236                ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
3237                ahd_freeze_scb(scb);
3238                if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
3239                        ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
3240                                       SCB_GET_CHANNEL(ahd, scb),
3241                                       SCB_GET_LUN(scb), SCB_LIST_NULL,
3242                                       ROLE_INITIATOR, CAM_REQ_ABORTED);
3243                } else {
3244#ifdef AHD_DEBUG
3245                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3246                                printk("PPR Negotiation Busfree.\n");
3247#endif
3248                        ahd_done(ahd, scb);
3249                }
3250                printerror = 0;
3251        }
3252        if (printerror != 0) {
3253                int aborted;
3254
3255                aborted = 0;
3256                if (scb != NULL) {
3257                        u_int tag;
3258
3259                        if ((scb->hscb->control & TAG_ENB) != 0)
3260                                tag = SCB_GET_TAG(scb);
3261                        else
3262                                tag = SCB_LIST_NULL;
3263                        ahd_print_path(ahd, scb);
3264                        aborted = ahd_abort_scbs(ahd, target, 'A',
3265                                       SCB_GET_LUN(scb), tag,
3266                                       ROLE_INITIATOR,
3267                                       CAM_UNEXP_BUSFREE);
3268                } else {
3269                        /*
3270                         * We had not fully identified this connection,
3271                         * so we cannot abort anything.
3272                         */
3273                        printk("%s: ", ahd_name(ahd));
3274                }
3275                printk("Unexpected busfree %s, %d SCBs aborted, "
3276                       "PRGMCNT == 0x%x\n",
3277                       ahd_lookup_phase_entry(lastphase)->phasemsg,
3278                       aborted,
3279                       ahd_inw(ahd, PRGMCNT));
3280                ahd_dump_card_state(ahd);
3281                if (lastphase != P_BUSFREE)
3282                        ahd_force_renegotiation(ahd, &devinfo);
3283        }
3284        /* Always restart the sequencer. */
3285        return (1);
3286}
3287
3288static void
3289ahd_handle_proto_violation(struct ahd_softc *ahd)
3290{
3291        struct  ahd_devinfo devinfo;
3292        struct  scb *scb;
3293        u_int   scbid;
3294        u_int   seq_flags;
3295        u_int   curphase;
3296        u_int   lastphase;
3297        int     found;
3298
3299        ahd_fetch_devinfo(ahd, &devinfo);
3300        scbid = ahd_get_scbptr(ahd);
3301        scb = ahd_lookup_scb(ahd, scbid);
3302        seq_flags = ahd_inb(ahd, SEQ_FLAGS);
3303        curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
3304        lastphase = ahd_inb(ahd, LASTPHASE);
3305        if ((seq_flags & NOT_IDENTIFIED) != 0) {
3306
3307                /*
3308                 * The reconnecting target either did not send an
3309                 * identify message, or did, but we didn't find an SCB
3310                 * to match.
3311                 */
3312                ahd_print_devinfo(ahd, &devinfo);
3313                printk("Target did not send an IDENTIFY message. "
3314                       "LASTPHASE = 0x%x.\n", lastphase);
3315                scb = NULL;
3316        } else if (scb == NULL) {
3317                /*
3318                 * We don't seem to have an SCB active for this
3319                 * transaction.  Print an error and reset the bus.
3320                 */
3321                ahd_print_devinfo(ahd, &devinfo);
3322                printk("No SCB found during protocol violation\n");
3323                goto proto_violation_reset;
3324        } else {
3325                ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3326                if ((seq_flags & NO_CDB_SENT) != 0) {
3327                        ahd_print_path(ahd, scb);
3328                        printk("No or incomplete CDB sent to device.\n");
3329                } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
3330                          & STATUS_RCVD) == 0) {
3331                        /*
3332                         * The target never bothered to provide status to
3333                         * us prior to completing the command.  Since we don't
3334                         * know the disposition of this command, we must attempt
3335                         * to abort it.  Assert ATN and prepare to send an abort
3336                         * message.
3337                         */
3338                        ahd_print_path(ahd, scb);
3339                        printk("Completed command without status.\n");
3340                } else {
3341                        ahd_print_path(ahd, scb);
3342                        printk("Unknown protocol violation.\n");
3343                        ahd_dump_card_state(ahd);
3344                }
3345        }
3346        if ((lastphase & ~P_DATAIN_DT) == 0
3347         || lastphase == P_COMMAND) {
3348proto_violation_reset:
3349                /*
3350                 * Target either went directly to data
3351                 * phase or didn't respond to our ATN.
3352                 * The only safe thing to do is to blow
3353                 * it away with a bus reset.
3354                 */
3355                found = ahd_reset_channel(ahd, 'A', TRUE);
3356                printk("%s: Issued Channel %c Bus Reset. "
3357                       "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
3358        } else {
3359                /*
3360                 * Leave the selection hardware off in case
3361                 * this abort attempt will affect yet to
3362                 * be sent commands.
3363                 */
3364                ahd_outb(ahd, SCSISEQ0,
3365                         ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3366                ahd_assert_atn(ahd);
3367                ahd_outb(ahd, MSG_OUT, HOST_MSG);
3368                if (scb == NULL) {
3369                        ahd_print_devinfo(ahd, &devinfo);
3370                        ahd->msgout_buf[0] = ABORT_TASK;
3371                        ahd->msgout_len = 1;
3372                        ahd->msgout_index = 0;
3373                        ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3374                } else {
3375                        ahd_print_path(ahd, scb);
3376                        scb->flags |= SCB_ABORT;
3377                }
3378                printk("Protocol violation %s.  Attempting to abort.\n",
3379                       ahd_lookup_phase_entry(curphase)->phasemsg);
3380        }
3381}
3382
3383/*
3384 * Force renegotiation to occur the next time we initiate
3385 * a command to the current device.
3386 */
3387static void
3388ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3389{
3390        struct  ahd_initiator_tinfo *targ_info;
3391        struct  ahd_tmode_tstate *tstate;
3392
3393#ifdef AHD_DEBUG
3394        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3395                ahd_print_devinfo(ahd, devinfo);
3396                printk("Forcing renegotiation\n");
3397        }
3398#endif
3399        targ_info = ahd_fetch_transinfo(ahd,
3400                                        devinfo->channel,
3401                                        devinfo->our_scsiid,
3402                                        devinfo->target,
3403                                        &tstate);
3404        ahd_update_neg_request(ahd, devinfo, tstate,
3405                               targ_info, AHD_NEG_IF_NON_ASYNC);
3406}
3407
3408#define AHD_MAX_STEPS 2000
3409static void
3410ahd_clear_critical_section(struct ahd_softc *ahd)
3411{
3412        ahd_mode_state  saved_modes;
3413        int             stepping;
3414        int             steps;
3415        int             first_instr;
3416        u_int           simode0;
3417        u_int           simode1;
3418        u_int           simode3;
3419        u_int           lqimode0;
3420        u_int           lqimode1;
3421        u_int           lqomode0;
3422        u_int           lqomode1;
3423
3424        if (ahd->num_critical_sections == 0)
3425                return;
3426
3427        stepping = FALSE;
3428        steps = 0;
3429        first_instr = 0;
3430        simode0 = 0;
3431        simode1 = 0;
3432        simode3 = 0;
3433        lqimode0 = 0;
3434        lqimode1 = 0;
3435        lqomode0 = 0;
3436        lqomode1 = 0;
3437        saved_modes = ahd_save_modes(ahd);
3438        for (;;) {
3439                struct  cs *cs;
3440                u_int   seqaddr;
3441                u_int   i;
3442
3443                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3444                seqaddr = ahd_inw(ahd, CURADDR);
3445
3446                cs = ahd->critical_sections;
3447                for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
3448                        if (cs->begin < seqaddr && cs->end >= seqaddr)
3449                                break;
3450                }
3451
3452                if (i == ahd->num_critical_sections)
3453                        break;
3454
3455                if (steps > AHD_MAX_STEPS) {
3456                        printk("%s: Infinite loop in critical section\n"
3457                               "%s: First Instruction 0x%x now 0x%x\n",
3458                               ahd_name(ahd), ahd_name(ahd), first_instr,
3459                               seqaddr);
3460                        ahd_dump_card_state(ahd);
3461                        panic("critical section loop");
3462                }
3463
3464                steps++;
3465#ifdef AHD_DEBUG
3466                if ((ahd_debug & AHD_SHOW_MISC) != 0)
3467                        printk("%s: Single stepping at 0x%x\n", ahd_name(ahd),
3468                               seqaddr);
3469#endif
3470                if (stepping == FALSE) {
3471
3472                        first_instr = seqaddr;
3473                        ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3474                        simode0 = ahd_inb(ahd, SIMODE0);
3475                        simode3 = ahd_inb(ahd, SIMODE3);
3476                        lqimode0 = ahd_inb(ahd, LQIMODE0);
3477                        lqimode1 = ahd_inb(ahd, LQIMODE1);
3478                        lqomode0 = ahd_inb(ahd, LQOMODE0);
3479                        lqomode1 = ahd_inb(ahd, LQOMODE1);
3480                        ahd_outb(ahd, SIMODE0, 0);
3481                        ahd_outb(ahd, SIMODE3, 0);
3482                        ahd_outb(ahd, LQIMODE0, 0);
3483                        ahd_outb(ahd, LQIMODE1, 0);
3484                        ahd_outb(ahd, LQOMODE0, 0);
3485                        ahd_outb(ahd, LQOMODE1, 0);
3486                        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3487                        simode1 = ahd_inb(ahd, SIMODE1);
3488                        /*
3489                         * We don't clear ENBUSFREE.  Unfortunately
3490                         * we cannot re-enable busfree detection within
3491                         * the current connection, so we must leave it
3492                         * on while single stepping.
3493                         */
3494                        ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
3495                        ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
3496                        stepping = TRUE;
3497                }
3498                ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
3499                ahd_outb(ahd, CLRINT, CLRSCSIINT);
3500                ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
3501                ahd_outb(ahd, HCNTRL, ahd->unpause);
3502                while (!ahd_is_paused(ahd))
3503                        ahd_delay(200);
3504                ahd_update_modes(ahd);
3505        }
3506        if (stepping) {
3507                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3508                ahd_outb(ahd, SIMODE0, simode0);
3509                ahd_outb(ahd, SIMODE3, simode3);
3510                ahd_outb(ahd, LQIMODE0, lqimode0);
3511                ahd_outb(ahd, LQIMODE1, lqimode1);
3512                ahd_outb(ahd, LQOMODE0, lqomode0);
3513                ahd_outb(ahd, LQOMODE1, lqomode1);
3514                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3515                ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
3516                ahd_outb(ahd, SIMODE1, simode1);
3517                /*
3518                 * SCSIINT seems to glitch occasionally when
3519                 * the interrupt masks are restored.  Clear SCSIINT
3520                 * one more time so that only persistent errors
3521                 * are seen as a real interrupt.
3522                 */
3523                ahd_outb(ahd, CLRINT, CLRSCSIINT);
3524        }
3525        ahd_restore_modes(ahd, saved_modes);
3526}
3527
3528/*
3529 * Clear any pending interrupt status.
3530 */
3531static void
3532ahd_clear_intstat(struct ahd_softc *ahd)
3533{
3534        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
3535                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
3536        /* Clear any interrupt conditions this may have caused */
3537        ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
3538                                 |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
3539        ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
3540                                 |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
3541                                 |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
3542        ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
3543                                 |CLRLQOATNPKT|CLRLQOTCRC);
3544        ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
3545                                 |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
3546        if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
3547                ahd_outb(ahd, CLRLQOINT0, 0);
3548                ahd_outb(ahd, CLRLQOINT1, 0);
3549        }
3550        ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
3551        ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
3552                                |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
3553        ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
3554                                |CLRIOERR|CLROVERRUN);
3555        ahd_outb(ahd, CLRINT, CLRSCSIINT);
3556}
3557
3558/**************************** Debugging Routines ******************************/
3559#ifdef AHD_DEBUG
3560uint32_t ahd_debug = AHD_DEBUG_OPTS;
3561#endif
3562
3563#if 0
3564void
3565ahd_print_scb(struct scb *scb)
3566{
3567        struct hardware_scb *hscb;
3568        int i;
3569
3570        hscb = scb->hscb;
3571        printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
3572               (void *)scb,
3573               hscb->control,
3574               hscb->scsiid,
3575               hscb->lun,
3576               hscb->cdb_len);
3577        printk("Shared Data: ");
3578        for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
3579                printk("%#02x", hscb->shared_data.idata.cdb[i]);
3580        printk("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
3581               (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
3582               (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
3583               ahd_le32toh(hscb->datacnt),
3584               ahd_le32toh(hscb->sgptr),
3585               SCB_GET_TAG(scb));
3586        ahd_dump_sglist(scb);
3587}
3588#endif  /*  0  */
3589
3590/************************* Transfer Negotiation *******************************/
3591/*
3592 * Allocate per target mode instance (ID we respond to as a target)
3593 * transfer negotiation data structures.
3594 */
3595static struct ahd_tmode_tstate *
3596ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
3597{
3598        struct ahd_tmode_tstate *master_tstate;
3599        struct ahd_tmode_tstate *tstate;
3600        int i;
3601
3602        master_tstate = ahd->enabled_targets[ahd->our_id];
3603        if (ahd->enabled_targets[scsi_id] != NULL
3604         && ahd->enabled_targets[scsi_id] != master_tstate)
3605                panic("%s: ahd_alloc_tstate - Target already allocated",
3606                      ahd_name(ahd));
3607        tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
3608        if (tstate == NULL)
3609                return (NULL);
3610
3611        /*
3612         * If we have allocated a master tstate, copy user settings from
3613         * the master tstate (taken from SRAM or the EEPROM) for this
3614         * channel, but reset our current and goal settings to async/narrow
3615         * until an initiator talks to us.
3616         */
3617        if (master_tstate != NULL) {
3618                memcpy(tstate, master_tstate, sizeof(*tstate));
3619                memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
3620                for (i = 0; i < 16; i++) {
3621                        memset(&tstate->transinfo[i].curr, 0,
3622                              sizeof(tstate->transinfo[i].curr));
3623                        memset(&tstate->transinfo[i].goal, 0,
3624                              sizeof(tstate->transinfo[i].goal));
3625                }
3626        } else
3627                memset(tstate, 0, sizeof(*tstate));
3628        ahd->enabled_targets[scsi_id] = tstate;
3629        return (tstate);
3630}
3631
3632#ifdef AHD_TARGET_MODE
3633/*
3634 * Free per target mode instance (ID we respond to as a target)
3635 * transfer negotiation data structures.
3636 */
3637static void
3638ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
3639{
3640        struct ahd_tmode_tstate *tstate;
3641
3642        /*
3643         * Don't clean up our "master" tstate.
3644         * It has our default user settings.
3645         */
3646        if (scsi_id == ahd->our_id
3647         && force == FALSE)
3648                return;
3649
3650        tstate = ahd->enabled_targets[scsi_id];
3651        kfree(tstate);
3652        ahd->enabled_targets[scsi_id] = NULL;
3653}
3654#endif
3655
3656/*
3657 * Called when we have an active connection to a target on the bus,
3658 * this function finds the nearest period to the input period limited
3659 * by the capabilities of the bus connectivity of and sync settings for
3660 * the target.
3661 */
3662static void
3663ahd_devlimited_syncrate(struct ahd_softc *ahd,
3664                        struct ahd_initiator_tinfo *tinfo,
3665                        u_int *period, u_int *ppr_options, role_t role)
3666{
3667        struct  ahd_transinfo *transinfo;
3668        u_int   maxsync;
3669
3670        if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
3671         && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
3672                maxsync = AHD_SYNCRATE_PACED;
3673        } else {
3674                maxsync = AHD_SYNCRATE_ULTRA;
3675                /* Can't do DT related options on an SE bus */
3676                *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3677        }
3678        /*
3679         * Never allow a value higher than our current goal
3680         * period otherwise we may allow a target initiated
3681         * negotiation to go above the limit as set by the
3682         * user.  In the case of an initiator initiated
3683         * sync negotiation, we limit based on the user
3684         * setting.  This allows the system to still accept
3685         * incoming negotiations even if target initiated
3686         * negotiation is not performed.
3687         */
3688        if (role == ROLE_TARGET)
3689                transinfo = &tinfo->user;
3690        else
3691                transinfo = &tinfo->goal;
3692        *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
3693        if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
3694                maxsync = max(maxsync, (u_int)AHD_SYNCRATE_ULTRA2);
3695                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3696        }
3697        if (transinfo->period == 0) {
3698                *period = 0;
3699                *ppr_options = 0;
3700        } else {
3701                *period = max(*period, (u_int)transinfo->period);
3702                ahd_find_syncrate(ahd, period, ppr_options, maxsync);
3703        }
3704}
3705
3706/*
3707 * Look up the valid period to SCSIRATE conversion in our table.
3708 * Return the period and offset that should be sent to the target
3709 * if this was the beginning of an SDTR.
3710 */
3711void
3712ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
3713                  u_int *ppr_options, u_int maxsync)
3714{
3715        if (*period < maxsync)
3716                *period = maxsync;
3717
3718        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
3719         && *period > AHD_SYNCRATE_MIN_DT)
3720                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3721
3722        if (*period > AHD_SYNCRATE_MIN)
3723                *period = 0;
3724
3725        /* Honor PPR option conformance rules. */
3726        if (*period > AHD_SYNCRATE_PACED)
3727                *ppr_options &= ~MSG_EXT_PPR_RTI;
3728
3729        if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3730                *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
3731
3732        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
3733                *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3734
3735        /* Skip all PACED only entries if IU is not available */
3736        if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
3737         && *period < AHD_SYNCRATE_DT)
3738                *period = AHD_SYNCRATE_DT;
3739
3740        /* Skip all DT only entries if DT is not available */
3741        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3742         && *period < AHD_SYNCRATE_ULTRA2)
3743                *period = AHD_SYNCRATE_ULTRA2;
3744}
3745
3746/*
3747 * Truncate the given synchronous offset to a value the
3748 * current adapter type and syncrate are capable of.
3749 */
3750static void
3751ahd_validate_offset(struct ahd_softc *ahd,
3752                    struct ahd_initiator_tinfo *tinfo,
3753                    u_int period, u_int *offset, int wide,
3754                    role_t role)
3755{
3756        u_int maxoffset;
3757
3758        /* Limit offset to what we can do */
3759        if (period == 0)
3760                maxoffset = 0;
3761        else if (period <= AHD_SYNCRATE_PACED) {
3762                if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
3763                        maxoffset = MAX_OFFSET_PACED_BUG;
3764                else
3765                        maxoffset = MAX_OFFSET_PACED;
3766        } else
3767                maxoffset = MAX_OFFSET_NON_PACED;
3768        *offset = min(*offset, maxoffset);
3769        if (tinfo != NULL) {
3770                if (role == ROLE_TARGET)
3771                        *offset = min(*offset, (u_int)tinfo->user.offset);
3772                else
3773                        *offset = min(*offset, (u_int)tinfo->goal.offset);
3774        }
3775}
3776
3777/*
3778 * Truncate the given transfer width parameter to a value the
3779 * current adapter type is capable of.
3780 */
3781static void
3782ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3783                   u_int *bus_width, role_t role)
3784{
3785        switch (*bus_width) {
3786        default:
3787                if (ahd->features & AHD_WIDE) {
3788                        /* Respond Wide */
3789                        *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3790                        break;
3791                }
3792                fallthrough;
3793        case MSG_EXT_WDTR_BUS_8_BIT:
3794                *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3795                break;
3796        }
3797        if (tinfo != NULL) {
3798                if (role == ROLE_TARGET)
3799                        *bus_width = min((u_int)tinfo->user.width, *bus_width);
3800                else
3801                        *bus_width = min((u_int)tinfo->goal.width, *bus_width);
3802        }
3803}
3804
3805/*
3806 * Update the bitmask of targets for which the controller should
3807 * negotiate with at the next convenient opportunity.  This currently
3808 * means the next time we send the initial identify messages for
3809 * a new transaction.
3810 */
3811int
3812ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3813                       struct ahd_tmode_tstate *tstate,
3814                       struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3815{
3816        u_int auto_negotiate_orig;
3817
3818        auto_negotiate_orig = tstate->auto_negotiate;
3819        if (neg_type == AHD_NEG_ALWAYS) {
3820                /*
3821                 * Force our "current" settings to be
3822                 * unknown so that unless a bus reset
3823                 * occurs the need to renegotiate is
3824                 * recorded persistently.
3825                 */
3826                if ((ahd->features & AHD_WIDE) != 0)
3827                        tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3828                tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3829                tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3830        }
3831        if (tinfo->curr.period != tinfo->goal.period
3832         || tinfo->curr.width != tinfo->goal.width
3833         || tinfo->curr.offset != tinfo->goal.offset
3834         || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3835         || (neg_type == AHD_NEG_IF_NON_ASYNC
3836          && (tinfo->goal.offset != 0
3837           || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3838           || tinfo->goal.ppr_options != 0)))
3839                tstate->auto_negotiate |= devinfo->target_mask;
3840        else
3841                tstate->auto_negotiate &= ~devinfo->target_mask;
3842
3843        return (auto_negotiate_orig != tstate->auto_negotiate);
3844}
3845
3846/*
3847 * Update the user/goal/curr tables of synchronous negotiation
3848 * parameters as well as, in the case of a current or active update,
3849 * any data structures on the host controller.  In the case of an
3850 * active update, the specified target is currently talking to us on
3851 * the bus, so the transfer parameter update must take effect
3852 * immediately.
3853 */
3854void
3855ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3856                 u_int period, u_int offset, u_int ppr_options,
3857                 u_int type, int paused)
3858{
3859        struct  ahd_initiator_tinfo *tinfo;
3860        struct  ahd_tmode_tstate *tstate;
3861        u_int   old_period;
3862        u_int   old_offset;
3863        u_int   old_ppr;
3864        int     active;
3865        int     update_needed;
3866
3867        active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3868        update_needed = 0;
3869
3870        if (period == 0 || offset == 0) {
3871                period = 0;
3872                offset = 0;
3873        }
3874
3875        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3876                                    devinfo->target, &tstate);
3877
3878        if ((type & AHD_TRANS_USER) != 0) {
3879                tinfo->user.period = period;
3880                tinfo->user.offset = offset;
3881                tinfo->user.ppr_options = ppr_options;
3882        }
3883
3884        if ((type & AHD_TRANS_GOAL) != 0) {
3885                tinfo->goal.period = period;
3886                tinfo->goal.offset = offset;
3887                tinfo->goal.ppr_options = ppr_options;
3888        }
3889
3890        old_period = tinfo->curr.period;
3891        old_offset = tinfo->curr.offset;
3892        old_ppr    = tinfo->curr.ppr_options;
3893
3894        if ((type & AHD_TRANS_CUR) != 0
3895         && (old_period != period
3896          || old_offset != offset
3897          || old_ppr != ppr_options)) {
3898
3899                update_needed++;
3900
3901                tinfo->curr.period = period;
3902                tinfo->curr.offset = offset;
3903                tinfo->curr.ppr_options = ppr_options;
3904
3905                ahd_send_async(ahd, devinfo->channel, devinfo->target,
3906                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
3907                if (bootverbose) {
3908                        if (offset != 0) {
3909                                int options;
3910
3911                                printk("%s: target %d synchronous with "
3912                                       "period = 0x%x, offset = 0x%x",
3913                                       ahd_name(ahd), devinfo->target,
3914                                       period, offset);
3915                                options = 0;
3916                                if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3917                                        printk("(RDSTRM");
3918                                        options++;
3919                                }
3920                                if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3921                                        printk("%s", options ? "|DT" : "(DT");
3922                                        options++;
3923                                }
3924                                if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3925                                        printk("%s", options ? "|IU" : "(IU");
3926                                        options++;
3927                                }
3928                                if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3929                                        printk("%s", options ? "|RTI" : "(RTI");
3930                                        options++;
3931                                }
3932                                if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3933                                        printk("%s", options ? "|QAS" : "(QAS");
3934                                        options++;
3935                                }
3936                                if (options != 0)
3937                                        printk(")\n");
3938                                else
3939                                        printk("\n");
3940                        } else {
3941                                printk("%s: target %d using "
3942                                       "asynchronous transfers%s\n",
3943                                       ahd_name(ahd), devinfo->target,
3944                                       (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3945                                     ?  "(QAS)" : "");
3946                        }
3947                }
3948        }
3949        /*
3950         * Always refresh the neg-table to handle the case of the
3951         * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3952         * We will always renegotiate in that case if this is a
3953         * packetized request.  Also manage the busfree expected flag
3954         * from this common routine so that we catch changes due to
3955         * WDTR or SDTR messages.
3956         */
3957        if ((type & AHD_TRANS_CUR) != 0) {
3958                if (!paused)
3959                        ahd_pause(ahd);
3960                ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3961                if (!paused)
3962                        ahd_unpause(ahd);
3963                if (ahd->msg_type != MSG_TYPE_NONE) {
3964                        if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3965                         != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3966#ifdef AHD_DEBUG
3967                                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3968                                        ahd_print_devinfo(ahd, devinfo);
3969                                        printk("Expecting IU Change busfree\n");
3970                                }
3971#endif
3972                                ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3973                                               |  MSG_FLAG_IU_REQ_CHANGED;
3974                        }
3975                        if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3976#ifdef AHD_DEBUG
3977                                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3978                                        printk("PPR with IU_REQ outstanding\n");
3979#endif
3980                                ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3981                        }
3982                }
3983        }
3984
3985        update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3986                                                tinfo, AHD_NEG_TO_GOAL);
3987
3988        if (update_needed && active)
3989                ahd_update_pending_scbs(ahd);
3990}
3991
3992/*
3993 * Update the user/goal/curr tables of wide negotiation
3994 * parameters as well as, in the case of a current or active update,
3995 * any data structures on the host controller.  In the case of an
3996 * active update, the specified target is currently talking to us on
3997 * the bus, so the transfer parameter update must take effect
3998 * immediately.
3999 */
4000void
4001ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4002              u_int width, u_int type, int paused)
4003{
4004        struct  ahd_initiator_tinfo *tinfo;
4005        struct  ahd_tmode_tstate *tstate;
4006        u_int   oldwidth;
4007        int     active;
4008        int     update_needed;
4009
4010        active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
4011        update_needed = 0;
4012        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4013                                    devinfo->target, &tstate);
4014
4015        if ((type & AHD_TRANS_USER) != 0)
4016                tinfo->user.width = width;
4017
4018        if ((type & AHD_TRANS_GOAL) != 0)
4019                tinfo->goal.width = width;
4020
4021        oldwidth = tinfo->curr.width;
4022        if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
4023
4024                update_needed++;
4025
4026                tinfo->curr.width = width;
4027                ahd_send_async(ahd, devinfo->channel, devinfo->target,
4028                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
4029                if (bootverbose) {
4030                        printk("%s: target %d using %dbit transfers\n",
4031                               ahd_name(ahd), devinfo->target,
4032                               8 * (0x01 << width));
4033                }
4034        }
4035
4036        if ((type & AHD_TRANS_CUR) != 0) {
4037                if (!paused)
4038                        ahd_pause(ahd);
4039                ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
4040                if (!paused)
4041                        ahd_unpause(ahd);
4042        }
4043
4044        update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
4045                                                tinfo, AHD_NEG_TO_GOAL);
4046        if (update_needed && active)
4047                ahd_update_pending_scbs(ahd);
4048
4049}
4050
4051/*
4052 * Update the current state of tagged queuing for a given target.
4053 */
4054static void
4055ahd_set_tags(struct ahd_softc *ahd, struct scsi_cmnd *cmd,
4056             struct ahd_devinfo *devinfo, ahd_queue_alg alg)
4057{
4058        struct scsi_device *sdev = cmd->device;
4059
4060        ahd_platform_set_tags(ahd, sdev, devinfo, alg);
4061        ahd_send_async(ahd, devinfo->channel, devinfo->