drivers/scsi/aic7xxx/aic79xx_core.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2003 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
  41  */
  42
  43 #include "aic79xx_osm.h"
  44 #include "aic79xx_inline.h"
  45 #include "aicasm/aicasm_insformat.h"
  46
  47 /***************************** Lookup Tables **********************************/
  48 static const char *const ahd_chip_names[] =
  49 {
  50         "NONE",
  51         "aic7901",
  52         "aic7902",
  53         "aic7901A"
  54 };
  55
  56 /*
  57  * Hardware error codes.
  58  */
  59 struct ahd_hard_error_entry {
  60         uint8_t errno;
  61         const char *errmesg;
  62 };
  63
  64 static const struct ahd_hard_error_entry ahd_hard_errors[] = {
  65         { DSCTMOUT,     "Discard Timer has timed out" },
  66         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  67         { SQPARERR,     "Sequencer Parity Error" },
  68         { DPARERR,      "Data-path Parity Error" },
  69         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  70         { CIOPARERR,    "CIOBUS Parity Error" },
  71 };
  72 static const u_int num_errors = ARRAY_SIZE(ahd_hard_errors);
  73
  74 static const struct ahd_phase_table_entry ahd_phase_table[] =
  75 {
  76         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  77         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  78         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  79         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  80         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  81         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  82         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  83         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  84         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  85         { 0,            MSG_NOOP,               "in unknown phase"      }
  86 };
  87
  88 /*
  89  * In most cases we only wish to itterate over real phases, so
  90  * exclude the last element from the count.
  91  */
  92 static const u_int num_phases = ARRAY_SIZE(ahd_phase_table) - 1;
  93
  94 /* Our Sequencer Program */
  95 #include "aic79xx_seq.h"
  96
  97 /**************************** Function Declarations ***************************/
  98 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
  99 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 100                                                   u_int lqistat1);
 101 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 102                                                u_int busfreetime);
 103 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 104 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 105 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 106                                                 struct ahd_devinfo *devinfo);
 107
 108 static struct ahd_tmode_tstate*
 109                         ahd_alloc_tstate(struct ahd_softc *ahd,
 110                                          u_int scsi_id, char channel);
 111 #ifdef AHD_TARGET_MODE
 112 static void             ahd_free_tstate(struct ahd_softc *ahd,
 113                                         u_int scsi_id, char channel, int force);
 114 #endif
 115 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 116                                                 struct ahd_initiator_tinfo *,
 117                                                 u_int *period,
 118                                                 u_int *ppr_options,
 119                                                 role_t role);
 120 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 121                                              struct ahd_devinfo *devinfo,
 122                                              struct ahd_transinfo *tinfo);
 123 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 124 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 125                                           struct ahd_devinfo *devinfo);
 126 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 127                                         struct ahd_devinfo *devinfo,
 128                                         struct scb *scb);
 129 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 130                                                    struct ahd_devinfo *devinfo,
 131                                                    struct scb *scb);
 132 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 133                                                struct ahd_devinfo *devinfo);
 134 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 135                                            struct ahd_devinfo *devinfo,
 136                                            u_int period, u_int offset);
 137 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 138                                            struct ahd_devinfo *devinfo,
 139                                            u_int bus_width);
 140 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 141                                           struct ahd_devinfo *devinfo,
 142                                           u_int period, u_int offset,
 143                                           u_int bus_width, u_int ppr_options);
 144 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 145 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 146 typedef enum {
 147         AHDMSG_1B,
 148         AHDMSG_2B,
 149         AHDMSG_EXT
 150 } ahd_msgtype;
 151 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 152                                      u_int msgval, int full);
 153 static int              ahd_parse_msg(struct ahd_softc *ahd,
 154                                       struct ahd_devinfo *devinfo);
 155 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 156                                               struct ahd_devinfo *devinfo);
 157 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 158                                                 struct ahd_devinfo *devinfo);
 159 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 160 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 161                                             struct ahd_devinfo *devinfo,
 162                                             u_int lun, cam_status status,
 163                                             char *message, int verbose_level);
 164 #ifdef AHD_TARGET_MODE
 165 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 166                                                struct ahd_devinfo *devinfo,
 167                                                struct scb *scb);
 168 #endif
 169
 170 static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 171 static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 172 static bus_dmamap_callback_t
 173                         ahd_dmamap_cb;
 174 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 175 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 176 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 177 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 178 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 179 static void             ahd_add_col_list(struct ahd_softc *ahd,
 180                                          struct scb *scb, u_int col_idx);
 181 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 182                                          struct scb *scb);
 183 static void             ahd_chip_init(struct ahd_softc *ahd);
 184 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 185                                             struct scb *prev_scb,
 186                                             struct scb *scb);
 187 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 188 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 189                                             char channel, int lun, u_int tag,
 190                                             role_t role, uint32_t status,
 191                                             ahd_search_action action,
 192                                             u_int *list_head, u_int *list_tail,
 193                                             u_int tid);
 194 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 195                                             u_int tid_prev, u_int tid_cur,
 196                                             u_int tid_next);
 197 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 198                                                  u_int scbid);
 199 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 200                                      u_int prev, u_int next, u_int tid);
 201 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 202 static void             ahd_stat_timer(struct timer_list *t);
 203 #ifdef AHD_DUMP_SEQ
 204 static void             ahd_dumpseq(struct ahd_softc *ahd);
 205 #endif
 206 static void             ahd_loadseq(struct ahd_softc *ahd);
 207 static int              ahd_check_patch(struct ahd_softc *ahd,
 208                                         const struct patch **start_patch,
 209                                         u_int start_instr, u_int *skip_addr);
 210 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 211                                             u_int address);
 212 static void             ahd_download_instr(struct ahd_softc *ahd,
 213                                            u_int instrptr, uint8_t *dconsts);
 214 static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 215 static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 216                                                struct scb *scb);
 217 static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 218                                           struct scb *scb);
 219
 220 #ifdef AHD_TARGET_MODE
 221 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 222                                                struct ahd_tmode_lstate *lstate,
 223                                                u_int initiator_id,
 224                                                u_int event_type,
 225                                                u_int event_arg);
 226 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 227                                           u_int targid_mask);
 228 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 229                                               struct target_cmd *cmd);
 230 #endif
 231
 232 static int              ahd_abort_scbs(struct ahd_softc *ahd, int target,
 233                                        char channel, int lun, u_int tag,
 234                                        role_t role, uint32_t status);
 235 static void             ahd_alloc_scbs(struct ahd_softc *ahd);
 236 static void             ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl,
 237                                      u_int scbid);
 238 static void             ahd_calc_residual(struct ahd_softc *ahd,
 239                                           struct scb *scb);
 240 static void             ahd_clear_critical_section(struct ahd_softc *ahd);
 241 static void             ahd_clear_intstat(struct ahd_softc *ahd);
 242 static void             ahd_enable_coalescing(struct ahd_softc *ahd,
 243                                               int enable);
 244 static u_int            ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl);
 245 static void             ahd_freeze_devq(struct ahd_softc *ahd,
 246                                         struct scb *scb);
 247 static void             ahd_handle_scb_status(struct ahd_softc *ahd,
 248                                               struct scb *scb);
 249 static const struct ahd_phase_table_entry* ahd_lookup_phase_entry(int phase);
 250 static void             ahd_shutdown(void *arg);
 251 static void             ahd_update_coalescing_values(struct ahd_softc *ahd,
 252                                                      u_int timer,
 253                                                      u_int maxcmds,
 254                                                      u_int mincmds);
 255 static int              ahd_verify_vpd_cksum(struct vpd_config *vpd);
 256 static int              ahd_wait_seeprom(struct ahd_softc *ahd);
 257 static int              ahd_match_scb(struct ahd_softc *ahd, struct scb *scb,
 258                                       int target, char channel, int lun,
 259                                       u_int tag, role_t role);
 260
 261 static void             ahd_reset_cmds_pending(struct ahd_softc *ahd);
 262
 263 /*************************** Interrupt Services *******************************/
 264 static void             ahd_run_qoutfifo(struct ahd_softc *ahd);
 265 #ifdef AHD_TARGET_MODE
 266 static void             ahd_run_tqinfifo(struct ahd_softc *ahd, int paused);
 267 #endif
 268 static void             ahd_handle_hwerrint(struct ahd_softc *ahd);
 269 static void             ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat);
 270 static void             ahd_handle_scsiint(struct ahd_softc *ahd,
 271                                            u_int intstat);
 272
 273 /************************ Sequencer Execution Control *************************/
 274 void
 275 ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst)
 276 {
 277         if (ahd->src_mode == src && ahd->dst_mode == dst)
 278                 return;
 279 #ifdef AHD_DEBUG
 280         if (ahd->src_mode == AHD_MODE_UNKNOWN
 281          || ahd->dst_mode == AHD_MODE_UNKNOWN)
 282                 panic("Setting mode prior to saving it.\n");
 283         if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 284                 printk("%s: Setting mode 0x%x\n", ahd_name(ahd),
 285                        ahd_build_mode_state(ahd, src, dst));
 286 #endif
 287         ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst));
 288         ahd->src_mode = src;
 289         ahd->dst_mode = dst;
 290 }
 291
 292 static void
 293 ahd_update_modes(struct ahd_softc *ahd)
 294 {
 295         ahd_mode_state mode_ptr;
 296         ahd_mode src;
 297         ahd_mode dst;
 298
 299         mode_ptr = ahd_inb(ahd, MODE_PTR);
 300 #ifdef AHD_DEBUG
 301         if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
 302                 printk("Reading mode 0x%x\n", mode_ptr);
 303 #endif
 304         ahd_extract_mode_state(ahd, mode_ptr, &src, &dst);
 305         ahd_known_modes(ahd, src, dst);
 306 }
 307
 308 static void
 309 ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode,
 310                  ahd_mode dstmode, const char *file, int line)
 311 {
 312 #ifdef AHD_DEBUG
 313         if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0
 314          || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) {
 315                 panic("%s:%s:%d: Mode assertion failed.\n",
 316                        ahd_name(ahd), file, line);
 317         }
 318 #endif
 319 }
 320
 321 #define AHD_ASSERT_MODES(ahd, source, dest) \
 322         ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__);
 323
 324 ahd_mode_state
 325 ahd_save_modes(struct ahd_softc *ahd)
 326 {
 327         if (ahd->src_mode == AHD_MODE_UNKNOWN
 328          || ahd->dst_mode == AHD_MODE_UNKNOWN)
 329                 ahd_update_modes(ahd);
 330
 331         return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode));
 332 }
 333
 334 void
 335 ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state)
 336 {
 337         ahd_mode src;
 338         ahd_mode dst;
 339
 340         ahd_extract_mode_state(ahd, state, &src, &dst);
 341         ahd_set_modes(ahd, src, dst);
 342 }
 343
 344 /*
 345  * Determine whether the sequencer has halted code execution.
 346  * Returns non-zero status if the sequencer is stopped.
 347  */
 348 int
 349 ahd_is_paused(struct ahd_softc *ahd)
 350 {
 351         return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0);
 352 }
 353
 354 /*
 355  * Request that the sequencer stop and wait, indefinitely, for it
 356  * to stop.  The sequencer will only acknowledge that it is paused
 357  * once it has reached an instruction boundary and PAUSEDIS is
 358  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 359  * for critical sections.
 360  */
 361 void
 362 ahd_pause(struct ahd_softc *ahd)
 363 {
 364         ahd_outb(ahd, HCNTRL, ahd->pause);
 365
 366         /*
 367          * Since the sequencer can disable pausing in a critical section, we
 368          * must loop until it actually stops.
 369          */
 370         while (ahd_is_paused(ahd) == 0)
 371                 ;
 372 }
 373
 374 /*
 375  * Allow the sequencer to continue program execution.
 376  * We check here to ensure that no additional interrupt
 377  * sources that would cause the sequencer to halt have been
 378  * asserted.  If, for example, a SCSI bus reset is detected
 379  * while we are fielding a different, pausing, interrupt type,
 380  * we don't want to release the sequencer before going back
 381  * into our interrupt handler and dealing with this new
 382  * condition.
 383  */
 384 void
 385 ahd_unpause(struct ahd_softc *ahd)
 386 {
 387         /*
 388          * Automatically restore our modes to those saved
 389          * prior to the first change of the mode.
 390          */
 391         if (ahd->saved_src_mode != AHD_MODE_UNKNOWN
 392          && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) {
 393                 if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0)
 394                         ahd_reset_cmds_pending(ahd);
 395                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
 396         }
 397
 398         if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0)
 399                 ahd_outb(ahd, HCNTRL, ahd->unpause);
 400
 401         ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN);
 402 }
 403
 404 /*********************** Scatter Gather List Handling *************************/
 405 void *
 406 ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb,
 407              void *sgptr, dma_addr_t addr, bus_size_t len, int last)
 408 {
 409         scb->sg_count++;
 410         if (sizeof(dma_addr_t) > 4
 411          && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 412                 struct ahd_dma64_seg *sg;
 413
 414                 sg = (struct ahd_dma64_seg *)sgptr;
 415                 sg->addr = ahd_htole64(addr);
 416                 sg->len = ahd_htole32(len | (last ? AHD_DMA_LAST_SEG : 0));
 417                 return (sg + 1);
 418         } else {
 419                 struct ahd_dma_seg *sg;
 420
 421                 sg = (struct ahd_dma_seg *)sgptr;
 422                 sg->addr = ahd_htole32(addr & 0xFFFFFFFF);
 423                 sg->len = ahd_htole32(len | ((addr >> 8) & 0x7F000000)
 424                                     | (last ? AHD_DMA_LAST_SEG : 0));
 425                 return (sg + 1);
 426         }
 427 }
 428
 429 static void
 430 ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb)
 431 {
 432         /* XXX Handle target mode SCBs. */
 433         scb->crc_retry_count = 0;
 434         if ((scb->flags & SCB_PACKETIZED) != 0) {
 435                 /* XXX what about ACA??  It is type 4, but TAG_TYPE == 0x3. */
 436                 scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE;
 437         } else {
 438                 if (ahd_get_transfer_length(scb) & 0x01)
 439                         scb->hscb->task_attribute = SCB_XFERLEN_ODD;
 440                 else
 441                         scb->hscb->task_attribute = 0;
 442         }
 443
 444         if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR
 445          || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0)
 446                 scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr =
 447                     ahd_htole32(scb->sense_busaddr);
 448 }
 449
 450 static void
 451 ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb)
 452 {
 453         /*
 454          * Copy the first SG into the "current" data ponter area.
 455          */
 456         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 457                 struct ahd_dma64_seg *sg;
 458
 459                 sg = (struct ahd_dma64_seg *)scb->sg_list;
 460                 scb->hscb->dataptr = sg->addr;
 461                 scb->hscb->datacnt = sg->len;
 462         } else {
 463                 struct ahd_dma_seg *sg;
 464                 uint32_t *dataptr_words;
 465
 466                 sg = (struct ahd_dma_seg *)scb->sg_list;
 467                 dataptr_words = (uint32_t*)&scb->hscb->dataptr;
 468                 dataptr_words[0] = sg->addr;
 469                 dataptr_words[1] = 0;
 470                 if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) {
 471                         uint64_t high_addr;
 472
 473                         high_addr = ahd_le32toh(sg->len) & 0x7F000000;
 474                         scb->hscb->dataptr |= ahd_htole64(high_addr << 8);
 475                 }
 476                 scb->hscb->datacnt = sg->len;
 477         }
 478         /*
 479          * Note where to find the SG entries in bus space.
 480          * We also set the full residual flag which the
 481          * sequencer will clear as soon as a data transfer
 482          * occurs.
 483          */
 484         scb->hscb->sgptr = ahd_htole32(scb->sg_list_busaddr|SG_FULL_RESID);
 485 }
 486
 487 static void
 488 ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb)
 489 {
 490         scb->hscb->sgptr = ahd_htole32(SG_LIST_NULL);
 491         scb->hscb->dataptr = 0;
 492         scb->hscb->datacnt = 0;
 493 }
 494
 495 /************************** Memory mapping routines ***************************/
 496 static void *
 497 ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr)
 498 {
 499         dma_addr_t sg_offset;
 500
 501         /* sg_list_phys points to entry 1, not 0 */
 502         sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd));
 503         return ((uint8_t *)scb->sg_list + sg_offset);
 504 }
 505
 506 static uint32_t
 507 ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg)
 508 {
 509         dma_addr_t sg_offset;
 510
 511         /* sg_list_phys points to entry 1, not 0 */
 512         sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list)
 513                   - ahd_sg_size(ahd);
 514
 515         return (scb->sg_list_busaddr + sg_offset);
 516 }
 517
 518 static void
 519 ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op)
 520 {
 521         ahd_dmamap_sync(ahd, ahd->scb_data.hscb_dmat,
 522                         scb->hscb_map->dmamap,
 523                         /*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr,
 524                         /*len*/sizeof(*scb->hscb), op);
 525 }
 526
 527 void
 528 ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op)
 529 {
 530         if (scb->sg_count == 0)
 531                 return;
 532
 533         ahd_dmamap_sync(ahd, ahd->scb_data.sg_dmat,
 534                         scb->sg_map->dmamap,
 535                         /*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd),
 536                         /*len*/ahd_sg_size(ahd) * scb->sg_count, op);
 537 }
 538
 539 static void
 540 ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op)
 541 {
 542         ahd_dmamap_sync(ahd, ahd->scb_data.sense_dmat,
 543                         scb->sense_map->dmamap,
 544                         /*offset*/scb->sense_busaddr,
 545                         /*len*/AHD_SENSE_BUFSIZE, op);
 546 }
 547
 548 #ifdef AHD_TARGET_MODE
 549 static uint32_t
 550 ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index)
 551 {
 552         return (((uint8_t *)&ahd->targetcmds[index])
 553                - (uint8_t *)ahd->qoutfifo);
 554 }
 555 #endif
 556
 557 /*********************** Miscellaneous Support Functions ***********************/
 558 /*
 559  * Return pointers to the transfer negotiation information
 560  * for the specified our_id/remote_id pair.
 561  */
 562 struct ahd_initiator_tinfo *
 563 ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id,
 564                     u_int remote_id, struct ahd_tmode_tstate **tstate)
 565 {
 566         /*
 567          * Transfer data structures are stored from the perspective
 568          * of the target role.  Since the parameters for a connection
 569          * in the initiator role to a given target are the same as
 570          * when the roles are reversed, we pretend we are the target.
 571          */
 572         if (channel == 'B')
 573                 our_id += 8;
 574         *tstate = ahd->enabled_targets[our_id];
 575         return (&(*tstate)->transinfo[remote_id]);
 576 }
 577
 578 uint16_t
 579 ahd_inw(struct ahd_softc *ahd, u_int port)
 580 {
 581         /*
 582          * Read high byte first as some registers increment
 583          * or have other side effects when the low byte is
 584          * read.
 585          */
 586         uint16_t r = ahd_inb(ahd, port+1) << 8;
 587         return r | ahd_inb(ahd, port);
 588 }
 589
 590 void
 591 ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
 592 {
 593         /*
 594          * Write low byte first to accommodate registers
 595          * such as PRGMCNT where the order maters.
 596          */
 597         ahd_outb(ahd, port, value & 0xFF);
 598         ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 599 }
 600
 601 uint32_t
 602 ahd_inl(struct ahd_softc *ahd, u_int port)
 603 {
 604         return ((ahd_inb(ahd, port))
 605               | (ahd_inb(ahd, port+1) << 8)
 606               | (ahd_inb(ahd, port+2) << 16)
 607               | (ahd_inb(ahd, port+3) << 24));
 608 }
 609
 610 void
 611 ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
 612 {
 613         ahd_outb(ahd, port, (value) & 0xFF);
 614         ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
 615         ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
 616         ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
 617 }
 618
 619 uint64_t
 620 ahd_inq(struct ahd_softc *ahd, u_int port)
 621 {
 622         return ((ahd_inb(ahd, port))
 623               | (ahd_inb(ahd, port+1) << 8)
 624               | (ahd_inb(ahd, port+2) << 16)
 625               | (ahd_inb(ahd, port+3) << 24)
 626               | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
 627               | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
 628               | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
 629               | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
 630 }
 631
 632 void
 633 ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
 634 {
 635         ahd_outb(ahd, port, value & 0xFF);
 636         ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
 637         ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
 638         ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
 639         ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
 640         ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
 641         ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
 642         ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
 643 }
 644
 645 u_int
 646 ahd_get_scbptr(struct ahd_softc *ahd)
 647 {
 648         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 649                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 650         return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
 651 }
 652
 653 void
 654 ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
 655 {
 656         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 657                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 658         ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
 659         ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
 660 }
 661
 662 #if 0 /* unused */
 663 static u_int
 664 ahd_get_hnscb_qoff(struct ahd_softc *ahd)
 665 {
 666         return (ahd_inw_atomic(ahd, HNSCB_QOFF));
 667 }
 668 #endif
 669
 670 static void
 671 ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
 672 {
 673         ahd_outw_atomic(ahd, HNSCB_QOFF, value);
 674 }
 675
 676 #if 0 /* unused */
 677 static u_int
 678 ahd_get_hescb_qoff(struct ahd_softc *ahd)
 679 {
 680         return (ahd_inb(ahd, HESCB_QOFF));
 681 }
 682 #endif
 683
 684 static void
 685 ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
 686 {
 687         ahd_outb(ahd, HESCB_QOFF, value);
 688 }
 689
 690 static u_int
 691 ahd_get_snscb_qoff(struct ahd_softc *ahd)
 692 {
 693         u_int oldvalue;
 694
 695         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 696         oldvalue = ahd_inw(ahd, SNSCB_QOFF);
 697         ahd_outw(ahd, SNSCB_QOFF, oldvalue);
 698         return (oldvalue);
 699 }
 700
 701 static void
 702 ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
 703 {
 704         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 705         ahd_outw(ahd, SNSCB_QOFF, value);
 706 }
 707
 708 #if 0 /* unused */
 709 static u_int
 710 ahd_get_sescb_qoff(struct ahd_softc *ahd)
 711 {
 712         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 713         return (ahd_inb(ahd, SESCB_QOFF));
 714 }
 715 #endif
 716
 717 static void
 718 ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
 719 {
 720         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 721         ahd_outb(ahd, SESCB_QOFF, value);
 722 }
 723
 724 #if 0 /* unused */
 725 static u_int
 726 ahd_get_sdscb_qoff(struct ahd_softc *ahd)
 727 {
 728         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 729         return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
 730 }
 731 #endif
 732
 733 static void
 734 ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
 735 {
 736         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
 737         ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
 738         ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
 739 }
 740
 741 u_int
 742 ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
 743 {
 744         u_int value;
 745
 746         /*
 747          * Workaround PCI-X Rev A. hardware bug.
 748          * After a host read of SCB memory, the chip
 749          * may become confused into thinking prefetch
 750          * was required.  This starts the discard timer
 751          * running and can cause an unexpected discard
 752          * timer interrupt.  The work around is to read
 753          * a normal register prior to the exhaustion of
 754          * the discard timer.  The mode pointer register
 755          * has no side effects and so serves well for
 756          * this purpose.
 757          *
 758          * Razor #528
 759          */
 760         value = ahd_inb(ahd, offset);
 761         if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
 762                 ahd_inb(ahd, MODE_PTR);
 763         return (value);
 764 }
 765
 766 u_int
 767 ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
 768 {
 769         return (ahd_inb_scbram(ahd, offset)
 770               | (ahd_inb_scbram(ahd, offset+1) << 8));
 771 }
 772
 773 static uint32_t
 774 ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
 775 {
 776         return (ahd_inw_scbram(ahd, offset)
 777               | (ahd_inw_scbram(ahd, offset+2) << 16));
 778 }
 779
 780 static uint64_t
 781 ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
 782 {
 783         return (ahd_inl_scbram(ahd, offset)
 784               | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
 785 }
 786
 787 struct scb *
 788 ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
 789 {
 790         struct scb* scb;
 791
 792         if (tag >= AHD_SCB_MAX)
 793                 return (NULL);
 794         scb = ahd->scb_data.scbindex[tag];
 795         if (scb != NULL)
 796                 ahd_sync_scb(ahd, scb,
 797                              BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 798         return (scb);
 799 }
 800
 801 static void
 802 ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
 803 {
 804         struct   hardware_scb *q_hscb;
 805         struct   map_node *q_hscb_map;
 806         uint32_t saved_hscb_busaddr;
 807
 808         /*
 809          * Our queuing method is a bit tricky.  The card
 810          * knows in advance which HSCB (by address) to download,
 811          * and we can't disappoint it.  To achieve this, the next
 812          * HSCB to download is saved off in ahd->next_queued_hscb.
 813          * When we are called to queue "an arbitrary scb",
 814          * we copy the contents of the incoming HSCB to the one
 815          * the sequencer knows about, swap HSCB pointers and
 816          * finally assign the SCB to the tag indexed location
 817          * in the scb_array.  This makes sure that we can still
 818          * locate the correct SCB by SCB_TAG.
 819          */
 820         q_hscb = ahd->next_queued_hscb;
 821         q_hscb_map = ahd->next_queued_hscb_map;
 822         saved_hscb_busaddr = q_hscb->hscb_busaddr;
 823         memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 824         q_hscb->hscb_busaddr = saved_hscb_busaddr;
 825         q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
 826
 827         /* Now swap HSCB pointers. */
 828         ahd->next_queued_hscb = scb->hscb;
 829         ahd->next_queued_hscb_map = scb->hscb_map;
 830         scb->hscb = q_hscb;
 831         scb->hscb_map = q_hscb_map;
 832
 833         /* Now define the mapping from tag to SCB in the scbindex */
 834         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
 835 }
 836
 837 /*
 838  * Tell the sequencer about a new transaction to execute.
 839  */
 840 void
 841 ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
 842 {
 843         ahd_swap_with_next_hscb(ahd, scb);
 844
 845         if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
 846                 panic("Attempt to queue invalid SCB tag %x\n",
 847                       SCB_GET_TAG(scb));
 848
 849         /*
 850          * Keep a history of SCBs we've downloaded in the qinfifo.
 851          */
 852         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
 853         ahd->qinfifonext++;
 854
 855         if (scb->sg_count != 0)
 856                 ahd_setup_data_scb(ahd, scb);
 857         else
 858                 ahd_setup_noxfer_scb(ahd, scb);
 859         ahd_setup_scb_common(ahd, scb);
 860
 861         /*
 862          * Make sure our data is consistent from the
 863          * perspective of the adapter.
 864          */
 865         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 866
 867 #ifdef AHD_DEBUG
 868         if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
 869                 uint64_t host_dataptr;
 870
 871                 host_dataptr = ahd_le64toh(scb->hscb->dataptr);
 872                 printk("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
 873                        ahd_name(ahd),
 874                        SCB_GET_TAG(scb), scb->hscb->scsiid,
 875                        ahd_le32toh(scb->hscb->hscb_busaddr),
 876                        (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
 877                        (u_int)(host_dataptr & 0xFFFFFFFF),
 878                        ahd_le32toh(scb->hscb->datacnt));
 879         }
 880 #endif
 881         /* Tell the adapter about the newly queued SCB */
 882         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
 883 }
 884
 885 /************************** Interrupt Processing ******************************/
 886 static void
 887 ahd_sync_qoutfifo(struct ahd_softc *ahd, int op)
 888 {
 889         ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 890                         /*offset*/0,
 891                         /*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op);
 892 }
 893
 894 static void
 895 ahd_sync_tqinfifo(struct ahd_softc *ahd, int op)
 896 {
 897 #ifdef AHD_TARGET_MODE
 898         if ((ahd->flags & AHD_TARGETROLE) != 0) {
 899                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 900                                 ahd->shared_data_map.dmamap,
 901                                 ahd_targetcmd_offset(ahd, 0),
 902                                 sizeof(struct target_cmd) * AHD_TMODE_CMDS,
 903                                 op);
 904         }
 905 #endif
 906 }
 907
 908 /*
 909  * See if the firmware has posted any completed commands
 910  * into our in-core command complete fifos.
 911  */
 912 #define AHD_RUN_QOUTFIFO 0x1
 913 #define AHD_RUN_TQINFIFO 0x2
 914 static u_int
 915 ahd_check_cmdcmpltqueues(struct ahd_softc *ahd)
 916 {
 917         u_int retval;
 918
 919         retval = 0;
 920         ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 921                         /*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo),
 922                         /*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD);
 923         if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag
 924           == ahd->qoutfifonext_valid_tag)
 925                 retval |= AHD_RUN_QOUTFIFO;
 926 #ifdef AHD_TARGET_MODE
 927         if ((ahd->flags & AHD_TARGETROLE) != 0
 928          && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) {
 929                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
 930                                 ahd->shared_data_map.dmamap,
 931                                 ahd_targetcmd_offset(ahd, ahd->tqinfifofnext),
 932                                 /*len*/sizeof(struct target_cmd),
 933                                 BUS_DMASYNC_POSTREAD);
 934                 if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0)
 935                         retval |= AHD_RUN_TQINFIFO;
 936         }
 937 #endif
 938         return (retval);
 939 }
 940
 941 /*
 942  * Catch an interrupt from the adapter
 943  */
 944 int
 945 ahd_intr(struct ahd_softc *ahd)
 946 {
 947         u_int   intstat;
 948
 949         if ((ahd->pause & INTEN) == 0) {
 950                 /*
 951                  * Our interrupt is not enabled on the chip
 952                  * and may be disabled for re-entrancy reasons,
 953                  * so just return.  This is likely just a shared
 954                  * interrupt.
 955                  */
 956                 return (0);
 957         }
 958
 959         /*
 960          * Instead of directly reading the interrupt status register,
 961          * infer the cause of the interrupt by checking our in-core
 962          * completion queues.  This avoids a costly PCI bus read in
 963          * most cases.
 964          */
 965         if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0
 966          && (ahd_check_cmdcmpltqueues(ahd) != 0))
 967                 intstat = CMDCMPLT;
 968         else
 969                 intstat = ahd_inb(ahd, INTSTAT);
 970
 971         if ((intstat & INT_PEND) == 0)
 972                 return (0);
 973
 974         if (intstat & CMDCMPLT) {
 975                 ahd_outb(ahd, CLRINT, CLRCMDINT);
 976
 977                 /*
 978                  * Ensure that the chip sees that we've cleared
 979                  * this interrupt before we walk the output fifo.
 980                  * Otherwise, we may, due to posted bus writes,
 981                  * clear the interrupt after we finish the scan,
 982                  * and after the sequencer has added new entries
 983                  * and asserted the interrupt again.
 984                  */
 985                 if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 986                         if (ahd_is_paused(ahd)) {
 987                                 /*
 988                                  * Potentially lost SEQINT.
 989                                  * If SEQINTCODE is non-zero,
 990                                  * simulate the SEQINT.
 991                                  */
 992                                 if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT)
 993                                         intstat |= SEQINT;
 994                         }
 995                 } else {
 996                         ahd_flush_device_writes(ahd);
 997                 }
 998                 ahd_run_qoutfifo(ahd);
 999                 ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++;
1000                 ahd->cmdcmplt_total++;
1001 #ifdef AHD_TARGET_MODE
1002                 if ((ahd->flags & AHD_TARGETROLE) != 0)
1003                         ahd_run_tqinfifo(ahd, /*paused*/FALSE);
1004 #endif
1005         }
1006
1007         /*
1008          * Handle statuses that may invalidate our cached
1009          * copy of INTSTAT separately.
1010          */
1011         if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) {
1012                 /* Hot eject.  Do nothing */
1013         } else if (intstat & HWERRINT) {
1014                 ahd_handle_hwerrint(ahd);
1015         } else if ((intstat & (PCIINT|SPLTINT)) != 0) {
1016                 ahd->bus_intr(ahd);
1017         } else {
1018
1019                 if ((intstat & SEQINT) != 0)
1020                         ahd_handle_seqint(ahd, intstat);
1021
1022                 if ((intstat & SCSIINT) != 0)
1023                         ahd_handle_scsiint(ahd, intstat);
1024         }
1025         return (1);
1026 }
1027
1028 /******************************** Private Inlines *****************************/
1029 static inline void
1030 ahd_assert_atn(struct ahd_softc *ahd)
1031 {
1032         ahd_outb(ahd, SCSISIGO, ATNO);
1033 }
1034
1035 /*
1036  * Determine if the current connection has a packetized
1037  * agreement.  This does not necessarily mean that we
1038  * are currently in a packetized transfer.  We could
1039  * just as easily be sending or receiving a message.
1040  */
1041 static int
1042 ahd_currently_packetized(struct ahd_softc *ahd)
1043 {
1044         ahd_mode_state   saved_modes;
1045         int              packetized;
1046
1047         saved_modes = ahd_save_modes(ahd);
1048         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
1049                 /*
1050                  * The packetized bit refers to the last
1051                  * connection, not the current one.  Check
1052                  * for non-zero LQISTATE instead.
1053                  */
1054                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1055                 packetized = ahd_inb(ahd, LQISTATE) != 0;
1056         } else {
1057                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1058                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
1059         }
1060         ahd_restore_modes(ahd, saved_modes);
1061         return (packetized);
1062 }
1063
1064 static inline int
1065 ahd_set_active_fifo(struct ahd_softc *ahd)
1066 {
1067         u_int active_fifo;
1068
1069         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1070         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
1071         switch (active_fifo) {
1072         case 0:
1073         case 1:
1074                 ahd_set_modes(ahd, active_fifo, active_fifo);
1075                 return (1);
1076         default:
1077                 return (0);
1078         }
1079 }
1080
1081 static inline void
1082 ahd_unbusy_tcl(struct ahd_softc *ahd, u_int tcl)
1083 {
1084         ahd_busy_tcl(ahd, tcl, SCB_LIST_NULL);
1085 }
1086
1087 /*
1088  * Determine whether the sequencer reported a residual
1089  * for this SCB/transaction.
1090  */
1091 static inline void
1092 ahd_update_residual(struct ahd_softc *ahd, struct scb *scb)
1093 {
1094         uint32_t sgptr;
1095
1096         sgptr = ahd_le32toh(scb->hscb->sgptr);
1097         if ((sgptr & SG_STATUS_VALID) != 0)
1098                 ahd_calc_residual(ahd, scb);
1099 }
1100
1101 static inline void
1102 ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb)
1103 {
1104         uint32_t sgptr;
1105
1106         sgptr = ahd_le32toh(scb->hscb->sgptr);
1107         if ((sgptr & SG_STATUS_VALID) != 0)
1108                 ahd_handle_scb_status(ahd, scb);
1109         else
1110                 ahd_done(ahd, scb);
1111 }
1112
1113
1114 /************************* Sequencer Execution Control ************************/
1115 /*
1116  * Restart the sequencer program from address zero
1117  */
1118 static void
1119 ahd_restart(struct ahd_softc *ahd)
1120 {
1121
1122         ahd_pause(ahd);
1123
1124         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1125
1126         /* No more pending messages */
1127         ahd_clear_msg_state(ahd);
1128         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
1129         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
1130         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
1131         ahd_outb(ahd, SEQINTCTL, 0);
1132         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
1133         ahd_outb(ahd, SEQ_FLAGS, 0);
1134         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
1135         ahd_outb(ahd, SAVED_LUN, 0xFF);
1136
1137         /*
1138          * Ensure that the sequencer's idea of TQINPOS
1139          * matches our own.  The sequencer increments TQINPOS
1140          * only after it sees a DMA complete and a reset could
1141          * occur before the increment leaving the kernel to believe
1142          * the command arrived but the sequencer to not.
1143          */
1144         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
1145
1146         /* Always allow reselection */
1147         ahd_outb(ahd, SCSISEQ1,
1148                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
1149         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1150
1151         /*
1152          * Clear any pending sequencer interrupt.  It is no
1153          * longer relevant since we're resetting the Program
1154          * Counter.
1155          */
1156         ahd_outb(ahd, CLRINT, CLRSEQINT);
1157
1158         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
1159         ahd_unpause(ahd);
1160 }
1161
1162 static void
1163 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
1164 {
1165         ahd_mode_state   saved_modes;
1166
1167 #ifdef AHD_DEBUG
1168         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
1169                 printk("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
1170 #endif
1171         saved_modes = ahd_save_modes(ahd);
1172         ahd_set_modes(ahd, fifo, fifo);
1173         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
1174         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1175                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
1176         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1177         ahd_outb(ahd, SG_STATE, 0);
1178         ahd_restore_modes(ahd, saved_modes);
1179 }
1180
1181 /************************* Input/Output Queues ********************************/
1182 /*
1183  * Flush and completed commands that are sitting in the command
1184  * complete queues down on the chip but have yet to be dma'ed back up.
1185  */
1186 static void
1187 ahd_flush_qoutfifo(struct ahd_softc *ahd)
1188 {
1189         struct          scb *scb;
1190         ahd_mode_state  saved_modes;
1191         u_int           saved_scbptr;
1192         u_int           ccscbctl;
1193         u_int           scbid;
1194         u_int           next_scbid;
1195
1196         saved_modes = ahd_save_modes(ahd);
1197
1198         /*
1199          * Flush the good status FIFO for completed packetized commands.
1200          */
1201         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1202         saved_scbptr = ahd_get_scbptr(ahd);
1203         while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
1204                 u_int fifo_mode;
1205                 u_int i;
1206
1207                 scbid = ahd_inw(ahd, GSFIFO);
1208                 scb = ahd_lookup_scb(ahd, scbid);
1209                 if (scb == NULL) {
1210                         printk("%s: Warning - GSFIFO SCB %d invalid\n",
1211                                ahd_name(ahd), scbid);
1212                         continue;
1213                 }
1214                 /*
1215                  * Determine if this transaction is still active in
1216                  * any FIFO.  If it is, we must flush that FIFO to
1217                  * the host before completing the  command.
1218                  */
1219                 fifo_mode = 0;
1220 rescan_fifos:
1221                 for (i = 0; i < 2; i++) {
1222                         /* Toggle to the other mode. */
1223                         fifo_mode ^= 1;
1224                         ahd_set_modes(ahd, fifo_mode, fifo_mode);
1225
1226                         if (ahd_scb_active_in_fifo(ahd, scb) == 0)
1227                                 continue;
1228
1229                         ahd_run_data_fifo(ahd, scb);
1230
1231                         /*
1232                          * Running this FIFO may cause a CFG4DATA for
1233                          * this same transaction to assert in the other
1234                          * FIFO or a new snapshot SAVEPTRS interrupt
1235                          * in this FIFO.  Even running a FIFO may not
1236                          * clear the transaction if we are still waiting
1237                          * for data to drain to the host. We must loop
1238                          * until the transaction is not active in either
1239                          * FIFO just to be sure.  Reset our loop counter
1240                          * so we will visit both FIFOs again before
1241                          * declaring this transaction finished.  We
1242                          * also delay a bit so that status has a chance
1243                          * to change before we look at this FIFO again.
1244                          */
1245                         ahd_delay(200);
1246                         goto rescan_fifos;
1247                 }
1248                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1249                 ahd_set_scbptr(ahd, scbid);
1250                 if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
1251                  && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
1252                   || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
1253                       & SG_LIST_NULL) != 0)) {
1254                         u_int comp_head;
1255
1256                         /*
1257                          * The transfer completed with a residual.
1258                          * Place this SCB on the complete DMA list
1259                          * so that we update our in-core copy of the
1260                          * SCB before completing the command.
1261                          */
1262                         ahd_outb(ahd, SCB_SCSI_STATUS, 0);
1263                         ahd_outb(ahd, SCB_SGPTR,
1264                                  ahd_inb_scbram(ahd, SCB_SGPTR)
1265                                  | SG_STATUS_VALID);
1266                         ahd_outw(ahd, SCB_TAG, scbid);
1267                         ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
1268                         comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1269                         if (SCBID_IS_NULL(comp_head)) {
1270                                 ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
1271                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1272                         } else {
1273                                 u_int tail;
1274
1275                                 tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
1276                                 ahd_set_scbptr(ahd, tail);
1277                                 ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
1278                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1279                                 ahd_set_scbptr(ahd, scbid);
1280                         }
1281                 } else
1282                         ahd_complete_scb(ahd, scb);
1283         }
1284         ahd_set_scbptr(ahd, saved_scbptr);
1285
1286         /*
1287          * Setup for command channel portion of flush.
1288          */
1289         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1290
1291         /*
1292          * Wait for any inprogress DMA to complete and clear DMA state
1293          * if this is for an SCB in the qinfifo.
1294          */
1295         while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
1296
1297                 if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
1298                         if ((ccscbctl & ARRDONE) != 0)
1299                                 break;
1300                 } else if ((ccscbctl & CCSCBDONE) != 0)
1301                         break;
1302                 ahd_delay(200);
1303         }
1304         /*
1305          * We leave the sequencer to cleanup in the case of DMA's to
1306          * update the qoutfifo.  In all other cases (DMA's to the
1307          * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
1308          * we disable the DMA engine so that the sequencer will not
1309          * attempt to handle the DMA completion.
1310          */
1311         if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
1312                 ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
1313
1314         /*
1315          * Complete any SCBs that just finished
1316          * being DMA'ed into the qoutfifo.
1317          */
1318         ahd_run_qoutfifo(ahd);
1319
1320         saved_scbptr = ahd_get_scbptr(ahd);
1321         /*
1322          * Manually update/complete any completed SCBs that are waiting to be
1323          * DMA'ed back up to the host.
1324          */
1325         scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1326         while (!SCBID_IS_NULL(scbid)) {
1327                 uint8_t *hscb_ptr;
1328                 u_int    i;
1329
1330                 ahd_set_scbptr(ahd, scbid);
1331                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1332                 scb = ahd_lookup_scb(ahd, scbid);
1333                 if (scb == NULL) {
1334                         printk("%s: Warning - DMA-up and complete "
1335                                "SCB %d invalid\n", ahd_name(ahd), scbid);
1336                         continue;
1337                 }
1338                 hscb_ptr = (uint8_t *)scb->hscb;
1339                 for (i = 0; i < sizeof(struct hardware_scb); i++)
1340                         *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
1341
1342                 ahd_complete_scb(ahd, scb);
1343                 scbid = next_scbid;
1344         }
1345         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
1346         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
1347
1348         scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
1349         while (!SCBID_IS_NULL(scbid)) {
1350
1351                 ahd_set_scbptr(ahd, scbid);
1352                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1353                 scb = ahd_lookup_scb(ahd, scbid);
1354                 if (scb == NULL) {
1355                         printk("%s: Warning - Complete Qfrz SCB %d invalid\n",
1356                                ahd_name(ahd), scbid);
1357                         continue;
1358                 }
1359
1360                 ahd_complete_scb(ahd, scb);
1361                 scbid = next_scbid;
1362         }
1363         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
1364
1365         scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
1366         while (!SCBID_IS_NULL(scbid)) {
1367
1368                 ahd_set_scbptr(ahd, scbid);
1369                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1370                 scb = ahd_lookup_scb(ahd, scbid);
1371                 if (scb == NULL) {
1372                         printk("%s: Warning - Complete SCB %d invalid\n",
1373                                ahd_name(ahd), scbid);
1374                         continue;
1375                 }
1376
1377                 ahd_complete_scb(ahd, scb);
1378                 scbid = next_scbid;
1379         }
1380         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
1381
1382         /*
1383          * Restore state.
1384          */
1385         ahd_set_scbptr(ahd, saved_scbptr);
1386         ahd_restore_modes(ahd, saved_modes);
1387         ahd->flags |= AHD_UPDATE_PEND_CMDS;
1388 }
1389
1390 /*
1391  * Determine if an SCB for a packetized transaction
1392  * is active in a FIFO.
1393  */
1394 static int
1395 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
1396 {
1397
1398         /*
1399          * The FIFO is only active for our transaction if
1400          * the SCBPTR matches the SCB's ID and the firmware
1401          * has installed a handler for the FIFO or we have
1402          * a pending SAVEPTRS or CFG4DATA interrupt.
1403          */
1404         if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
1405          || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
1406           && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
1407                 return (0);
1408
1409         return (1);
1410 }
1411
1412 /*
1413  * Run a data fifo to completion for a transaction we know
1414  * has completed across the SCSI bus (good status has been
1415  * received).  We are already set to the correct FIFO mode
1416  * on entry to this routine.
1417  *
1418  * This function attempts to operate exactly as the firmware
1419  * would when running this FIFO.  Care must be taken to update
1420  * this routine any time the firmware's FIFO algorithm is
1421  * changed.
1422  */
1423 static void
1424 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
1425 {
1426         u_int seqintsrc;
1427
1428         seqintsrc = ahd_inb(ahd, SEQINTSRC);
1429         if ((seqintsrc & CFG4DATA) != 0) {
1430                 uint32_t datacnt;
1431                 uint32_t sgptr;
1432
1433                 /*
1434                  * Clear full residual flag.
1435                  */
1436                 sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
1437                 ahd_outb(ahd, SCB_SGPTR, sgptr);
1438
1439                 /*
1440                  * Load datacnt and address.
1441                  */
1442                 datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
1443                 if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
1444                         sgptr |= LAST_SEG;
1445                         ahd_outb(ahd, SG_STATE, 0);
1446                 } else
1447                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1448                 ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
1449                 ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
1450                 ahd_outb(ahd, SG_CACHE_PRE, sgptr);
1451                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1452
1453                 /*
1454                  * Initialize Residual Fields.
1455                  */
1456                 ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
1457                 ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
1458
1459                 /*
1460                  * Mark the SCB as having a FIFO in use.
1461                  */
1462                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1463                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
1464
1465                 /*
1466                  * Install a "fake" handler for this FIFO.
1467                  */
1468                 ahd_outw(ahd, LONGJMP_ADDR, 0);
1469
1470                 /*
1471                  * Notify the hardware that we have satisfied
1472                  * this sequencer interrupt.
1473                  */
1474                 ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
1475         } else if ((seqintsrc & SAVEPTRS) != 0) {
1476                 uint32_t sgptr;
1477                 uint32_t resid;
1478
1479                 if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
1480                         /*
1481                          * Snapshot Save Pointers.  All that
1482                          * is necessary to clear the snapshot
1483                          * is a CLRCHN.
1484                          */
1485                         goto clrchn;
1486                 }
1487
1488                 /*
1489                  * Disable S/G fetch so the DMA engine
1490                  * is available to future users.
1491                  */
1492                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1493                         ahd_outb(ahd, CCSGCTL, 0);
1494                 ahd_outb(ahd, SG_STATE, 0);
1495
1496                 /*
1497                  * Flush the data FIFO.  Strickly only
1498                  * necessary for Rev A parts.
1499                  */
1500                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
1501
1502                 /*
1503                  * Calculate residual.
1504                  */
1505                 sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1506                 resid = ahd_inl(ahd, SHCNT);
1507                 resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
1508                 ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
1509                 if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
1510                         /*
1511                          * Must back up to the correct S/G element.
1512                          * Typically this just means resetting our
1513                          * low byte to the offset in the SG_CACHE,
1514                          * but if we wrapped, we have to correct
1515                          * the other bytes of the sgptr too.
1516                          */
1517                         if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
1518                          && (sgptr & 0x80) == 0)
1519                                 sgptr -= 0x100;
1520                         sgptr &= ~0xFF;
1521                         sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
1522                                & SG_ADDR_MASK;
1523                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1524                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
1525                 } else if ((resid & AHD_SG_LEN_MASK) == 0) {
1526                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
1527                                  sgptr | SG_LIST_NULL);
1528                 }
1529                 /*
1530                  * Save Pointers.
1531                  */
1532                 ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
1533                 ahd_outl(ahd, SCB_DATACNT, resid);
1534                 ahd_outl(ahd, SCB_SGPTR, sgptr);
1535                 ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
1536                 ahd_outb(ahd, SEQIMODE,
1537                          ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
1538                 /*
1539                  * If the data is to the SCSI bus, we are
1540                  * done, otherwise wait for FIFOEMP.
1541                  */
1542                 if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
1543                         goto clrchn;
1544         } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
1545                 uint32_t sgptr;
1546                 uint64_t data_addr;
1547                 uint32_t data_len;
1548                 u_int    dfcntrl;
1549
1550                 /*
1551                  * Disable S/G fetch so the DMA engine
1552                  * is available to future users.  We won't
1553                  * be using the DMA engine to load segments.
1554                  */
1555                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
1556                         ahd_outb(ahd, CCSGCTL, 0);
1557                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1558                 }
1559
1560                 /*
1561                  * Wait for the DMA engine to notice that the
1562                  * host transfer is enabled and that there is
1563                  * space in the S/G FIFO for new segments before
1564                  * loading more segments.
1565                  */
1566                 if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
1567                  && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
1568
1569                         /*
1570                          * Determine the offset of the next S/G
1571                          * element to load.
1572                          */
1573                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1574                         sgptr &= SG_PTR_MASK;
1575                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
1576                                 struct ahd_dma64_seg *sg;
1577
1578                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1579                                 data_addr = sg->addr;
1580                                 data_len = sg->len;
1581                                 sgptr += sizeof(*sg);
1582                         } else {
1583                                 struct  ahd_dma_seg *sg;
1584
1585                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1586                                 data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
1587                                 data_addr <<= 8;
1588                                 data_addr |= sg->addr;
1589                                 data_len = sg->len;
1590                                 sgptr += sizeof(*sg);
1591                         }
1592
1593                         /*
1594                          * Update residual information.
1595                          */
1596                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
1597                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1598
1599                         /*
1600                          * Load the S/G.
1601                          */
1602                         if (data_len & AHD_DMA_LAST_SEG) {
1603                                 sgptr |= LAST_SEG;
1604                                 ahd_outb(ahd, SG_STATE, 0);
1605                         }
1606                         ahd_outq(ahd, HADDR, data_addr);
1607                         ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
1608                         ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
1609
1610                         /*
1611                          * Advertise the segment to the hardware.
1612                          */
1613                         dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
1614                         if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
1615                                 /*
1616                                  * Use SCSIENWRDIS so that SCSIEN
1617                                  * is never modified by this
1618                                  * operation.
1619                                  */
1620                                 dfcntrl |= SCSIENWRDIS;
1621                         }
1622                         ahd_outb(ahd, DFCNTRL, dfcntrl);
1623                 }
1624         } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
1625
1626                 /*
1627                  * Transfer completed to the end of SG list
1628                  * and has flushed to the host.
1629                  */
1630                 ahd_outb(ahd, SCB_SGPTR,
1631                          ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
1632                 goto clrchn;
1633         } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
1634 clrchn:
1635                 /*
1636                  * Clear any handler for this FIFO, decrement
1637                  * the FIFO use count for the SCB, and release
1638                  * the FIFO.
1639                  */
1640                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1641                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1642                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
1643                 ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
1644         }
1645 }
1646
1647 /*
1648  * Look for entries in the QoutFIFO that have completed.
1649  * The valid_tag completion field indicates the validity
1650  * of the entry - the valid value toggles each time through
1651  * the queue. We use the sg_status field in the completion
1652  * entry to avoid referencing the hscb if the completion
1653  * occurred with no errors and no residual.  sg_status is
1654  * a copy of the first byte (little endian) of the sgptr
1655  * hscb field.
1656  */
1657 static void
1658 ahd_run_qoutfifo(struct ahd_softc *ahd)
1659 {
1660         struct ahd_completion *completion;
1661         struct scb *scb;
1662         u_int  scb_index;
1663
1664         if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
1665                 panic("ahd_run_qoutfifo recursion");
1666         ahd->flags |= AHD_RUNNING_QOUTFIFO;
1667         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
1668         for (;;) {
1669                 completion = &ahd->qoutfifo[ahd->qoutfifonext];
1670
1671                 if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
1672                         break;
1673
1674                 scb_index = ahd_le16toh(completion->tag);
1675                 scb = ahd_lookup_scb(ahd, scb_index);
1676                 if (scb == NULL) {
1677                         printk("%s: WARNING no command for scb %d "
1678                                "(cmdcmplt)\nQOUTPOS = %d\n",
1679                                ahd_name(ahd), scb_index,
1680                                ahd->qoutfifonext);
1681                         ahd_dump_card_state(ahd);
1682                 } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
1683                         ahd_handle_scb_status(ahd, scb);
1684                 } else {
1685                         ahd_done(ahd, scb);
1686                 }
1687
1688                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
1689                 if (ahd->qoutfifonext == 0)
1690                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
1691         }
1692         ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
1693 }
1694
1695 /************************* Interrupt Handling *********************************/
1696 static void
1697 ahd_handle_hwerrint(struct ahd_softc *ahd)
1698 {
1699         /*
1700          * Some catastrophic hardware error has occurred.
1701          * Print it for the user and disable the controller.
1702          */
1703         int i;
1704         int error;
1705
1706         error = ahd_inb(ahd, ERROR);
1707         for (i = 0; i < num_errors; i++) {
1708                 if ((error & ahd_hard_errors[i].errno) != 0)
1709                         printk("%s: hwerrint, %s\n",
1710                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
1711         }
1712
1713         ahd_dump_card_state(ahd);
1714         panic("BRKADRINT");
1715
1716         /* Tell everyone that this HBA is no longer available */
1717         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
1718                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
1719                        CAM_NO_HBA);
1720
1721         /* Tell the system that this controller has gone away. */
1722         ahd_free(ahd);
1723 }
1724
1725 #ifdef AHD_DEBUG
1726 static void
1727 ahd_dump_sglist(struct scb *scb)
1728 {
1729         int i;
1730
1731         if (scb->sg_count > 0) {
1732                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
1733                         struct ahd_dma64_seg *sg_list;
1734
1735                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
1736                         for (i = 0; i < scb->sg_count; i++) {
1737                                 uint64_t addr;
1738
1739                                 addr = ahd_le64toh(sg_list[i].addr);
1740                                 printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1741                                        i,
1742                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
1743                                        (uint32_t)(addr & 0xFFFFFFFF),
1744                                        sg_list[i].len & AHD_SG_LEN_MASK,
1745                                        (sg_list[i].len & AHD_DMA_LAST_SEG)
1746                                      ? " Last" : "");
1747                         }
1748                 } else {
1749                         struct ahd_dma_seg *sg_list;
1750
1751                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
1752                         for (i = 0; i < scb->sg_count; i++) {
1753                                 uint32_t len;
1754
1755                                 len = ahd_le32toh(sg_list[i].len);
1756                                 printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1757                                        i,
1758                                        (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
1759                                        ahd_le32toh(sg_list[i].addr),
1760                                        len & AHD_SG_LEN_MASK,
1761                                        len & AHD_DMA_LAST_SEG ? " Last" : "");
1762                         }
1763                 }
1764         }
1765 }
1766 #endif  /*  AHD_DEBUG  */
1767
1768 static void
1769 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
1770 {
1771         u_int seqintcode;
1772
1773         /*
1774          * Save the sequencer interrupt code and clear the SEQINT
1775          * bit. We will unpause the sequencer, if appropriate,
1776          * after servicing the request.
1777          */
1778         seqintcode = ahd_inb(ahd, SEQINTCODE);
1779         ahd_outb(ahd, CLRINT, CLRSEQINT);
1780         if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
1781                 /*
1782                  * Unpause the sequencer and let it clear
1783                  * SEQINT by writing NO_SEQINT to it.  This
1784                  * will cause the sequencer to be paused again,
1785                  * which is the expected state of this routine.
1786                  */
1787                 ahd_unpause(ahd);
1788                 while (!ahd_is_paused(ahd))
1789                         ;
1790                 ahd_outb(ahd, CLRINT, CLRSEQINT);
1791         }
1792         ahd_update_modes(ahd);
1793 #ifdef AHD_DEBUG
1794         if ((ahd_debug & AHD_SHOW_MISC) != 0)
1795                 printk("%s: Handle Seqint Called for code %d\n",
1796                        ahd_name(ahd), seqintcode);
1797 #endif
1798         switch (seqintcode) {
1799         case ENTERING_NONPACK:
1800         {
1801                 struct  scb *scb;
1802                 u_int   scbid;
1803
1804                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1805                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1806                 scbid = ahd_get_scbptr(ahd);
1807                 scb = ahd_lookup_scb(ahd, scbid);
1808                 if (scb == NULL) {
1809                         /*
1810                          * Somehow need to know if this
1811                          * is from a selection or reselection.
1812                          * From that, we can determine target
1813                          * ID so we at least have an I_T nexus.
1814                          */
1815                 } else {
1816                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1817                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
1818                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
1819                 }
1820                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
1821                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
1822                         /*
1823                          * Phase change after read stream with
1824                          * CRC error with P0 asserted on last
1825                          * packet.
1826                          */
1827 #ifdef AHD_DEBUG
1828                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1829                                 printk("%s: Assuming LQIPHASE_NLQ with "
1830                                        "P0 assertion\n", ahd_name(ahd));
1831 #endif
1832                 }
1833 #ifdef AHD_DEBUG
1834                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1835                         printk("%s: Entering NONPACK\n", ahd_name(ahd));
1836 #endif
1837                 break;
1838         }
1839         case INVALID_SEQINT:
1840                 printk("%s: Invalid Sequencer interrupt occurred, "
1841                        "resetting channel.\n",
1842                        ahd_name(ahd));
1843 #ifdef AHD_DEBUG
1844                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1845                         ahd_dump_card_state(ahd);
1846 #endif
1847                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1848                 break;
1849         case STATUS_OVERRUN:
1850         {
1851                 struct  scb *scb;
1852                 u_int   scbid;
1853
1854                 scbid = ahd_get_scbptr(ahd);
1855                 scb = ahd_lookup_scb(ahd, scbid);
1856                 if (scb != NULL)
1857                         ahd_print_path(ahd, scb);
1858                 else
1859                         printk("%s: ", ahd_name(ahd));
1860                 printk("SCB %d Packetized Status Overrun", scbid);
1861                 ahd_dump_card_state(ahd);
1862                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1863                 break;
1864         }
1865         case CFG4ISTAT_INTR:
1866         {
1867                 struct  scb *scb;
1868                 u_int   scbid;
1869
1870                 scbid = ahd_get_scbptr(ahd);
1871                 scb = ahd_lookup_scb(ahd, scbid);
1872                 if (scb == NULL) {
1873                         ahd_dump_card_state(ahd);
1874                         printk("CFG4ISTAT: Free SCB %d referenced", scbid);
1875                         panic("For safety");
1876                 }
1877                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
1878                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1879                 ahd_outb(ahd, HCNT + 2, 0);
1880                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1881                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1882                 break;
1883         }
1884         case ILLEGAL_PHASE:
1885         {
1886                 u_int bus_phase;
1887
1888                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1889                 printk("%s: ILLEGAL_PHASE 0x%x\n",
1890                        ahd_name(ahd), bus_phase);
1891
1892                 switch (bus_phase) {
1893                 case P_DATAOUT:
1894                 case P_DATAIN:
1895                 case P_DATAOUT_DT:
1896                 case P_DATAIN_DT:
1897                 case P_MESGOUT:
1898                 case P_STATUS:
1899                 case P_MESGIN:
1900                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1901                         printk("%s: Issued Bus Reset.\n", ahd_name(ahd));
1902                         break;
1903                 case P_COMMAND:
1904                 {
1905                         struct  ahd_devinfo devinfo;
1906                         struct  scb *scb;
1907                         u_int   scbid;
1908
1909                         /*
1910                          * If a target takes us into the command phase
1911                          * assume that it has been externally reset and
1912                          * has thus lost our previous packetized negotiation
1913                          * agreement.  Since we have not sent an identify
1914                          * message and may not have fully qualified the
1915                          * connection, we change our command to TUR, assert
1916                          * ATN and ABORT the task when we go to message in
1917                          * phase.  The OSM will see the REQUEUE_REQUEST
1918                          * status and retry the command.
1919                          */
1920                         scbid = ahd_get_scbptr(ahd);
1921                         scb = ahd_lookup_scb(ahd, scbid);
1922                         if (scb == NULL) {
1923                                 printk("Invalid phase with no valid SCB.  "
1924                                        "Resetting bus.\n");
1925                                 ahd_reset_channel(ahd, 'A',
1926                                                   /*Initiate Reset*/TRUE);
1927                                 break;
1928                         }
1929                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1930                                             SCB_GET_TARGET(ahd, scb),
1931                                             SCB_GET_LUN(scb),
1932                                             SCB_GET_CHANNEL(ahd, scb),
1933                                             ROLE_INITIATOR);
1934                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1935                                       AHD_TRANS_ACTIVE, /*paused*/TRUE);
1936                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1937                                          /*offset*/0, /*ppr_options*/0,
1938                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
1939                         /* Hand-craft TUR command */
1940                         ahd_outb(ahd, SCB_CDB_STORE, 0);
1941                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1942                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1943                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1944                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1945                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1946                         ahd_outb(ahd, SCB_CDB_LEN, 6);
1947                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1948                         scb->hscb->control |= MK_MESSAGE;
1949                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1950                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1951                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1952                         /*
1953                          * The lun is 0, regardless of the SCB's lun
1954                          * as we have not sent an identify message.
1955                          */
1956                         ahd_outb(ahd, SAVED_LUN, 0);
1957                         ahd_outb(ahd, SEQ_FLAGS, 0);
1958                         ahd_assert_atn(ahd);
1959                         scb->flags &= ~SCB_PACKETIZED;
1960                         scb->flags |= SCB_ABORT|SCB_EXTERNAL_RESET;
1961                         ahd_freeze_devq(ahd, scb);
1962                         ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1963                         ahd_freeze_scb(scb);
1964
1965                         /* Notify XPT */
1966                         ahd_send_async(ahd, devinfo.channel, devinfo.target,
1967                                        CAM_LUN_WILDCARD, AC_SENT_BDR);
1968
1969                         /*
1970                          * Allow the sequencer to continue with
1971                          * non-pack processing.
1972                          */
1973                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1974                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1975                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1976                                 ahd_outb(ahd, CLRLQOINT1, 0);
1977                         }
1978 #ifdef AHD_DEBUG
1979                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1980                                 ahd_print_path(ahd, scb);
1981                                 printk("Unexpected command phase from "
1982                                        "packetized target\n");
1983                         }
1984 #endif
1985                         break;
1986                 }
1987                 }
1988                 break;
1989         }
1990         case CFG4OVERRUN:
1991         {
1992                 struct  scb *scb;
1993                 u_int   scb_index;
1994
1995 #ifdef AHD_DEBUG
1996                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1997                         printk("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1998                                ahd_inb(ahd, MODE_PTR));
1999                 }
2000 #endif
2001                 scb_index = ahd_get_scbptr(ahd);
2002                 scb = ahd_lookup_scb(ahd, scb_index);
2003                 if (scb == NULL) {
2004                         /*
2005                          * Attempt to transfer to an SCB that is
2006                          * not outstanding.
2007                          */
2008                         ahd_assert_atn(ahd);
2009                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2010                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
2011                         ahd->msgout_len = 1;
2012                         ahd->msgout_index = 0;
2013                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2014                         /*
2015                          * Clear status received flag to prevent any
2016                          * attempt to complete this bogus SCB.
2017                          */
2018                         ahd_outb(ahd, SCB_CONTROL,
2019                                  ahd_inb_scbram(ahd, SCB_CONTROL)
2020                                  & ~STATUS_RCVD);
2021                 }
2022                 break;
2023         }
2024         case DUMP_CARD_STATE:
2025         {
2026                 ahd_dump_card_state(ahd);
2027                 break;
2028         }
2029         case PDATA_REINIT:
2030         {
2031 #ifdef AHD_DEBUG
2032                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2033                         printk("%s: PDATA_REINIT - DFCNTRL = 0x%x "
2034                                "SG_CACHE_SHADOW = 0x%x\n",
2035                                ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
2036                                ahd_inb(ahd, SG_CACHE_SHADOW));
2037                 }
2038 #endif
2039                 ahd_reinitialize_dataptrs(ahd);
2040                 break;
2041         }
2042         case HOST_MSG_LOOP:
2043         {
2044                 struct ahd_devinfo devinfo;
2045
2046                 /*
2047                  * The sequencer has encountered a message phase
2048                  * that requires host assistance for completion.
2049                  * While handling the message phase(s), we will be
2050                  * notified by the sequencer after each byte is
2051                  * transferred so we can track bus phase changes.
2052                  *
2053                  * If this is the first time we've seen a HOST_MSG_LOOP
2054                  * interrupt, initialize the state of the host message
2055                  * loop.
2056                  */
2057                 ahd_fetch_devinfo(ahd, &devinfo);
2058                 if (ahd->msg_type == MSG_TYPE_NONE) {
2059                         struct scb *scb;
2060                         u_int scb_index;
2061                         u_int bus_phase;
2062
2063                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2064                         if (bus_phase != P_MESGIN
2065                          && bus_phase != P_MESGOUT) {
2066                                 printk("ahd_intr: HOST_MSG_LOOP bad "
2067                                        "phase 0x%x\n", bus_phase);
2068                                 /*
2069                                  * Probably transitioned to bus free before
2070                                  * we got here.  Just punt the message.
2071                                  */
2072                                 ahd_dump_card_state(ahd);
2073                                 ahd_clear_intstat(ahd);
2074                                 ahd_restart(ahd);
2075                                 return;
2076                         }
2077
2078                         scb_index = ahd_get_scbptr(ahd);
2079                         scb = ahd_lookup_scb(ahd, scb_index);
2080                         if (devinfo.role == ROLE_INITIATOR) {
2081                                 if (bus_phase == P_MESGOUT)
2082                                         ahd_setup_initiator_msgout(ahd,
2083                                                                    &devinfo,
2084                                                                    scb);
2085                                 else {
2086                                         ahd->msg_type =
2087                                             MSG_TYPE_INITIATOR_MSGIN;
2088                                         ahd->msgin_index = 0;
2089                                 }
2090                         }
2091 #ifdef AHD_TARGET_MODE
2092                         else {
2093                                 if (bus_phase == P_MESGOUT) {
2094                                         ahd->msg_type =
2095                                             MSG_TYPE_TARGET_MSGOUT;
2096                                         ahd->msgin_index = 0;
2097                                 }
2098                                 else
2099                                         ahd_setup_target_msgin(ahd,
2100                                                                &devinfo,
2101                                                                scb);
2102                         }
2103 #endif
2104                 }
2105
2106                 ahd_handle_message_phase(ahd);
2107                 break;
2108         }
2109         case NO_MATCH:
2110         {
2111                 /* Ensure we don't leave the selection hardware on */
2112                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
2113                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2114
2115                 printk("%s:%c:%d: no active SCB for reconnecting "
2116                        "target - issuing BUS DEVICE RESET\n",
2117                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
2118                 printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
2119                        "REG0 == 0x%x ACCUM = 0x%x\n",
2120                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
2121                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
2122                 printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
2123                        "SINDEX == 0x%x\n",
2124                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
2125                        ahd_find_busy_tcl(ahd,
2126                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
2127                                                    ahd_inb(ahd, SAVED_LUN))),
2128                        ahd_inw(ahd, SINDEX));
2129                 printk("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
2130                        "SCB_CONTROL == 0x%x\n",
2131                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
2132                        ahd_inb_scbram(ahd, SCB_LUN),
2133                        ahd_inb_scbram(ahd, SCB_CONTROL));
2134                 printk("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
2135                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
2136                 printk("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
2137                 printk("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
2138                 ahd_dump_card_state(ahd);
2139                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
2140                 ahd->msgout_len = 1;
2141                 ahd->msgout_index = 0;
2142                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2143                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
2144                 ahd_assert_atn(ahd);
2145                 break;
2146         }
2147         case PROTO_VIOLATION:
2148         {
2149                 ahd_handle_proto_violation(ahd);
2150                 break;
2151         }
2152         case IGN_WIDE_RES:
2153         {
2154                 struct ahd_devinfo devinfo;
2155
2156                 ahd_fetch_devinfo(ahd, &devinfo);
2157                 ahd_handle_ign_wide_residue(ahd, &devinfo);
2158                 break;
2159         }
2160         case BAD_PHASE:
2161         {
2162                 u_int lastphase;
2163
2164                 lastphase = ahd_inb(ahd, LASTPHASE);
2165                 printk("%s:%c:%d: unknown scsi bus phase %x, "
2166                        "lastphase = 0x%x.  Attempting to continue\n",
2167                        ahd_name(ahd), 'A',
2168                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2169                        lastphase, ahd_inb(ahd, SCSISIGI));
2170                 break;
2171         }
2172         case MISSED_BUSFREE:
2173         {
2174                 u_int lastphase;
2175
2176                 lastphase = ahd_inb(ahd, LASTPHASE);
2177                 printk("%s:%c:%d: Missed busfree. "
2178                        "Lastphase = 0x%x, Curphase = 0x%x\n",
2179                        ahd_name(ahd), 'A',
2180                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2181                        lastphase, ahd_inb(ahd, SCSISIGI));
2182                 ahd_restart(ahd);
2183                 return;
2184         }
2185         case DATA_OVERRUN:
2186         {
2187                 /*
2188                  * When the sequencer detects an overrun, it
2189                  * places the controller in "BITBUCKET" mode
2190                  * and allows the target to complete its transfer.
2191                  * Unfortunately, none of the counters get updated
2192                  * when the controller is in this mode, so we have
2193                  * no way of knowing how large the overrun was.
2194                  */
2195                 struct  scb *scb;
2196                 u_int   scbindex;
2197 #ifdef AHD_DEBUG
2198                 u_int   lastphase;
2199 #endif
2200
2201                 scbindex = ahd_get_scbptr(ahd);
2202                 scb = ahd_lookup_scb(ahd, scbindex);
2203 #ifdef AHD_DEBUG
2204                 lastphase = ahd_inb(ahd, LASTPHASE);
2205                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2206                         ahd_print_path(ahd, scb);
2207                         printk("data overrun detected %s.  Tag == 0x%x.\n",
2208                                ahd_lookup_phase_entry(lastphase)->phasemsg,
2209                                SCB_GET_TAG(scb));
2210                         ahd_print_path(ahd, scb);
2211                         printk("%s seen Data Phase.  Length = %ld.  "
2212                                "NumSGs = %d.\n",
2213                                ahd_inb(ahd, SEQ_FLAGS) & DPHASE
2214                                ? "Have" : "Haven't",
2215                                ahd_get_transfer_length(scb), scb->sg_count);
2216                         ahd_dump_sglist(scb);
2217                 }
2218 #endif
2219
2220                 /*
2221                  * Set this and it will take effect when the
2222                  * target does a command complete.
2223                  */
2224                 ahd_freeze_devq(ahd, scb);
2225                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
2226                 ahd_freeze_scb(scb);
2227                 break;
2228         }
2229         case MKMSG_FAILED:
2230         {
2231                 struct ahd_devinfo devinfo;
2232                 struct scb *scb;
2233                 u_int scbid;
2234
2235                 ahd_fetch_devinfo(ahd, &devinfo);
2236                 printk("%s:%c:%d:%d: Attempt to issue message failed\n",
2237                        ahd_name(ahd), devinfo.channel, devinfo.target,
2238                        devinfo.lun);
2239                 scbid = ahd_get_scbptr(ahd);
2240                 scb = ahd_lookup_scb(ahd, scbid);
2241                 if (scb != NULL
2242                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
2243                         /*
2244                          * Ensure that we didn't put a second instance of this
2245                          * SCB into the QINFIFO.
2246                          */
2247                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2248                                            SCB_GET_CHANNEL(ahd, scb),
2249                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2250                                            ROLE_INITIATOR, /*status*/0,
2251                                            SEARCH_REMOVE);
2252                 ahd_outb(ahd, SCB_CONTROL,
2253                          ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
2254                 break;
2255         }
2256         case TASKMGMT_FUNC_COMPLETE:
2257         {
2258                 u_int   scbid;
2259                 struct  scb *scb;
2260
2261                 scbid = ahd_get_scbptr(ahd);
2262                 scb = ahd_lookup_scb(ahd, scbid);
2263                 if (scb != NULL) {
2264                         u_int      lun;
2265                         u_int      tag;
2266                         cam_status error;
2267
2268                         ahd_print_path(ahd, scb);
2269                         printk("Task Management Func 0x%x Complete\n",
2270                                scb->hscb->task_management);
2271                         lun = CAM_LUN_WILDCARD;
2272                         tag = SCB_LIST_NULL;
2273
2274                         switch (scb->hscb->task_management) {
2275                         case SIU_TASKMGMT_ABORT_TASK:
2276                                 tag = SCB_GET_TAG(scb);
2277                                 fallthrough;
2278                         case SIU_TASKMGMT_ABORT_TASK_SET:
2279                         case SIU_TASKMGMT_CLEAR_TASK_SET:
2280                                 lun = scb->hscb->lun;
2281                                 error = CAM_REQ_ABORTED;
2282                                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2283                                                'A', lun, tag, ROLE_INITIATOR,
2284                                                error);
2285                                 break;
2286                         case SIU_TASKMGMT_LUN_RESET:
2287                                 lun = scb->hscb->lun;
2288                                 fallthrough;
2289                         case SIU_TASKMGMT_TARGET_RESET:
2290                         {
2291                                 struct ahd_devinfo devinfo;
2292
2293                                 ahd_scb_devinfo(ahd, &devinfo, scb);
2294                                 error = CAM_BDR_SENT;
2295                                 ahd_handle_devreset(ahd, &devinfo, lun,
2296                                                     CAM_BDR_SENT,
2297                                                     lun != CAM_LUN_WILDCARD
2298                                                     ? "Lun Reset"
2299                                                     : "Target Reset",
2300                                                     /*verbose_level*/0);
2301                                 break;
2302                         }
2303                         default:
2304                                 panic("Unexpected TaskMgmt Func\n");
2305                                 break;
2306                         }
2307                 }
2308                 break;
2309         }
2310         case TASKMGMT_CMD_CMPLT_OKAY:
2311         {
2312                 u_int   scbid;
2313                 struct  scb *scb;
2314
2315                 /*
2316                  * An ABORT TASK TMF failed to be delivered before
2317                  * the targeted command completed normally.
2318                  */
2319                 scbid = ahd_get_scbptr(ahd);
2320                 scb = ahd_lookup_scb(ahd, scbid);
2321                 if (scb != NULL) {
2322                         /*
2323                          * Remove the second instance of this SCB from
2324                          * the QINFIFO if it is still there.
2325                          */
2326                         ahd_print_path(ahd, scb);
2327                         printk("SCB completes before TMF\n");
2328                         /*
2329                          * Handle losing the race.  Wait until any
2330                          * current selection completes.  We will then
2331                          * set the TMF back to zero in this SCB so that
2332                          * the sequencer doesn't bother to issue another
2333                          * sequencer interrupt for its completion.
2334                          */
2335                         while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
2336                             && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
2337                             && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
2338                                 ;
2339                         ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
2340                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2341                                            SCB_GET_CHANNEL(ahd, scb),
2342                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2343                                            ROLE_INITIATOR, /*status*/0,
2344                                            SEARCH_REMOVE);
2345                 }
2346                 break;
2347         }
2348         case TRACEPOINT0:
2349         case TRACEPOINT1:
2350         case TRACEPOINT2:
2351         case TRACEPOINT3:
2352                 printk("%s: Tracepoint %d\n", ahd_name(ahd),
2353                        seqintcode - TRACEPOINT0);
2354                 break;
2355         case NO_SEQINT:
2356                 break;
2357         case SAW_HWERR:
2358                 ahd_handle_hwerrint(ahd);
2359                 break;
2360         default:
2361                 printk("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
2362                        seqintcode);
2363                 break;
2364         }
2365         /*
2366          *  The sequencer is paused immediately on
2367          *  a SEQINT, so we should restart it when
2368          *  we're done.
2369          */
2370         ahd_unpause(ahd);
2371 }
2372
2373 static void
2374 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
2375 {
2376         struct scb      *scb;
2377         u_int            status0;
2378         u_int            status3;
2379         u_int            status;
2380         u_int            lqistat1;
2381         u_int            lqostat0;
2382         u_int            scbid;
2383         u_int            busfreetime;
2384
2385         ahd_update_modes(ahd);
2386         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2387
2388         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
2389         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
2390         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
2391         lqistat1 = ahd_inb(ahd, LQISTAT1);
2392         lqostat0 = ahd_inb(ahd, LQOSTAT0);
2393         busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2394
2395         /*
2396          * Ignore external resets after a bus reset.
2397          */
2398         if (((status & SCSIRSTI) != 0) && (ahd->flags & AHD_BUS_RESET_ACTIVE)) {
2399                 ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
2400                 return;
2401         }
2402
2403         /*
2404          * Clear bus reset flag
2405          */
2406         ahd->flags &= ~AHD_BUS_RESET_ACTIVE;
2407
2408         if ((status0 & (SELDI|SELDO)) != 0) {
2409                 u_int simode0;
2410
2411                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2412                 simode0 = ahd_inb(ahd, SIMODE0);
2413                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
2414                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2415         }
2416         scbid = ahd_get_scbptr(ahd);
2417         scb = ahd_lookup_scb(ahd, scbid);
2418         if (scb != NULL
2419          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2420                 scb = NULL;
2421
2422         if ((status0 & IOERR) != 0) {
2423                 u_int now_lvd;
2424
2425                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
2426                 printk("%s: Transceiver State Has Changed to %s mode\n",
2427                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
2428                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
2429                 /*
2430                  * A change in I/O mode is equivalent to a bus reset.
2431                  */
2432                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2433                 ahd_pause(ahd);
2434                 ahd_setup_iocell_workaround(ahd);
2435                 ahd_unpause(ahd);
2436         } else if ((status0 & OVERRUN) != 0) {
2437
2438                 printk("%s: SCSI offset overrun detected.  Resetting bus.\n",
2439                        ahd_name(ahd));
2440                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2441         } else if ((status & SCSIRSTI) != 0) {
2442
2443                 printk("%s: Someone reset channel A\n", ahd_name(ahd));
2444                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
2445         } else if ((status & SCSIPERR) != 0) {
2446
2447                 /* Make sure the sequencer is in a safe location. */
2448                 ahd_clear_critical_section(ahd);
2449
2450                 ahd_handle_transmission_error(ahd);
2451         } else if (lqostat0 != 0) {
2452
2453                 printk("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
2454                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
2455                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2456                         ahd_outb(ahd, CLRLQOINT1, 0);
2457         } else if ((status & SELTO) != 0) {
2458                 /* Stop the selection */
2459                 ahd_outb(ahd, SCSISEQ0, 0);
2460
2461                 /* Make sure the sequencer is in a safe location. */
2462                 ahd_clear_critical_section(ahd);
2463
2464                 /* No more pending messages */
2465                 ahd_clear_msg_state(ahd);
2466
2467                 /* Clear interrupt state */
2468                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
2469
2470                 /*
2471                  * Although the driver does not care about the
2472                  * 'Selection in Progress' status bit, the busy
2473                  * LED does.  SELINGO is only cleared by a successful
2474                  * selection, so we must manually clear it to insure
2475                  * the LED turns off just incase no future successful
2476                  * selections occur (e.g. no devices on the bus).
2477                  */
2478                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
2479
2480                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
2481                 scb = ahd_lookup_scb(ahd, scbid);
2482                 if (scb == NULL) {
2483                         printk("%s: ahd_intr - referenced scb not "
2484                                "valid during SELTO scb(0x%x)\n",
2485                                ahd_name(ahd), scbid);
2486                         ahd_dump_card_state(ahd);
2487                 } else {
2488                         struct ahd_devinfo devinfo;
2489 #ifdef AHD_DEBUG
2490                         if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
2491                                 ahd_print_path(ahd, scb);
2492                                 printk("Saw Selection Timeout for SCB 0x%x\n",
2493                                        scbid);
2494                         }
2495 #endif
2496                         ahd_scb_devinfo(ahd, &devinfo, scb);
2497                         ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
2498                         ahd_freeze_devq(ahd, scb);
2499
2500                         /*
2501                          * Cancel any pending transactions on the device
2502                          * now that it seems to be missing.  This will
2503                          * also revert us to async/narrow transfers until
2504                          * we can renegotiate with the device.
2505                          */
2506                         ahd_handle_devreset(ahd, &devinfo,
2507                                             CAM_LUN_WILDCARD,
2508                                             CAM_SEL_TIMEOUT,
2509                                             "Selection Timeout",
2510                                             /*verbose_level*/1);
2511                 }
2512                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2513                 ahd_iocell_first_selection(ahd);
2514                 ahd_unpause(ahd);
2515         } else if ((status0 & (SELDI|SELDO)) != 0) {
2516
2517                 ahd_iocell_first_selection(ahd);
2518                 ahd_unpause(ahd);
2519         } else if (status3 != 0) {
2520                 printk("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
2521                        ahd_name(ahd), status3);
2522                 ahd_outb(ahd, CLRSINT3, status3);
2523         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
2524
2525                 /* Make sure the sequencer is in a safe location. */
2526                 ahd_clear_critical_section(ahd);
2527
2528                 ahd_handle_lqiphase_error(ahd, lqistat1);
2529         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2530                 /*
2531                  * This status can be delayed during some
2532                  * streaming operations.  The SCSIPHASE
2533                  * handler has already dealt with this case
2534                  * so just clear the error.
2535                  */
2536                 ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
2537         } else if ((status & BUSFREE) != 0
2538                 || (lqistat1 & LQOBUSFREE) != 0) {
2539                 u_int lqostat1;
2540                 int   restart;
2541                 int   clear_fifo;
2542                 int   packetized;
2543                 u_int mode;
2544
2545                 /*
2546                  * Clear our selection hardware as soon as possible.
2547                  * We may have an entry in the waiting Q for this target,
2548                  * that is affected by this busfree and we don't want to
2549                  * go about selecting the target while we handle the event.
2550                  */
2551                 ahd_outb(ahd, SCSISEQ0, 0);
2552
2553                 /* Make sure the sequencer is in a safe location. */
2554                 ahd_clear_critical_section(ahd);
2555
2556                 /*
2557                  * Determine what we were up to at the time of
2558                  * the busfree.
2559                  */
2560                 mode = AHD_MODE_SCSI;
2561                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2562                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
2563                 switch (busfreetime) {
2564                 case BUSFREE_DFF0:
2565                 case BUSFREE_DFF1:
2566                 {
2567                         mode = busfreetime == BUSFREE_DFF0
2568                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
2569                         ahd_set_modes(ahd, mode, mode);
2570                         scbid = ahd_get_scbptr(ahd);
2571                         scb = ahd_lookup_scb(ahd, scbid);
2572                         if (scb == NULL) {
2573                                 printk("%s: Invalid SCB %d in DFF%d "
2574                                        "during unexpected busfree\n",
2575                                        ahd_name(ahd), scbid, mode);
2576                                 packetized = 0;
2577                         } else
2578                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
2579                         clear_fifo = 1;
2580                         break;
2581                 }
2582                 case BUSFREE_LQO:
2583                         clear_fifo = 0;
2584                         packetized = 1;
2585                         break;
2586                 default:
2587                         clear_fifo = 0;
2588                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
2589                         if (!packetized
2590                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
2591                          && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
2592                          && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
2593                           || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
2594                                 /*
2595                                  * Assume packetized if we are not
2596                                  * on the bus in a non-packetized
2597                                  * capacity and any pending selection
2598                                  * was a packetized selection.
2599                                  */
2600                                 packetized = 1;
2601                         break;
2602                 }
2603
2604 #ifdef AHD_DEBUG
2605                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
2606                         printk("Saw Busfree.  Busfreetime = 0x%x.\n",
2607                                busfreetime);
2608 #endif
2609                 /*
2610                  * Busfrees that occur in non-packetized phases are
2611                  * handled by the nonpkt_busfree handler.
2612                  */
2613                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
2614                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
2615                 } else {
2616                         packetized = 0;
2617                         restart = ahd_handle_nonpkt_busfree(ahd);
2618                 }
2619                 /*
2620                  * Clear the busfree interrupt status.  The setting of
2621                  * the interrupt is a pulse, so in a perfect world, we
2622                  * would not need to muck with the ENBUSFREE logic.  This
2623                  * would ensure that if the bus moves on to another
2624                  * connection, busfree protection is still in force.  If
2625                  * BUSFREEREV is broken, however, we must manually clear
2626                  * the ENBUSFREE if the busfree occurred during a non-pack
2627                  * connection so that we don't get false positives during
2628                  * future, packetized, connections.
2629                  */
2630                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2631                 if (packetized == 0
2632                  && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
2633                         ahd_outb(ahd, SIMODE1,
2634                                  ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
2635
2636                 if (clear_fifo)
2637                         ahd_clear_fifo(ahd, mode);
2638
2639                 ahd_clear_msg_state(ahd);
2640                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2641                 if (restart) {
2642                         ahd_restart(ahd);
2643                 } else {
2644                         ahd_unpause(ahd);
2645                 }
2646         } else {
2647                 printk("%s: Missing case in ahd_handle_scsiint. status = %x\n",
2648                        ahd_name(ahd), status);
2649                 ahd_dump_card_state(ahd);
2650                 ahd_clear_intstat(ahd);
2651                 ahd_unpause(ahd);
2652         }
2653 }
2654
2655 static void
2656 ahd_handle_transmission_error(struct ahd_softc *ahd)
2657 {
2658         struct  scb *scb;
2659         u_int   scbid;
2660         u_int   lqistat1;
2661         u_int   msg_out;
2662         u_int   curphase;
2663         u_int   lastphase;
2664         u_int   perrdiag;
2665         u_int   cur_col;
2666         int     silent;
2667
2668         scb = NULL;
2669         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2670         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
2671         ahd_inb(ahd, LQISTAT2);
2672         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
2673          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
2674                 u_int lqistate;
2675
2676                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2677                 lqistate = ahd_inb(ahd, LQISTATE);
2678                 if ((lqistate >= 0x1E && lqistate <= 0x24)
2679                  || (lqistate == 0x29)) {
2680 #ifdef AHD_DEBUG
2681                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2682                                 printk("%s: NLQCRC found via LQISTATE\n",
2683                                        ahd_name(ahd));
2684                         }
2685 #endif
2686                         lqistat1 |= LQICRCI_NLQ;
2687                 }
2688                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2689         }
2690
2691         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2692         lastphase = ahd_inb(ahd, LASTPHASE);
2693         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2694         perrdiag = ahd_inb(ahd, PERRDIAG);
2695         msg_out = MSG_INITIATOR_DET_ERR;
2696         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
2697
2698         /*
2699          * Try to find the SCB associated with this error.
2700          */
2701         silent = FALSE;
2702         if (lqistat1 == 0
2703          || (lqistat1 & LQICRCI_NLQ) != 0) {
2704                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
2705                         ahd_set_active_fifo(ahd);
2706                 scbid = ahd_get_scbptr(ahd);
2707                 scb = ahd_lookup_scb(ahd, scbid);
2708                 if (scb != NULL && SCB_IS_SILENT(scb))
2709                         silent = TRUE;
2710         }
2711
2712         cur_col = 0;
2713         if (silent == FALSE) {
2714                 printk("%s: Transmission error detected\n", ahd_name(ahd));
2715                 ahd_lqistat1_print(lqistat1, &cur_col, 50);
2716                 ahd_lastphase_print(lastphase, &cur_col, 50);
2717                 ahd_scsisigi_print(curphase, &cur_col, 50);
2718                 ahd_perrdiag_print(perrdiag, &cur_col, 50);
2719                 printk("\n");
2720                 ahd_dump_card_state(ahd);
2721         }
2722
2723         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
2724                 if (silent == FALSE) {
2725                         printk("%s: Gross protocol error during incoming "
2726                                "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
2727                                ahd_name(ahd), lqistat1);
2728                 }
2729                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2730                 return;
2731         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
2732                 /*
2733                  * A CRC error has been detected on an incoming LQ.
2734                  * The bus is currently hung on the last ACK.
2735                  * Hit LQIRETRY to release the last ack, and
2736                  * wait for the sequencer to determine that ATNO
2737                  * is asserted while in message out to take us
2738                  * to our host message loop.  No NONPACKREQ or
2739                  * LQIPHASE type errors will occur in this
2740                  * scenario.  After this first LQIRETRY, the LQI
2741                  * manager will be in ISELO where it will
2742                  * happily sit until another packet phase begins.
2743                  * Unexpected bus free detection is enabled
2744                  * through any phases that occur after we release
2745                  * this last ack until the LQI manager sees a
2746                  * packet phase.  This implies we may have to
2747                  * ignore a perfectly valid "unexected busfree"
2748                  * after our "initiator detected error" message is
2749                  * sent.  A busfree is the expected response after
2750                  * we tell the target that it's L_Q was corrupted.
2751                  * (SPI4R09 10.7.3.3.3)
2752                  */
2753                 ahd_outb(ahd, LQCTL2, LQIRETRY);
2754                 printk("LQIRetry for LQICRCI_LQ to release ACK\n");
2755         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2756                 /*
2757                  * We detected a CRC error in a NON-LQ packet.
2758                  * The hardware has varying behavior in this situation
2759                  * depending on whether this packet was part of a
2760                  * stream or not.
2761                  *
2762                  * PKT by PKT mode:
2763                  * The hardware has already acked the complete packet.
2764                  * If the target honors our outstanding ATN condition,
2765                  * we should be (or soon will be) in MSGOUT phase.
2766                  * This will trigger the LQIPHASE_LQ status bit as the
2767                  * hardware was expecting another LQ.  Unexpected
2768                  * busfree detection is enabled.  Once LQIPHASE_LQ is
2769                  * true (first entry into host message loop is much
2770                  * the same), we must clear LQIPHASE_LQ and hit
2771                  * LQIRETRY so the hardware is ready to handle
2772                  * a future LQ.  NONPACKREQ will not be asserted again
2773                  * once we hit LQIRETRY until another packet is
2774                  * processed.  The target may either go busfree
2775                  * or start another packet in response to our message.
2776                  *
2777                  * Read Streaming P0 asserted:
2778                  * If we raise ATN and the target completes the entire
2779                  * stream (P0 asserted during the last packet), the
2780                  * hardware will ack all data and return to the ISTART
2781                  * state.  When the target reponds to our ATN condition,
2782                  * LQIPHASE_LQ will be asserted.  We should respond to
2783                  * this with an LQIRETRY to prepare for any future
2784                  * packets.  NONPACKREQ will not be asserted again
2785                  * once we hit LQIRETRY until another packet is
2786                  * processed.  The target may either go busfree or
2787                  * start another packet in response to our message.
2788                  * Busfree detection is enabled.
2789                  *
2790                  * Read Streaming P0 not asserted:
2791                  * If we raise ATN and the target transitions to
2792                  * MSGOUT in or after a packet where P0 is not
2793                  * asserted, the hardware will assert LQIPHASE_NLQ.
2794                  * We should respond to the LQIPHASE_NLQ with an
2795                  * LQIRETRY.  Should the target stay in a non-pkt
2796                  * phase after we send our message, the hardware
2797                  * will assert LQIPHASE_LQ.  Recovery is then just as
2798                  * listed above for the read streaming with P0 asserted.
2799                  * Busfree detection is enabled.
2800                  */
2801                 if (silent == FALSE)
2802                         printk("LQICRC_NLQ\n");
2803                 if (scb == NULL) {
2804                         printk("%s: No SCB valid for LQICRC_NLQ.  "
2805                                "Resetting bus\n", ahd_name(ahd));
2806                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2807                         return;
2808                 }
2809         } else if ((lqistat1 & LQIBADLQI) != 0) {
2810                 printk("Need to handle BADLQI!\n");
2811                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2812                 return;
2813         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
2814                 if ((curphase & ~P_DATAIN_DT) != 0) {
2815                         /* Ack the byte.  So we can continue. */
2816                         if (silent == FALSE)
2817                                 printk("Acking %s to clear perror\n",
2818                                     ahd_lookup_phase_entry(curphase)->phasemsg);
2819                         ahd_inb(ahd, SCSIDAT);
2820                 }
2821
2822                 if (curphase == P_MESGIN)
2823                         msg_out = MSG_PARITY_ERROR;
2824         }
2825
2826         /*
2827          * We've set the hardware to assert ATN if we
2828          * get a parity error on "in" phases, so all we
2829          * need to do is stuff the message buffer with
2830          * the appropriate message.  "In" phases have set
2831          * mesg_out to something other than MSG_NOP.
2832          */
2833         ahd->send_msg_perror = msg_out;
2834         if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
2835                 scb->flags |= SCB_TRANSMISSION_ERROR;
2836         ahd_outb(ahd, MSG_OUT, HOST_MSG);
2837         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2838         ahd_unpause(ahd);
2839 }
2840
2841 static void
2842 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2843 {
2844         /*
2845          * Clear the sources of the interrupts.
2846          */
2847         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2848         ahd_outb(ahd, CLRLQIINT1, lqistat1);
2849
2850         /*
2851          * If the "illegal" phase changes were in response
2852          * to our ATN to flag a CRC error, AND we ended up
2853          * on packet boundaries, clear the error, restart the
2854          * LQI manager as appropriate, and go on our merry
2855          * way toward sending the message.  Otherwise, reset
2856          * the bus to clear the error.
2857          */
2858         ahd_set_active_fifo(ahd);
2859         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2860          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2861                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
2862                         printk("LQIRETRY for LQIPHASE_LQ\n");
2863                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2864                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2865                         printk("LQIRETRY for LQIPHASE_NLQ\n");
2866                         ahd_outb(ahd, LQCTL2, LQIRETRY);
2867                 } else
2868                         panic("ahd_handle_lqiphase_error: No phase errors\n");
2869                 ahd_dump_card_state(ahd);
2870                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2871                 ahd_unpause(ahd);
2872         } else {
2873                 printk("Resetting Channel for LQI Phase error\n");
2874                 ahd_dump_card_state(ahd);
2875                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2876         }
2877 }
2878
2879 /*
2880  * Packetized unexpected or expected busfree.
2881  * Entered in mode based on busfreetime.
2882  */
2883 static int
2884 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2885 {
2886         u_int lqostat1;
2887
2888         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2889                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2890         lqostat1 = ahd_inb(ahd, LQOSTAT1);
2891         if ((lqostat1 & LQOBUSFREE) != 0) {
2892                 struct scb *scb;
2893                 u_int scbid;
2894                 u_int saved_scbptr;
2895                 u_int waiting_h;
2896                 u_int waiting_t;
2897                 u_int next;
2898
2899                 /*
2900                  * The LQO manager detected an unexpected busfree
2901                  * either:
2902                  *
2903                  * 1) During an outgoing LQ.
2904                  * 2) After an outgoing LQ but before the first
2905                  *    REQ of the command packet.
2906                  * 3) During an outgoing command packet.
2907                  *
2908                  * In all cases, CURRSCB is pointing to the
2909                  * SCB that encountered the failure.  Clean
2910                  * up the queue, clear SELDO and LQOBUSFREE,
2911                  * and allow the sequencer to restart the select
2912                  * out at its lesure.
2913                  */
2914                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2915                 scbid = ahd_inw(ahd, CURRSCB);
2916                 scb = ahd_lookup_scb(ahd, scbid);
2917                 if (scb == NULL)
2918                        panic("SCB not valid during LQOBUSFREE");
2919                 /*
2920                  * Clear the status.
2921                  */
2922                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2923                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2924                         ahd_outb(ahd, CLRLQOINT1, 0);
2925                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2926                 ahd_flush_device_writes(ahd);
2927                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
2928
2929                 /*
2930                  * Return the LQO manager to its idle loop.  It will
2931                  * not do this automatically if the busfree occurs
2932                  * after the first REQ of either the LQ or command
2933                  * packet or between the LQ and command packet.
2934                  */
2935                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2936
2937                 /*
2938                  * Update the waiting for selection queue so
2939                  * we restart on the correct SCB.
2940                  */
2941                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2942                 saved_scbptr = ahd_get_scbptr(ahd);
2943                 if (waiting_h != scbid) {
2944
2945                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2946                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2947                         if (waiting_t == waiting_h) {
2948                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2949                                 next = SCB_LIST_NULL;
2950                         } else {
2951                                 ahd_set_scbptr(ahd, waiting_h);
2952                                 next = ahd_inw_scbram(ahd, SCB_NEXT2);
2953                         }
2954                         ahd_set_scbptr(ahd, scbid);
2955                         ahd_outw(ahd, SCB_NEXT2, next);
2956                 }
2957                 ahd_set_scbptr(ahd, saved_scbptr);
2958                 if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2959                         if (SCB_IS_SILENT(scb) == FALSE) {
2960                                 ahd_print_path(ahd, scb);
2961                                 printk("Probable outgoing LQ CRC error.  "
2962                                        "Retrying command\n");
2963                         }
2964                         scb->crc_retry_count++;
2965                 } else {
2966                         ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
2967                         ahd_freeze_scb(scb);
2968                         ahd_freeze_devq(ahd, scb);
2969                 }
2970                 /* Return unpausing the sequencer. */
2971                 return (0);
2972         } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2973                 /*
2974                  * Ignore what are really parity errors that
2975                  * occur on the last REQ of a free running
2976                  * clock prior to going busfree.  Some drives
2977                  * do not properly active negate just before
2978                  * going busfree resulting in a parity glitch.
2979                  */
2980                 ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2981 #ifdef AHD_DEBUG
2982                 if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2983                         printk("%s: Parity on last REQ detected "
2984                                "during busfree phase.\n",
2985                                ahd_name(ahd));
2986 #endif
2987                 /* Return unpausing the sequencer. */
2988                 return (0);
2989         }
2990         if (ahd->src_mode != AHD_MODE_SCSI) {
2991                 u_int   scbid;
2992                 struct  scb *scb;
2993
2994                 scbid = ahd_get_scbptr(ahd);
2995                 scb = ahd_lookup_scb(ahd, scbid);
2996                 ahd_print_path(ahd, scb);
2997                 printk("Unexpected PKT busfree condition\n");
2998                 ahd_dump_card_state(ahd);
2999                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
3000                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
3001                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
3002
3003                 /* Return restarting the sequencer. */
3004                 return (1);
3005         }
3006         printk("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
3007         ahd_dump_card_state(ahd);
3008         /* Restart the sequencer. */
3009         return (1);
3010 }
3011
3012 /*
3013  * Non-packetized unexpected or expected busfree.
3014  */
3015 static int
3016 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
3017 {
3018         struct  ahd_devinfo devinfo;
3019         struct  scb *scb;
3020         u_int   lastphase;
3021         u_int   saved_scsiid;
3022         u_int   saved_lun;
3023         u_int   target;
3024         u_int   initiator_role_id;
3025         u_int   scbid;
3026         u_int   ppr_busfree;
3027         int     printerror;
3028
3029         /*
3030          * Look at what phase we were last in.  If its message out,
3031          * chances are pretty good that the busfree was in response
3032          * to one of our abort requests.
3033          */
3034         lastphase = ahd_inb(ahd, LASTPHASE);
3035         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3036         saved_lun = ahd_inb(ahd, SAVED_LUN);
3037         target = SCSIID_TARGET(ahd, saved_scsiid);
3038         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
3039         ahd_compile_devinfo(&devinfo, initiator_role_id,
3040                             target, saved_lun, 'A', ROLE_INITIATOR);
3041         printerror = 1;
3042
3043         scbid = ahd_get_scbptr(ahd);
3044         scb = ahd_lookup_scb(ahd, scbid);
3045         if (scb != NULL
3046          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
3047                 scb = NULL;
3048
3049         ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
3050         if (lastphase == P_MESGOUT) {
3051                 u_int tag;
3052
3053                 tag = SCB_LIST_NULL;
3054                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
3055                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
3056                         int found;
3057                         int sent_msg;
3058
3059                         if (scb == NULL) {
3060                                 ahd_print_devinfo(ahd, &devinfo);
3061                                 printk("Abort for unidentified "
3062                                        "connection completed.\n");
3063                                 /* restart the sequencer. */
3064                                 return (1);
3065                         }
3066                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
3067                         ahd_print_path(ahd, scb);
3068                         printk("SCB %d - Abort%s Completed.\n",
3069                                SCB_GET_TAG(scb),
3070                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
3071
3072                         if (sent_msg == MSG_ABORT_TAG)
3073                                 tag = SCB_GET_TAG(scb);
3074
3075                         if ((scb->flags & SCB_EXTERNAL_RESET) != 0) {
3076                                 /*
3077                                  * This abort is in response to an
3078                                  * unexpected switch to command phase
3079                                  * for a packetized connection.  Since
3080                                  * the identify message was never sent,
3081                                  * "saved lun" is 0.  We really want to
3082                                  * abort only the SCB that encountered
3083                                  * this error, which could have a different
3084                                  * lun.  The SCB will be retried so the OS
3085                                  * will see the UA after renegotiating to
3086                                  * packetized.
3087                                  */
3088                                 tag = SCB_GET_TAG(scb);
3089                                 saved_lun = scb->hscb->lun;
3090                         }
3091                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
3092                                                tag, ROLE_INITIATOR,
3093                                                CAM_REQ_ABORTED);
3094                         printk("found == 0x%x\n", found);
3095                         printerror = 0;
3096                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
3097                                         MSG_BUS_DEV_RESET, TRUE)) {
3098                         ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
3099                                             CAM_BDR_SENT, "Bus Device Reset",
3100                                             /*verbose_level*/0);
3101                         printerror = 0;
3102                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
3103                         && ppr_busfree == 0) {
3104                         struct ahd_initiator_tinfo *tinfo;
3105                         struct ahd_tmode_tstate *tstate;
3106
3107                         /*
3108                          * PPR Rejected.
3109                          *
3110                          * If the previous negotiation was packetized,
3111                          * this could be because the device has been
3112                          * reset without our knowledge.  Force our
3113                          * current negotiation to async and retry the
3114                          * negotiation.  Otherwise retry the command
3115                          * with non-ppr negotiation.
3116                          */
3117 #ifdef AHD_DEBUG
3118                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3119                                 printk("PPR negotiation rejected busfree.\n");
3120 #endif
3121                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3122                                                     devinfo.our_scsiid,
3123                                                     devinfo.target, &tstate);
3124                         if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
3125                                 ahd_set_width(ahd, &devinfo,
3126                                               MSG_EXT_WDTR_BUS_8_BIT,
3127                                               AHD_TRANS_CUR,
3128                                               /*paused*/TRUE);
3129                                 ahd_set_syncrate(ahd, &devinfo,
3130                                                 /*period*/0, /*offset*/0,
3131                                                 /*ppr_options*/0,
3132                                                 AHD_TRANS_CUR,
3133                                                 /*paused*/TRUE);
3134                                 /*
3135                                  * The expect PPR busfree handler below
3136                                  * will effect the retry and necessary
3137                                  * abort.
3138                                  */
3139                         } else {
3140                                 tinfo->curr.transport_version = 2;
3141                                 tinfo->goal.transport_version = 2;
3142                                 tinfo->goal.ppr_options = 0;
3143                                 if (scb != NULL) {
3144                                         /*
3145                                          * Remove any SCBs in the waiting
3146                                          * for selection queue that may
3147                                          * also be for this target so that
3148                                          * command ordering is preserved.
3149                                          */
3150                                         ahd_freeze_devq(ahd, scb);
3151                                         ahd_qinfifo_requeue_tail(ahd, scb);
3152                                 }
3153                                 printerror = 0;
3154                         }
3155                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
3156                         && ppr_busfree == 0) {
3157                         /*
3158                          * Negotiation Rejected.  Go-narrow and
3159                          * retry command.
3160                          */
3161 #ifdef AHD_DEBUG
3162                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3163                                 printk("WDTR negotiation rejected busfree.\n");
3164 #endif
3165                         ahd_set_width(ahd, &devinfo,
3166                                       MSG_EXT_WDTR_BUS_8_BIT,
3167                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
3168                                       /*paused*/TRUE);
3169                         if (scb != NULL) {
3170                                 /*
3171                                  * Remove any SCBs in the waiting for
3172                                  * selection queue that may also be for
3173                                  * this target so that command ordering
3174                                  * is preserved.
3175                                  */
3176                                 ahd_freeze_devq(ahd, scb);
3177                                 ahd_qinfifo_requeue_tail(ahd, scb);
3178                         }
3179                         printerror = 0;
3180                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
3181                         && ppr_busfree == 0) {
3182                         /*
3183                          * Negotiation Rejected.  Go-async and
3184                          * retry command.
3185                          */
3186 #ifdef AHD_DEBUG
3187                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3188                                 printk("SDTR negotiation rejected busfree.\n");
3189 #endif
3190                         ahd_set_syncrate(ahd, &devinfo,
3191                                         /*period*/0, /*offset*/0,
3192                                         /*ppr_options*/0,
3193                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
3194                                         /*paused*/TRUE);
3195                         if (scb != NULL) {
3196                                 /*
3197                                  * Remove any SCBs in the waiting for
3198                                  * selection queue that may also be for
3199                                  * this target so that command ordering
3200                                  * is preserved.
3201                                  */
3202                                 ahd_freeze_devq(ahd, scb);
3203                                 ahd_qinfifo_requeue_tail(ahd, scb);
3204                         }
3205                         printerror = 0;
3206                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
3207                         && ahd_sent_msg(ahd, AHDMSG_1B,
3208                                          MSG_INITIATOR_DET_ERR, TRUE)) {
3209
3210 #ifdef AHD_DEBUG
3211                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3212                                 printk("Expected IDE Busfree\n");
3213 #endif
3214                         printerror = 0;
3215                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
3216                         && ahd_sent_msg(ahd, AHDMSG_1B,
3217                                         MSG_MESSAGE_REJECT, TRUE)) {
3218
3219 #ifdef AHD_DEBUG
3220                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3221                                 printk("Expected QAS Reject Busfree\n");
3222 #endif
3223                         printerror = 0;
3224                 }
3225         }
3226
3227         /*
3228          * The busfree required flag is honored at the end of
3229          * the message phases.  We check it last in case we
3230          * had to send some other message that caused a busfree.
3231          */
3232         if (scb != NULL && printerror != 0
3233          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
3234          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
3235
3236                 ahd_freeze_devq(ahd, scb);
3237                 ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
3238                 ahd_freeze_scb(scb);
3239                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
3240                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
3241                                        SCB_GET_CHANNEL(ahd, scb),
3242                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
3243                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
3244                 } else {
3245 #ifdef AHD_DEBUG
3246                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3247                                 printk("PPR Negotiation Busfree.\n");
3248 #endif
3249                         ahd_done(ahd, scb);
3250                 }
3251                 printerror = 0;
3252         }
3253         if (printerror != 0) {
3254                 int aborted;
3255
3256                 aborted = 0;
3257                 if (scb != NULL) {
3258                         u_int tag;
3259
3260                         if ((scb->hscb->control & TAG_ENB) != 0)
3261                                 tag = SCB_GET_TAG(scb);
3262                         else
3263                                 tag = SCB_LIST_NULL;
3264                         ahd_print_path(ahd, scb);
3265                         aborted = ahd_abort_scbs(ahd, target, 'A',
3266                                        SCB_GET_LUN(scb), tag,
3267                                        ROLE_INITIATOR,
3268                                        CAM_UNEXP_BUSFREE);
3269                 } else {
3270                         /*
3271                          * We had not fully identified this connection,
3272                          * so we cannot abort anything.
3273                          */
3274                         printk("%s: ", ahd_name(ahd));
3275                 }
3276                 printk("Unexpected busfree %s, %d SCBs aborted, "
3277                        "PRGMCNT == 0x%x\n",
3278                        ahd_lookup_phase_entry(lastphase)->phasemsg,
3279                        aborted,
3280                        ahd_inw(ahd, PRGMCNT));
3281                 ahd_dump_card_state(ahd);
3282                 if (lastphase != P_BUSFREE)
3283                         ahd_force_renegotiation(ahd, &devinfo);
3284         }
3285         /* Always restart the sequencer. */
3286         return (1);
3287 }
3288
3289 static void
3290 ahd_handle_proto_violation(struct ahd_softc *ahd)
3291 {
3292         struct  ahd_devinfo devinfo;
3293         struct  scb *scb;
3294         u_int   scbid;
3295         u_int   seq_flags;
3296         u_int   curphase;
3297         u_int   lastphase;
3298         int     found;
3299
3300         ahd_fetch_devinfo(ahd, &devinfo);
3301         scbid = ahd_get_scbptr(ahd);
3302         scb = ahd_lookup_scb(ahd, scbid);
3303         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
3304         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
3305         lastphase = ahd_inb(ahd, LASTPHASE);
3306         if ((seq_flags & NOT_IDENTIFIED) != 0) {
3307
3308                 /*
3309                  * The reconnecting target either did not send an
3310                  * identify message, or did, but we didn't find an SCB
3311                  * to match.
3312                  */
3313                 ahd_print_devinfo(ahd, &devinfo);
3314                 printk("Target did not send an IDENTIFY message. "
3315                        "LASTPHASE = 0x%x.\n", lastphase);
3316                 scb = NULL;
3317         } else if (scb == NULL) {
3318                 /*
3319                  * We don't seem to have an SCB active for this
3320                  * transaction.  Print an error and reset the bus.
3321                  */
3322                 ahd_print_devinfo(ahd, &devinfo);
3323                 printk("No SCB found during protocol violation\n");
3324                 goto proto_violation_reset;
3325         } else {
3326                 ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3327                 if ((seq_flags & NO_CDB_SENT) != 0) {
3328                         ahd_print_path(ahd, scb);
3329                         printk("No or incomplete CDB sent to device.\n");
3330                 } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
3331                           & STATUS_RCVD) == 0) {
3332                         /*
3333                          * The target never bothered to provide status to
3334                          * us prior to completing the command.  Since we don't
3335                          * know the disposition of this command, we must attempt
3336                          * to abort it.  Assert ATN and prepare to send an abort
3337                          * message.
3338                          */
3339                         ahd_print_path(ahd, scb);
3340                         printk("Completed command without status.\n");
3341                 } else {
3342                         ahd_print_path(ahd, scb);
3343                         printk("Unknown protocol violation.\n");
3344                         ahd_dump_card_state(ahd);
3345                 }
3346         }
3347         if ((lastphase & ~P_DATAIN_DT) == 0
3348          || lastphase == P_COMMAND) {
3349 proto_violation_reset:
3350                 /*
3351                  * Target either went directly to data
3352                  * phase or didn't respond to our ATN.
3353                  * The only safe thing to do is to blow
3354                  * it away with a bus reset.
3355                  */
3356                 found = ahd_reset_channel(ahd, 'A', TRUE);
3357                 printk("%s: Issued Channel %c Bus Reset. "
3358                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
3359         } else {
3360                 /*
3361                  * Leave the selection hardware off in case
3362                  * this abort attempt will affect yet to
3363                  * be sent commands.
3364                  */
3365                 ahd_outb(ahd, SCSISEQ0,
3366                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3367                 ahd_assert_atn(ahd);
3368                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
3369                 if (scb == NULL) {
3370                         ahd_print_devinfo(ahd, &devinfo);
3371                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
3372                         ahd->msgout_len = 1;
3373                         ahd->msgout_index = 0;
3374                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3375                 } else {
3376                         ahd_print_path(ahd, scb);
3377                         scb->flags |= SCB_ABORT;
3378                 }
3379                 printk("Protocol violation %s.  Attempting to abort.\n",
3380                        ahd_lookup_phase_entry(curphase)->phasemsg);
3381         }
3382 }
3383
3384 /*
3385  * Force renegotiation to occur the next time we initiate
3386  * a command to the current device.
3387  */
3388 static void
3389 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3390 {
3391         struct  ahd_initiator_tinfo *targ_info;
3392         struct  ahd_tmode_tstate *tstate;
3393
3394 #ifdef AHD_DEBUG
3395         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3396                 ahd_print_devinfo(ahd, devinfo);
3397                 printk("Forcing renegotiation\n");
3398         }
3399 #endif
3400         targ_info = ahd_fetch_transinfo(ahd,
3401                                         devinfo->channel,
3402                                         devinfo->our_scsiid,
3403                                         devinfo->target,
3404                                         &tstate);
3405         ahd_update_neg_request(ahd, devinfo, tstate,
3406                                targ_info, AHD_NEG_IF_NON_ASYNC);
3407 }
3408
3409 #define AHD_MAX_STEPS 2000
3410 static void
3411 ahd_clear_critical_section(struct ahd_softc *ahd)
3412 {
3413         ahd_mode_state  saved_modes;
3414         int             stepping;
3415         int             steps;
3416         int             first_instr;
3417         u_int           simode0;
3418         u_int           simode1;
3419         u_int           simode3;
3420         u_int           lqimode0;
3421         u_int           lqimode1;
3422         u_int           lqomode0;
3423         u_int           lqomode1;
3424
3425         if (ahd->num_critical_sections == 0)
3426                 return;
3427
3428         stepping = FALSE;
3429         steps = 0;
3430         first_instr = 0;
3431         simode0 = 0;
3432         simode1 = 0;
3433         simode3 = 0;
3434         lqimode0 = 0;
3435         lqimode1 = 0;
3436         lqomode0 = 0;
3437         lqomode1 = 0;
3438         saved_modes = ahd_save_modes(ahd);
3439         for (;;) {
3440                 struct  cs *cs;
3441                 u_int   seqaddr;
3442                 u_int   i;
3443
3444                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3445                 seqaddr = ahd_inw(ahd, CURADDR);
3446
3447                 cs = ahd->critical_sections;
3448                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
3449
3450                         if (cs->begin < seqaddr && cs->end >= seqaddr)
3451                                 break;
3452                 }
3453
3454                 if (i == ahd->num_critical_sections)
3455                         break;
3456
3457                 if (steps > AHD_MAX_STEPS) {
3458                         printk("%s: Infinite loop in critical section\n"
3459                                "%s: First Instruction 0x%x now 0x%x\n",
3460                                ahd_name(ahd), ahd_name(ahd), first_instr,
3461                                seqaddr);
3462                         ahd_dump_card_state(ahd);
3463                         panic("critical section loop");
3464                 }
3465
3466                 steps++;
3467 #ifdef AHD_DEBUG
3468                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
3469                         printk("%s: Single stepping at 0x%x\n", ahd_name(ahd),
3470                                seqaddr);
3471 #endif
3472                 if (stepping == FALSE) {
3473
3474                         first_instr = seqaddr;
3475                         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3476                         simode0 = ahd_inb(ahd, SIMODE0);
3477                         simode3 = ahd_inb(ahd, SIMODE3);
3478                         lqimode0 = ahd_inb(ahd, LQIMODE0);
3479                         lqimode1 = ahd_inb(ahd, LQIMODE1);
3480                         lqomode0 = ahd_inb(ahd, LQOMODE0);
3481                         lqomode1 = ahd_inb(ahd, LQOMODE1);
3482                         ahd_outb(ahd, SIMODE0, 0);
3483                         ahd_outb(ahd, SIMODE3, 0);
3484                         ahd_outb(ahd, LQIMODE0, 0);
3485                         ahd_outb(ahd, LQIMODE1, 0);
3486                         ahd_outb(ahd, LQOMODE0, 0);
3487                         ahd_outb(ahd, LQOMODE1, 0);
3488                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3489                         simode1 = ahd_inb(ahd, SIMODE1);
3490                         /*
3491                          * We don't clear ENBUSFREE.  Unfortunately
3492                          * we cannot re-enable busfree detection within
3493                          * the current connection, so we must leave it
3494                          * on while single stepping.
3495                          */
3496                         ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
3497                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
3498                         stepping = TRUE;
3499                 }
3500                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
3501                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
3502                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
3503                 ahd_outb(ahd, HCNTRL, ahd->unpause);
3504                 while (!ahd_is_paused(ahd))
3505                         ahd_delay(200);
3506                 ahd_update_modes(ahd);
3507         }
3508         if (stepping) {
3509                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3510                 ahd_outb(ahd, SIMODE0, simode0);
3511                 ahd_outb(ahd, SIMODE3, simode3);
3512                 ahd_outb(ahd, LQIMODE0, lqimode0);
3513                 ahd_outb(ahd, LQIMODE1, lqimode1);
3514                 ahd_outb(ahd, LQOMODE0, lqomode0);
3515                 ahd_outb(ahd, LQOMODE1, lqomode1);
3516                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3517                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
3518                 ahd_outb(ahd, SIMODE1, simode1);
3519                 /*
3520                  * SCSIINT seems to glitch occasionally when
3521                  * the interrupt masks are restored.  Clear SCSIINT
3522                  * one more time so that only persistent errors
3523                  * are seen as a real interrupt.
3524                  */
3525                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
3526         }
3527         ahd_restore_modes(ahd, saved_modes);
3528 }
3529
3530 /*
3531  * Clear any pending interrupt status.
3532  */
3533 static void
3534 ahd_clear_intstat(struct ahd_softc *ahd)
3535 {
3536         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
3537                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
3538         /* Clear any interrupt conditions this may have caused */
3539         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
3540                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
3541         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
3542                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
3543                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
3544         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
3545                                  |CLRLQOATNPKT|CLRLQOTCRC);
3546         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
3547                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
3548         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
3549                 ahd_outb(ahd, CLRLQOINT0, 0);
3550                 ahd_outb(ahd, CLRLQOINT1, 0);
3551         }
3552         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
3553         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
3554                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
3555         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
3556                                 |CLRIOERR|CLROVERRUN);
3557         ahd_outb(ahd, CLRINT, CLRSCSIINT);
3558 }
3559
3560 /**************************** Debugging Routines ******************************/
3561 #ifdef AHD_DEBUG
3562 uint32_t ahd_debug = AHD_DEBUG_OPTS;
3563 #endif
3564
3565 #if 0
3566 void
3567 ahd_print_scb(struct scb *scb)
3568 {
3569         struct hardware_scb *hscb;
3570         int i;
3571
3572         hscb = scb->hscb;
3573         printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
3574                (void *)scb,
3575                hscb->control,
3576                hscb->scsiid,
3577                hscb->lun,
3578                hscb->cdb_len);
3579         printk("Shared Data: ");
3580         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
3581                 printk("%#02x", hscb->shared_data.idata.cdb[i]);
3582         printk("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
3583                (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
3584                (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
3585                ahd_le32toh(hscb->datacnt),
3586                ahd_le32toh(hscb->sgptr),
3587                SCB_GET_TAG(scb));
3588         ahd_dump_sglist(scb);
3589 }
3590 #endif  /*  0  */
3591
3592 /************************* Transfer Negotiation *******************************/
3593 /*
3594  * Allocate per target mode instance (ID we respond to as a target)
3595  * transfer negotiation data structures.
3596  */
3597 static struct ahd_tmode_tstate *
3598 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
3599 {
3600         struct ahd_tmode_tstate *master_tstate;
3601         struct ahd_tmode_tstate *tstate;
3602         int i;
3603
3604         master_tstate = ahd->enabled_targets[ahd->our_id];
3605         if (ahd->enabled_targets[scsi_id] != NULL
3606          && ahd->enabled_targets[scsi_id] != master_tstate)
3607                 panic("%s: ahd_alloc_tstate - Target already allocated",
3608                       ahd_name(ahd));
3609         tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
3610         if (tstate == NULL)
3611                 return (NULL);
3612
3613         /*
3614          * If we have allocated a master tstate, copy user settings from
3615          * the master tstate (taken from SRAM or the EEPROM) for this
3616          * channel, but reset our current and goal settings to async/narrow
3617          * until an initiator talks to us.
3618          */
3619         if (master_tstate != NULL) {
3620                 memcpy(tstate, master_tstate, sizeof(*tstate));
3621                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
3622                 for (i = 0; i < 16; i++) {
3623                         memset(&tstate->transinfo[i].curr, 0,
3624                               sizeof(tstate->transinfo[i].curr));
3625                         memset(&tstate->transinfo[i].goal, 0,
3626                               sizeof(tstate->transinfo[i].goal));
3627                 }
3628         } else
3629                 memset(tstate, 0, sizeof(*tstate));
3630         ahd->enabled_targets[scsi_id] = tstate;
3631         return (tstate);
3632 }
3633
3634 #ifdef AHD_TARGET_MODE
3635 /*
3636  * Free per target mode instance (ID we respond to as a target)
3637  * transfer negotiation data structures.
3638  */
3639 static void
3640 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
3641 {
3642         struct ahd_tmode_tstate *tstate;
3643
3644         /*
3645          * Don't clean up our "master" tstate.
3646          * It has our default user settings.
3647          */
3648         if (scsi_id == ahd->our_id
3649          && force == FALSE)
3650                 return;
3651
3652         tstate = ahd->enabled_targets[scsi_id];
3653         kfree(tstate);
3654         ahd->enabled_targets[scsi_id] = NULL;
3655 }
3656 #endif
3657
3658 /*
3659  * Called when we have an active connection to a target on the bus,
3660  * this function finds the nearest period to the input period limited
3661  * by the capabilities of the bus connectivity of and sync settings for
3662  * the target.
3663  */
3664 static void
3665 ahd_devlimited_syncrate(struct ahd_softc *ahd,
3666                         struct ahd_initiator_tinfo *tinfo,
3667                         u_int *period, u_int *ppr_options, role_t role)
3668 {
3669         struct  ahd_transinfo *transinfo;
3670         u_int   maxsync;
3671
3672         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
3673          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
3674                 maxsync = AHD_SYNCRATE_PACED;
3675         } else {
3676                 maxsync = AHD_SYNCRATE_ULTRA;
3677                 /* Can't do DT related options on an SE bus */
3678                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3679         }
3680         /*
3681          * Never allow a value higher than our current goal
3682          * period otherwise we may allow a target initiated
3683          * negotiation to go above the limit as set by the
3684          * user.  In the case of an initiator initiated
3685          * sync negotiation, we limit based on the user
3686          * setting.  This allows the system to still accept
3687          * incoming negotiations even if target initiated
3688          * negotiation is not performed.
3689          */
3690         if (role == ROLE_TARGET)
3691                 transinfo = &tinfo->user;
3692         else
3693                 transinfo = &tinfo->goal;
3694         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
3695         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
3696                 maxsync = max(maxsync, (u_int)AHD_SYNCRATE_ULTRA2);
3697                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3698         }
3699         if (transinfo->period == 0) {
3700                 *period = 0;
3701                 *ppr_options = 0;
3702         } else {
3703                 *period = max(*period, (u_int)transinfo->period);
3704                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
3705         }
3706 }
3707
3708 /*
3709  * Look up the valid period to SCSIRATE conversion in our table.
3710  * Return the period and offset that should be sent to the target
3711  * if this was the beginning of an SDTR.
3712  */
3713 void
3714 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
3715                   u_int *ppr_options, u_int maxsync)
3716 {
3717         if (*period < maxsync)
3718                 *period = maxsync;
3719
3720         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
3721          && *period > AHD_SYNCRATE_MIN_DT)
3722                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3723
3724         if (*period > AHD_SYNCRATE_MIN)
3725                 *period = 0;
3726
3727         /* Honor PPR option conformance rules. */
3728         if (*period > AHD_SYNCRATE_PACED)
3729                 *ppr_options &= ~MSG_EXT_PPR_RTI;
3730
3731         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3732                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
3733
3734         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
3735                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
3736
3737         /* Skip all PACED only entries if IU is not available */
3738         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
3739          && *period < AHD_SYNCRATE_DT)
3740                 *period = AHD_SYNCRATE_DT;
3741
3742         /* Skip all DT only entries if DT is not available */
3743         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3744          && *period < AHD_SYNCRATE_ULTRA2)
3745                 *period = AHD_SYNCRATE_ULTRA2;
3746 }
3747
3748 /*
3749  * Truncate the given synchronous offset to a value the
3750  * current adapter type and syncrate are capable of.
3751  */
3752 static void
3753 ahd_validate_offset(struct ahd_softc *ahd,
3754                     struct ahd_initiator_tinfo *tinfo,
3755                     u_int period, u_int *offset, int wide,
3756                     role_t role)
3757 {
3758         u_int maxoffset;
3759
3760         /* Limit offset to what we can do */
3761         if (period == 0)
3762                 maxoffset = 0;
3763         else if (period <= AHD_SYNCRATE_PACED) {
3764                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
3765                         maxoffset = MAX_OFFSET_PACED_BUG;
3766                 else
3767                         maxoffset = MAX_OFFSET_PACED;
3768         } else
3769                 maxoffset = MAX_OFFSET_NON_PACED;
3770         *offset = min(*offset, maxoffset);
3771         if (tinfo != NULL) {
3772                 if (role == ROLE_TARGET)
3773                         *offset = min(*offset, (u_int)tinfo->user.offset);
3774                 else
3775                         *offset = min(*offset, (u_int)tinfo->goal.offset);
3776         }
3777 }
3778
3779 /*
3780  * Truncate the given transfer width parameter to a value the
3781  * current adapter type is capable of.
3782  */
3783 static void
3784 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3785                    u_int *bus_width, role_t role)
3786 {
3787         switch (*bus_width) {
3788         default:
3789                 if (ahd->features & AHD_WIDE) {
3790                         /* Respond Wide */
3791                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3792                         break;
3793                 }
3794                 fallthrough;
3795         case MSG_EXT_WDTR_BUS_8_BIT:
3796                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3797                 break;
3798         }
3799         if (tinfo != NULL) {
3800                 if (role == ROLE_TARGET)
3801                         *bus_width = min((u_int)tinfo->user.width, *bus_width);
3802                 else
3803                         *bus_width = min((u_int)tinfo->goal.width, *bus_width);
3804         }
3805 }
3806
3807 /*
3808  * Update the bitmask of targets for which the controller should
3809  * negotiate with at the next convenient opportunity.  This currently
3810  * means the next time we send the initial identify messages for
3811  * a new transaction.
3812  */
3813 int
3814 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3815                        struct ahd_tmode_tstate *tstate,
3816                        struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3817 {
3818         u_int auto_negotiate_orig;
3819
3820         auto_negotiate_orig = tstate->auto_negotiate;
3821         if (neg_type == AHD_NEG_ALWAYS) {
3822                 /*
3823                  * Force our "current" settings to be
3824                  * unknown so that unless a bus reset
3825                  * occurs the need to renegotiate is
3826                  * recorded persistently.
3827                  */
3828                 if ((ahd->features & AHD_WIDE) != 0)
3829                         tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3830                 tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3831                 tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3832         }
3833         if (tinfo->curr.period != tinfo->goal.period
3834          || tinfo->curr.width != tinfo->goal.width
3835          || tinfo->curr.offset != tinfo->goal.offset
3836          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3837          || (neg_type == AHD_NEG_IF_NON_ASYNC
3838           && (tinfo->goal.offset != 0
3839            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3840            || tinfo->goal.ppr_options != 0)))
3841                 tstate->auto_negotiate |= devinfo->target_mask;
3842         else
3843                 tstate->auto_negotiate &= ~devinfo->target_mask;
3844
3845         return (auto_negotiate_orig != tstate->auto_negotiate);
3846 }
3847
3848 /*
3849  * Update the user/goal/curr tables of synchronous negotiation
3850  * parameters as well as, in the case of a current or active update,
3851  * any data structures on the host controller.  In the case of an
3852  * active update, the specified target is currently talking to us on
3853  * the bus, so the transfer parameter update must take effect
3854  * immediately.
3855  */
3856 void
3857 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3858                  u_int period, u_int offset, u_int ppr_options,
3859                  u_int type, int paused)
3860 {
3861         struct  ahd_initiator_tinfo *tinfo;
3862         struct  ahd_tmode_tstate *tstate;
3863         u_int   old_period;
3864         u_int   old_offset;
3865         u_int   old_ppr;
3866         int     active;
3867         int     update_needed;
3868
3869         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3870         update_needed = 0;
3871
3872         if (period == 0 || offset == 0) {
3873                 period = 0;
3874                 offset = 0;
3875         }
3876
3877         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3878                                     devinfo->target, &tstate);
3879
3880         if ((type & AHD_TRANS_USER) != 0) {
3881                 tinfo->user.period = period;
3882                 tinfo->user.offset = offset;
3883                 tinfo->user.ppr_options = ppr_options;
3884         }
3885
3886         if ((type & AHD_TRANS_GOAL) != 0) {
3887                 tinfo->goal.period = period;
3888                 tinfo->goal.offset = offset;
3889                 tinfo->goal.ppr_options = ppr_options;
3890         }
3891
3892         old_period = tinfo->curr.period;
3893         old_offset = tinfo->curr.offset;
3894         old_ppr    = tinfo->curr.ppr_options;
3895
3896         if ((type & AHD_TRANS_CUR) != 0
3897          && (old_period != period
3898           || old_offset != offset
3899           || old_ppr != ppr_options)) {
3900
3901                 update_needed++;
3902
3903                 tinfo->curr.period = period;
3904                 tinfo->curr.offset = offset;
3905                 tinfo->curr.ppr_options = ppr_options;
3906
3907                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3908                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
3909                 if (bootverbose) {
3910                         if (offset != 0) {
3911                                 int options;
3912
3913                                 printk("%s: target %d synchronous with "
3914                                        "period = 0x%x, offset = 0x%x",
3915                                        ahd_name(ahd), devinfo->target,
3916                                        period, offset);
3917                                 options = 0;
3918                                 if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3919                                         printk("(RDSTRM");
3920                                         options++;
3921                                 }
3922                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3923                                         printk("%s", options ? "|DT" : "(DT");
3924                                         options++;
3925                                 }
3926                                 if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3927                                         printk("%s", options ? "|IU" : "(IU");
3928                                         options++;
3929                                 }
3930                                 if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3931                                         printk("%s", options ? "|RTI" : "(RTI");
3932                                         options++;
3933                                 }
3934                                 if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3935                                         printk("%s", options ? "|QAS" : "(QAS");
3936                                         options++;
3937                                 }
3938                                 if (options != 0)
3939                                         printk(")\n");
3940                                 else
3941                                         printk("\n");
3942                         } else {
3943                                 printk("%s: target %d using "
3944                                        "asynchronous transfers%s\n",
3945                                        ahd_name(ahd), devinfo->target,
3946                                        (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3947                                      ?  "(QAS)" : "");
3948                         }
3949                 }
3950         }
3951         /*
3952          * Always refresh the neg-table to handle the case of the
3953          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3954          * We will always renegotiate in that case if this is a
3955          * packetized request.  Also manage the busfree expected flag
3956          * from this common routine so that we catch changes due to
3957          * WDTR or SDTR messages.
3958          */
3959         if ((type & AHD_TRANS_CUR) != 0) {
3960                 if (!paused)
3961                         ahd_pause(ahd);
3962                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3963                 if (!paused)
3964                         ahd_unpause(ahd);
3965                 if (ahd->msg_type != MSG_TYPE_NONE) {
3966                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3967                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3968 #ifdef AHD_DEBUG
3969                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3970                                         ahd_print_devinfo(ahd, devinfo);
3971                                         printk("Expecting IU Change busfree\n");
3972                                 }
3973 #endif
3974                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3975                                                |  MSG_FLAG_IU_REQ_CHANGED;
3976                         }
3977                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3978 #ifdef AHD_DEBUG
3979                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3980                                         printk("PPR with IU_REQ outstanding\n");
3981 #endif
3982                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3983                         }
3984                 }
3985         }
3986
3987         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3988                                                 tinfo, AHD_NEG_TO_GOAL);
3989
3990         if (update_needed && active)
3991                 ahd_update_pending_scbs(ahd);
3992 }
3993
3994 /*
3995  * Update the user/goal/curr tables of wide negotiation
3996  * parameters as well as, in the case of a current or active update,
3997  * any data structures on the host controller.  In the case of an
3998  * active update, the specified target is currently talking to us on
3999  * the bus, so the transfer parameter update must take effect
4000  * immediately.
4001  */
4002 void
4003 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4004               u_int width, u_int type, int paused)
4005 {
4006         struct  ahd_initiator_tinfo *tinfo;
4007         struct  ahd_tmode_tstate *tstate;
4008         u_int   oldwidth;
4009         int     active;
4010         int     update_needed;
4011
4012         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
4013         update_needed = 0;
4014         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4015                                     devinfo->target, &tstate);
4016
4017         if ((type & AHD_TRANS_USER) != 0)
4018                 tinfo->user.width = width;
4019
4020         if ((type & AHD_TRANS_GOAL) != 0)
4021                 tinfo->goal.width = width;
4022
4023         oldwidth = tinfo->curr.width;
4024         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
4025
4026                 update_needed++;
4027
4028                 tinfo->curr.width = width;
4029                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
4030                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
4031                 if (bootverbose) {
4032                         printk("%s: target %d using %dbit transfers\n",
4033                                ahd_name(ahd), devinfo->target,
4034                                8 * (0x01 << width));
4035                 }
4036         }
4037
4038         if ((type & AHD_TRANS_CUR) != 0) {
4039                 if (!paused)
4040                         ahd_pause(ahd);
4041                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
4042                 if (!paused)
4043                         ahd_unpause(ahd);
4044         }
4045
4046         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
4047                                                 tinfo, AHD_NEG_TO_GOAL);
4048         if (update_needed && active)
4049                 ahd_update_pending_scbs(ahd);
4050
4051 }
4052
4053 /*
4054  * Update the current state of tagged queuing for a given target.
4055  */
4056 static void
4057 ahd_set_tags(struct ahd_softc *ahd, struct scsi_cmnd *cmd,
4058              struct ahd_devinfo *devinfo, ahd_queue_alg alg)
4059 {
4060         struct scsi_device *sdev = cmd->device;
4061
4062         ahd_platform_set_tags(ahd, sdev, devinfo, alg);
4063         ahd_send_async(ahd, devinfo->channel, devinfo->target,
4064                        devinfo->lun, AC_TRANSFER_NEG);
4065 }
4066
4067 static void
4068 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4069                      struct ahd_transinfo *tinfo)
4070 {
4071         ahd_mode_state  saved_modes;
4072         u_int           period;
4073         u_int           ppr_opts;
4074         u_int           con_opts;
4075         u_int           offset;
4076         u_int           saved_negoaddr;
4077         uint8_t         iocell_opts[sizeof(ahd->iocell_opts)];
4078
4079         saved_modes = ahd_save_modes(ahd);
4080         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4081
4082         saved_negoaddr = ahd_inb(ahd, NEGOADDR);
4083         ahd_outb(ahd, NEGOADDR, devinfo->target);
4084         period = tinfo->period;
4085         offset = tinfo->offset;
4086         memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
4087         ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
4088                                         |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
4089         con_opts = 0;
4090         if (period == 0)
4091                 period = AHD_SYNCRATE_ASYNC;
4092         if (period == AHD_SYNCRATE_160) {
4093
4094                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4095                         /*
4096                          * When the SPI4 spec was finalized, PACE transfers
4097                          * was not made a configurable option in the PPR
4098                          * message.  Instead it is assumed to be enabled for
4099                          * any syncrate faster than 80MHz.  Nevertheless,
4100                          * Harpoon2A4 allows this to be configurable.
4101                          *
4102                          * Harpoon2A4 also assumes at most 2 data bytes per
4103                          * negotiated REQ/ACK offset.  Paced transfers take
4104                          * 4, so we must adjust our offset.
4105                          */
4106                         ppr_opts |= PPROPT_PACE;
4107                         offset *= 2;
4108
4109                         /*
4110                          * Harpoon2A assumed that there would be a
4111                          * fallback rate between 160MHz and 80MHz,
4112                          * so 7 is used as the period factor rather
4113                          * than 8 for 160MHz.
4114                          */
4115                         period = AHD_SYNCRATE_REVA_160;
4116                 }
4117                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
4118                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4119                             ~AHD_PRECOMP_MASK;
4120         } else {
4121                 /*
4122                  * Precomp should be disabled for non-paced transfers.
4123                  */
4124                 iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
4125
4126                 if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
4127                  && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
4128                  && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
4129                         /*
4130                          * Slow down our CRC interval to be
4131                          * compatible with non-packetized
4132                          * U160 devices that can't handle a
4133                          * CRC at full speed.
4134                          */
4135                         con_opts |= ENSLOWCRC;
4136                 }
4137
4138                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4139                         /*
4140                          * On H2A4, revert to a slower slewrate
4141                          * on non-paced transfers.
4142                          */
4143                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4144                             ~AHD_SLEWRATE_MASK;
4145                 }
4146         }
4147
4148         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
4149         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
4150         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
4151         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
4152
4153         ahd_outb(ahd, NEGPERIOD, period);
4154         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
4155         ahd_outb(ahd, NEGOFFSET, offset);
4156
4157         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
4158                 con_opts |= WIDEXFER;
4159
4160         /*
4161          * Slow down our CRC interval to be
4162          * compatible with packetized U320 devices
4163          * that can't handle a CRC at full speed
4164          */
4165         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
4166                 con_opts |= ENSLOWCRC;
4167         }
4168
4169         /*
4170          * During packetized transfers, the target will
4171          * give us the opportunity to send command packets
4172          * without us asserting attention.
4173          */
4174         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
4175                 con_opts |= ENAUTOATNO;
4176         ahd_outb(ahd, NEGCONOPTS, con_opts);
4177         ahd_outb(ahd, NEGOADDR, saved_negoaddr);
4178         ahd_restore_modes(ahd, saved_modes);
4179 }
4180
4181 /*
4182  * When the transfer settings for a connection change, setup for
4183  * negotiation in pending SCBs to effect the change as quickly as
4184  * possible.  We also cancel any negotiations that are scheduled
4185  * for inflight SCBs that have not been started yet.
4186  */
4187 static void
4188 ahd_update_pending_scbs(struct ahd_softc *ahd)
4189 {
4190         struct          scb *pending_scb;
4191         int             pending_scb_count;
4192         int             paused;
4193         u_int           saved_scbptr;
4194         ahd_mode_state  saved_modes;
4195
4196         /*
4197          * Traverse the pending SCB list and ensure that all of the
4198          * SCBs there have the proper settings.  We can only safely
4199          * clear the negotiation required flag (setting requires the
4200          * execution queue to be modified) and this is only possible
4201          * if we are not already attempting to select out for this
4202          * SCB.  For this reason, all callers only call this routine
4203          * if we are changing the negotiation settings for the currently
4204          * active transaction on the bus.
4205          */
4206         pending_scb_count = 0;
4207         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4208                 struct ahd_devinfo devinfo;
4209                 struct ahd_tmode_tstate *tstate;
4210
4211                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
4212                 ahd_fetch_transinfo(ahd, devinfo.channel, devinfo.our_scsiid,
4213                                     devinfo.target, &tstate);
4214                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
4215                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
4216                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
4217                         pending_scb->hscb->control &= ~MK_MESSAGE;
4218                 }
4219                 ahd_sync_scb(ahd, pending_scb,
4220                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
4221                 pending_scb_count++;
4222         }
4223
4224         if (pending_scb_count == 0)
4225                 return;
4226
4227         if (ahd_is_paused(ahd)) {
4228                 paused = 1;
4229         } else {
4230                 paused = 0;
4231                 ahd_pause(ahd);
4232         }
4233
4234         /*
4235          * Force the sequencer to reinitialize the selection for
4236          * the command at the head of the execution queue if it
4237          * has already been setup.  The negotiation changes may
4238          * effect whether we select-out with ATN.  It is only
4239          * safe to clear ENSELO when the bus is not free and no
4240          * selection is in progres or completed.
4241          */
4242         saved_modes = ahd_save_modes(ahd);
4243         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4244         if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
4245          && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
4246                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
4247         saved_scbptr = ahd_get_scbptr(ahd);
4248         /* Ensure that the hscbs down on the card match the new information */
4249         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4250                 u_int   scb_tag;
4251                 u_int   control;
4252
4253                 scb_tag = SCB_GET_TAG(pending_scb);
4254                 ahd_set_scbptr(ahd, scb_tag);
4255                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
4256                 control &= ~MK_MESSAGE;
4257                 control |= pending_scb->hscb->control & MK_MESSAGE;
4258                 ahd_outb(ahd, SCB_CONTROL, control);
4259         }
4260         ahd_set_scbptr(ahd, saved_scbptr);
4261         ahd_restore_modes(ahd, saved_modes);
4262
4263         if (paused == 0)
4264                 ahd_unpause(ahd);
4265 }
4266
4267 /**************************** Pathing Information *****************************/
4268 static void
4269 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4270 {
4271         ahd_mode_state  saved_modes;
4272         u_int           saved_scsiid;
4273         role_t          role;
4274         int             our_id;
4275
4276         saved_modes = ahd_save_modes(ahd);
4277         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4278
4279         if (ahd_inb(ahd, SSTAT0) & TARGET)
4280                 role = ROLE_TARGET;
4281         else
4282                 role = ROLE_INITIATOR;
4283
4284         if (role == ROLE_TARGET
4285          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
4286                 /* We were selected, so pull our id from TARGIDIN */
4287                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
4288         } else if (role == ROLE_TARGET)
4289                 our_id = ahd_inb(ahd, TOWNID);
4290         else
4291                 our_id = ahd_inb(ahd, IOWNID);
4292
4293         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
4294         ahd_compile_devinfo(devinfo,
4295                             our_id,
4296                             SCSIID_TARGET(ahd, saved_scsiid),
4297                             ahd_inb(ahd, SAVED_LUN),
4298                             SCSIID_CHANNEL(ahd, saved_scsiid),
4299                             role);
4300         ahd_restore_modes(ahd, saved_modes);
4301 }
4302
4303 void
4304 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4305 {
4306         printk("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
4307                devinfo->target, devinfo->lun);
4308 }
4309
4310 static const struct ahd_phase_table_entry*
4311 ahd_lookup_phase_entry(int phase)
4312 {
4313         const struct ahd_phase_table_entry *entry;
4314         const struct ahd_phase_table_entry *last_entry;
4315
4316         /*
4317          * num_phases doesn't include the default entry which
4318          * will be returned if the phase doesn't match.
4319          */
4320         last_entry = &ahd_phase_table[num_phases];
4321         for (entry = ahd_phase_table; entry < last_entry; entry++) {
4322                 if (phase == entry->phase)
4323                         break;
4324         }
4325         return (entry);
4326 }
4327
4328 void
4329 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
4330                     u_int lun, char channel, role_t role)
4331 {
4332         devinfo->our_scsiid = our_id;
4333         devinfo->target = target;
4334         devinfo->lun = lun;
4335         devinfo->target_offset = target;
4336         devinfo->channel = channel;
4337         devinfo->role = role;
4338         if (channel == 'B')
4339                 devinfo->target_offset += 8;
4340         devinfo->target_mask = (0x01 << devinfo->target_offset);
4341 }
4342
4343 static void
4344 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4345                 struct scb *scb)
4346 {
4347         role_t  role;
4348         int     our_id;
4349
4350         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
4351         role = ROLE_INITIATOR;
4352         if ((scb->hscb->control & TARGET_SCB) != 0)
4353                 role = ROLE_TARGET;
4354         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
4355                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
4356 }
4357
4358
4359 /************************ Message Phase Processing ****************************/
4360 /*
4361  * When an initiator transaction with the MK_MESSAGE flag either reconnects
4362  * or enters the initial message out phase, we are interrupted.  Fill our
4363  * outgoing message buffer with the appropriate message and beging handing
4364  * the message phase(s) manually.
4365  */
4366 static void
4367 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4368                            struct scb *scb)
4369 {
4370         /*
4371          * To facilitate adding multiple messages together,
4372          * each routine should increment the index and len
4373          * variables instead of setting them explicitly.
4374          */
4375         ahd->msgout_index = 0;
4376         ahd->msgout_len = 0;
4377
4378         if (ahd_currently_packetized(ahd))
4379                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
4380
4381         if (ahd->send_msg_perror
4382          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
4383                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
4384                 ahd->msgout_len++;
4385                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4386 #ifdef AHD_DEBUG
4387                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4388                         printk("Setting up for Parity Error delivery\n");
4389 #endif
4390                 return;
4391         } else if (scb == NULL) {
4392                 printk("%s: WARNING. No pending message for "
4393                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
4394                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
4395                 ahd->msgout_len++;
4396                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4397                 return;
4398         }
4399
4400         if ((scb->flags & SCB_DEVICE_RESET) == 0
4401          && (scb->flags & SCB_PACKETIZED) == 0
4402          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
4403                 u_int identify_msg;
4404
4405                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
4406                 if ((scb->hscb->control & DISCENB) != 0)
4407                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
4408                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
4409                 ahd->msgout_len++;
4410
4411                 if ((scb->hscb->control & TAG_ENB) != 0) {
4412                         ahd->msgout_buf[ahd->msgout_index++] =
4413                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
4414                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
4415                         ahd->msgout_len += 2;
4416                 }
4417         }
4418
4419         if (scb->flags & SCB_DEVICE_RESET) {
4420                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
4421                 ahd->msgout_len++;
4422                 ahd_print_path(ahd, scb);
4423                 printk("Bus Device Reset Message Sent\n");
4424                 /*
4425                  * Clear our selection hardware in advance of
4426                  * the busfree.  We may have an entry in the waiting
4427                  * Q for this target, and we don't want to go about
4428                  * selecting while we handle the busfree and blow it
4429                  * away.
4430                  */
4431                 ahd_outb(ahd, SCSISEQ0, 0);
4432         } else if ((scb->flags & SCB_ABORT) != 0) {
4433
4434                 if ((scb->hscb->control & TAG_ENB) != 0) {
4435                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
4436                 } else {
4437                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
4438                 }
4439                 ahd->msgout_len++;
4440                 ahd_print_path(ahd, scb);
4441                 printk("Abort%s Message Sent\n",
4442                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
4443                 /*
4444                  * Clear our selection hardware in advance of
4445                  * the busfree.  We may have an entry in the waiting
4446                  * Q for this target, and we don't want to go about
4447                  * selecting while we handle the busfree and blow it
4448                  * away.
4449                  */
4450                 ahd_outb(ahd, SCSISEQ0, 0);
4451         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
4452                 ahd_build_transfer_msg(ahd, devinfo);
4453                 /*
4454                  * Clear our selection hardware in advance of potential
4455                  * PPR IU status change busfree.  We may have an entry in
4456                  * the waiting Q for this target, and we don't want to go
4457                  * about selecting while we handle the busfree and blow
4458                  * it away.
4459                  */
4460                 ahd_outb(ahd, SCSISEQ0, 0);
4461         } else {
4462                 printk("ahd_intr: AWAITING_MSG for an SCB that "
4463                        "does not have a waiting message\n");
4464                 printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
4465                        devinfo->target_mask);
4466                 panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
4467                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
4468                       ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
4469                       scb->flags);
4470         }
4471
4472         /*
4473          * Clear the MK_MESSAGE flag from the SCB so we aren't
4474          * asked to send this message again.
4475          */
4476         ahd_outb(ahd, SCB_CONTROL,
4477                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
4478         scb->hscb->control &= ~MK_MESSAGE;
4479         ahd->msgout_index = 0;
4480         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4481 }
4482
4483 /*
4484  * Build an appropriate transfer negotiation message for the
4485  * currently active target.
4486  */
4487 static void
4488 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4489 {
4490         /*
4491          * We need to initiate transfer negotiations.
4492          * If our current and goal settings are identical,
4493          * we want to renegotiate due to a check condition.
4494          */
4495         struct  ahd_initiator_tinfo *tinfo;
4496         struct  ahd_tmode_tstate *tstate;
4497         int     dowide;
4498         int     dosync;
4499         int     doppr;
4500         u_int   period;
4501         u_int   ppr_options;
4502         u_int   offset;
4503
4504         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4505                                     devinfo->target, &tstate);
4506         /*
4507          * Filter our period based on the current connection.
4508          * If we can't perform DT transfers on this segment (not in LVD
4509          * mode for instance), then our decision to issue a PPR message
4510          * may change.
4511          */
4512         period = tinfo->goal.period;
4513         offset = tinfo->goal.offset;
4514         ppr_options = tinfo->goal.ppr_options;
4515         /* Target initiated PPR is not allowed in the SCSI spec */
4516         if (devinfo->role == ROLE_TARGET)
4517                 ppr_options = 0;
4518         ahd_devlimited_syncrate(ahd, tinfo, &period,
4519                                 &ppr_options, devinfo->role);
4520         dowide = tinfo->curr.width != tinfo->goal.width;
4521         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
4522         /*
4523          * Only use PPR if we have options that need it, even if the device
4524          * claims to support it.  There might be an expander in the way
4525          * that doesn't.
4526          */
4527         doppr = ppr_options != 0;
4528
4529         if (!dowide && !dosync && !doppr) {
4530                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
4531                 dosync = tinfo->goal.offset != 0;
4532         }
4533
4534         if (!dowide && !dosync && !doppr) {
4535                 /*
4536                  * Force async with a WDTR message if we have a wide bus,
4537                  * or just issue an SDTR with a 0 offset.
4538                  */
4539                 if ((ahd->features & AHD_WIDE) != 0)
4540                         dowide = 1;
4541                 else
4542                         dosync = 1;
4543
4544                 if (bootverbose) {
4545                         ahd_print_devinfo(ahd, devinfo);
4546                         printk("Ensuring async\n");
4547                 }
4548         }
4549         /* Target initiated PPR is not allowed in the SCSI spec */
4550         if (devinfo->role == ROLE_TARGET)
4551                 doppr = 0;
4552
4553         /*
4554          * Both the PPR message and SDTR message require the
4555          * goal syncrate to be limited to what the target device
4556          * is capable of handling (based on whether an LVD->SE
4557          * expander is on the bus), so combine these two cases.
4558          * Regardless, guarantee that if we are using WDTR and SDTR
4559          * messages that WDTR comes first.
4560          */
4561         if (doppr || (dosync && !dowide)) {
4562
4563                 offset = tinfo->goal.offset;
4564                 ahd_validate_offset(ahd, tinfo, period, &offset,
4565                                     doppr ? tinfo->goal.width
4566                                           : tinfo->curr.width,
4567                                     devinfo->role);
4568                 if (doppr) {
4569                         ahd_construct_ppr(ahd, devinfo, period, offset,
4570                                           tinfo->goal.width, ppr_options);
4571                 } else {
4572                         ahd_construct_sdtr(ahd, devinfo, period, offset);
4573                 }
4574         } else {
4575                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
4576         }
4577 }
4578
4579 /*
4580  * Build a synchronous negotiation message in our message
4581  * buffer based on the input parameters.
4582  */
4583 static void
4584 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4585                    u_int period, u_int offset)
4586 {
4587         if (offset == 0)
4588                 period = AHD_ASYNC_XFER_PERIOD;
4589         ahd->msgout_index += spi_populate_sync_msg(
4590                         ahd->msgout_buf + ahd->msgout_index, period, offset);
4591         ahd->msgout_len += 5;
4592         if (bootverbose) {
4593                 printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
4594                        ahd_name(ahd), devinfo->channel, devinfo->target,
4595                        devinfo->lun, period, offset);
4596         }
4597 }
4598
4599 /*
4600  * Build a wide negotiateion message in our message
4601  * buffer based on the input parameters.
4602  */
4603 static void
4604 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4605                    u_int bus_width)
4606 {
4607         ahd->msgout_index += spi_populate_width_msg(
4608                         ahd->msgout_buf + ahd->msgout_index, bus_width);
4609         ahd->msgout_len += 4;
4610         if (bootverbose) {
4611                 printk("(%s:%c:%d:%d): Sending WDTR %x\n",
4612                        ahd_name(ahd), devinfo->channel, devinfo->target,
4613                        devinfo->lun, bus_width);
4614         }
4615 }
4616
4617 /*
4618  * Build a parallel protocol request message in our message
4619  * buffer based on the input parameters.
4620  */
4621 static void
4622 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4623                   u_int period, u_int offset, u_int bus_width,
4624                   u_int ppr_options)
4625 {
4626         /*
4627          * Always request precompensation from
4628          * the other target if we are running
4629          * at paced syncrates.
4630          */
4631         if (period <= AHD_SYNCRATE_PACED)
4632                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
4633         if (offset == 0)
4634                 period = AHD_ASYNC_XFER_PERIOD;
4635         ahd->msgout_index += spi_populate_ppr_msg(
4636                         ahd->msgout_buf + ahd->msgout_index, period, offset,
4637                         bus_width, ppr_options);
4638         ahd->msgout_len += 8;
4639         if (bootverbose) {
4640                 printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
4641                        "offset %x, ppr_options %x\n", ahd_name(ahd),
4642                        devinfo->channel, devinfo->target, devinfo->lun,
4643                        bus_width, period, offset, ppr_options);
4644         }
4645 }
4646
4647 /*
4648  * Clear any active message state.
4649  */
4650 static void
4651 ahd_clear_msg_state(struct ahd_softc *ahd)
4652 {
4653         ahd_mode_state saved_modes;
4654
4655         saved_modes = ahd_save_modes(ahd);
4656         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4657         ahd->send_msg_perror = 0;
4658         ahd->msg_flags = MSG_FLAG_NONE;
4659         ahd->msgout_len = 0;
4660         ahd->msgin_index = 0;
4661         ahd->msg_type = MSG_TYPE_NONE;
4662         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
4663                 /*
4664                  * The target didn't care to respond to our
4665                  * message request, so clear ATN.
4666                  */
4667                 ahd_outb(ahd, CLRSINT1, CLRATNO);
4668         }
4669         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
4670         ahd_outb(ahd, SEQ_FLAGS2,
4671                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
4672         ahd_restore_modes(ahd, saved_modes);
4673 }
4674
4675 /*
4676  * Manual message loop handler.
4677  */
4678 static void
4679 ahd_handle_message_phase(struct ahd_softc *ahd)
4680 {
4681         struct  ahd_devinfo devinfo;
4682         u_int   bus_phase;
4683         int     end_session;
4684
4685         ahd_fetch_devinfo(ahd, &devinfo);
4686         end_session = FALSE;
4687         bus_phase = ahd_inb(ahd, LASTPHASE);
4688
4689         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
4690                 printk("LQIRETRY for LQIPHASE_OUTPKT\n");
4691                 ahd_outb(ahd, LQCTL2, LQIRETRY);
4692         }
4693 reswitch:
4694         switch (ahd->msg_type) {
4695         case MSG_TYPE_INITIATOR_MSGOUT:
4696         {
4697                 int lastbyte;
4698                 int phasemis;
4699                 int msgdone;
4700
4701                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
4702                         panic("HOST_MSG_LOOP interrupt with no active message");
4703
4704 #ifdef AHD_DEBUG
4705                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4706                         ahd_print_devinfo(ahd, &devinfo);
4707                         printk("INITIATOR_MSG_OUT");
4708                 }
4709 #endif
4710                 phasemis = bus_phase != P_MESGOUT;
4711                 if (phasemis) {
4712 #ifdef AHD_DEBUG
4713                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4714                                 printk(" PHASEMIS %s\n",
4715                                        ahd_lookup_phase_entry(bus_phase)
4716                                                              ->phasemsg);
4717                         }
4718 #endif
4719                         if (bus_phase == P_MESGIN) {
4720                                 /*
4721                                  * Change gears and see if
4722                                  * this messages is of interest to
4723                                  * us or should be passed back to
4724                                  * the sequencer.
4725                                  */
4726                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
4727                                 ahd->send_msg_perror = 0;
4728                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
4729                                 ahd->msgin_index = 0;
4730                                 goto reswitch;
4731                         }
4732                         end_session = TRUE;
4733                         break;
4734                 }
4735
4736                 if (ahd->send_msg_perror) {
4737                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4738                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4739 #ifdef AHD_DEBUG
4740                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4741                                 printk(" byte 0x%x\n", ahd->send_msg_perror);
4742 #endif
4743                         /*
4744                          * If we are notifying the target of a CRC error
4745                          * during packetized operations, the target is
4746                          * within its rights to acknowledge our message
4747                          * with a busfree.
4748                          */
4749                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
4750                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
4751                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
4752
4753                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
4754                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4755                         break;
4756                 }
4757
4758                 msgdone = ahd->msgout_index == ahd->msgout_len;
4759                 if (msgdone) {
4760                         /*
4761                          * The target has requested a retry.
4762                          * Re-assert ATN, reset our message index to
4763                          * 0, and try again.
4764                          */
4765                         ahd->msgout_index = 0;
4766                         ahd_assert_atn(ahd);
4767                 }
4768
4769                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
4770                 if (lastbyte) {
4771                         /* Last byte is signified by dropping ATN */
4772                         ahd_outb(ahd, CLRSINT1, CLRATNO);
4773                 }
4774
4775                 /*
4776                  * Clear our interrupt status and present
4777                  * the next byte on the bus.
4778                  */
4779                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4780 #ifdef AHD_DEBUG
4781                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4782                         printk(" byte 0x%x\n",
4783                                ahd->msgout_buf[ahd->msgout_index]);
4784 #endif
4785                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
4786                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4787                 break;
4788         }
4789         case MSG_TYPE_INITIATOR_MSGIN:
4790         {
4791                 int phasemis;
4792                 int message_done;
4793
4794 #ifdef AHD_DEBUG
4795                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4796                         ahd_print_devinfo(ahd, &devinfo);
4797                         printk("INITIATOR_MSG_IN");
4798                 }
4799 #endif
4800                 phasemis = bus_phase != P_MESGIN;
4801                 if (phasemis) {
4802 #ifdef AHD_DEBUG
4803                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4804                                 printk(" PHASEMIS %s\n",
4805                                        ahd_lookup_phase_entry(bus_phase)
4806                                                              ->phasemsg);
4807                         }
4808 #endif
4809                         ahd->msgin_index = 0;
4810                         if (bus_phase == P_MESGOUT
4811                          && (ahd->send_msg_perror != 0
4812                           || (ahd->msgout_len != 0
4813                            && ahd->msgout_index == 0))) {
4814                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4815                                 goto reswitch;
4816                         }
4817                         end_session = TRUE;
4818                         break;
4819                 }
4820
4821                 /* Pull the byte in without acking it */
4822                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4823 #ifdef AHD_DEBUG
4824                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4825                         printk(" byte 0x%x\n",
4826                                ahd->msgin_buf[ahd->msgin_index]);
4827 #endif
4828
4829                 message_done = ahd_parse_msg(ahd, &devinfo);
4830
4831                 if (message_done) {
4832                         /*
4833                          * Clear our incoming message buffer in case there
4834                          * is another message following this one.
4835                          */
4836                         ahd->msgin_index = 0;
4837
4838                         /*
4839                          * If this message illicited a response,
4840                          * assert ATN so the target takes us to the
4841                          * message out phase.
4842                          */
4843                         if (ahd->msgout_len != 0) {
4844 #ifdef AHD_DEBUG
4845                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4846                                         ahd_print_devinfo(ahd, &devinfo);
4847                                         printk("Asserting ATN for response\n");
4848                                 }
4849 #endif
4850                                 ahd_assert_atn(ahd);
4851                         }
4852                 } else
4853                         ahd->msgin_index++;
4854
4855                 if (message_done == MSGLOOP_TERMINATED) {
4856                         end_session = TRUE;
4857                 } else {
4858                         /* Ack the byte */
4859                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4860                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4861                 }
4862                 break;
4863         }
4864         case MSG_TYPE_TARGET_MSGIN:
4865         {
4866                 int msgdone;
4867                 int msgout_request;
4868
4869                 /*
4870                  * By default, the message loop will continue.
4871                  */
4872                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4873
4874                 if (ahd->msgout_len == 0)
4875                         panic("Target MSGIN with no active message");
4876
4877                 /*
4878                  * If we interrupted a mesgout session, the initiator
4879                  * will not know this until our first REQ.  So, we
4880                  * only honor mesgout requests after we've sent our
4881                  * first byte.
4882                  */
4883                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4884                  && ahd->msgout_index > 0)
4885                         msgout_request = TRUE;
4886                 else
4887                         msgout_request = FALSE;
4888
4889                 if (msgout_request) {
4890
4891                         /*
4892                          * Change gears and see if
4893                          * this messages is of interest to
4894                          * us or should be passed back to
4895                          * the sequencer.
4896                          */
4897                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4898                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4899                         ahd->msgin_index = 0;
4900                         /* Dummy read to REQ for first byte */
4901                         ahd_inb(ahd, SCSIDAT);
4902                         ahd_outb(ahd, SXFRCTL0,
4903                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4904                         break;
4905                 }
4906
4907                 msgdone = ahd->msgout_index == ahd->msgout_len;
4908                 if (msgdone) {
4909                         ahd_outb(ahd, SXFRCTL0,
4910                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4911                         end_session = TRUE;
4912                         break;
4913                 }
4914
4915                 /*
4916                  * Present the next byte on the bus.
4917                  */
4918                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4919                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4920                 break;
4921         }
4922         case MSG_TYPE_TARGET_MSGOUT:
4923         {
4924                 int lastbyte;
4925                 int msgdone;
4926
4927                 /*
4928                  * By default, the message loop will continue.
4929                  */
4930                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4931
4932                 /*
4933                  * The initiator signals that this is
4934                  * the last byte by dropping ATN.
4935                  */
4936                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4937
4938                 /*
4939                  * Read the latched byte, but turn off SPIOEN first
4940                  * so that we don't inadvertently cause a REQ for the
4941                  * next byte.
4942                  */
4943                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4944                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4945                 msgdone = ahd_parse_msg(ahd, &devinfo);
4946                 if (msgdone == MSGLOOP_TERMINATED) {
4947                         /*
4948                          * The message is *really* done in that it caused
4949                          * us to go to bus free.  The sequencer has already
4950                          * been reset at this point, so pull the ejection
4951                          * handle.
4952                          */
4953                         return;
4954                 }
4955
4956                 ahd->msgin_index++;
4957
4958                 /*
4959                  * XXX Read spec about initiator dropping ATN too soon
4960                  *     and use msgdone to detect it.
4961                  */
4962                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
4963                         ahd->msgin_index = 0;
4964
4965                         /*
4966                          * If this message illicited a response, transition
4967                          * to the Message in phase and send it.
4968                          */
4969                         if (ahd->msgout_len != 0) {
4970                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
4971                                 ahd_outb(ahd, SXFRCTL0,
4972                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4973                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4974                                 ahd->msgin_index = 0;
4975                                 break;
4976                         }
4977                 }
4978
4979                 if (lastbyte)
4980                         end_session = TRUE;
4981                 else {
4982                         /* Ask for the next byte. */
4983                         ahd_outb(ahd, SXFRCTL0,
4984                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4985                 }
4986
4987                 break;
4988         }
4989         default:
4990                 panic("Unknown REQINIT message type");
4991         }
4992
4993         if (end_session) {
4994                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
4995                         printk("%s: Returning to Idle Loop\n",
4996                                ahd_name(ahd));
4997                         ahd_clear_msg_state(ahd);
4998
4999                         /*
5000                          * Perform the equivalent of a clear_target_state.
5001                          */
5002                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
5003                         ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
5004                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
5005                 } else {
5006                         ahd_clear_msg_state(ahd);
5007                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
5008                 }
5009         }
5010 }
5011
5012 /*
5013  * See if we sent a particular extended message to the target.
5014  * If "full" is true, return true only if the target saw the full
5015  * message.  If "full" is false, return true if the target saw at
5016  * least the first byte of the message.
5017  */
5018 static int
5019 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
5020 {
5021         int found;
5022         u_int index;
5023
5024         found = FALSE;
5025         index = 0;
5026
5027         while (index < ahd->msgout_len) {
5028                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
5029                         u_int end_index;
5030
5031                         end_index = index + 1 + ahd->msgout_buf[index + 1];
5032                         if (ahd->msgout_buf[index+2] == msgval
5033                          && type == AHDMSG_EXT) {
5034
5035                                 if (full) {
5036                                         if (ahd->msgout_index > end_index)
5037                                                 found = TRUE;
5038                                 } else if (ahd->msgout_index > index)
5039                                         found = TRUE;
5040                         }
5041                         index = end_index;
5042                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
5043                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
5044
5045                         /* Skip tag type and tag id or residue param*/
5046                         index += 2;
5047                 } else {
5048                         /* Single byte message */
5049                         if (type == AHDMSG_1B
5050                          && ahd->msgout_index > index
5051                          && (ahd->msgout_buf[index] == msgval
5052                           || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
5053                            && msgval == MSG_IDENTIFYFLAG)))
5054                                 found = TRUE;
5055                         index++;
5056                 }
5057
5058                 if (found)
5059                         break;
5060         }
5061         return (found);
5062 }
5063
5064 /*
5065  * Wait for a complete incoming message, parse it, and respond accordingly.
5066  */
5067 static int
5068 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5069 {
5070         struct  ahd_initiator_tinfo *tinfo;
5071         struct  ahd_tmode_tstate *tstate;
5072         int     reject;
5073         int     done;
5074         int     response;
5075
5076         done = MSGLOOP_IN_PROG;
5077         response = FALSE;
5078         reject = FALSE;
5079         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
5080                                     devinfo->target, &tstate);
5081
5082         /*
5083          * Parse as much of the message as is available,
5084          * rejecting it if we don't support it.  When
5085          * the entire message is available and has been
5086          * handled, return MSGLOOP_MSGCOMPLETE, indicating
5087          * that we have parsed an entire message.
5088          *
5089          * In the case of extended messages, we accept the length
5090          * byte outright and perform more checking once we know the
5091          * extended message type.
5092          */
5093         switch (ahd->msgin_buf[0]) {
5094         case MSG_DISCONNECT:
5095         case MSG_SAVEDATAPOINTER:
5096         case MSG_CMDCOMPLETE:
5097         case MSG_RESTOREPOINTERS:
5098         case MSG_IGN_WIDE_RESIDUE:
5099                 /*
5100                  * End our message loop as these are messages
5101                  * the sequencer handles on its own.
5102                  */
5103                 done = MSGLOOP_TERMINATED;
5104                 break;
5105         case MSG_MESSAGE_REJECT:
5106                 response = ahd_handle_msg_reject(ahd, devinfo);
5107                 fallthrough;
5108         case MSG_NOOP:
5109                 done = MSGLOOP_MSGCOMPLETE;
5110                 break;
5111         case MSG_EXTENDED:
5112         {
5113                 /* Wait for enough of the message to begin validation */
5114                 if (ahd->msgin_index < 2)
5115                         break;
5116                 switch (ahd->msgin_buf[2]) {
5117                 case MSG_EXT_SDTR:
5118                 {
5119                         u_int    period;
5120                         u_int    ppr_options;
5121                         u_int    offset;
5122                         u_int    saved_offset;
5123
5124                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
5125                                 reject = TRUE;
5126                                 break;
5127                         }
5128
5129                         /*
5130                          * Wait until we have both args before validating
5131                          * and acting on this message.
5132                          *
5133                          * Add one to MSG_EXT_SDTR_LEN to account for
5134                          * the extended message preamble.
5135                          */
5136                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
5137                                 break;
5138
5139                         period = ahd->msgin_buf[3];
5140                         ppr_options = 0;
5141                         saved_offset = offset = ahd->msgin_buf[4];
5142                         ahd_devlimited_syncrate(ahd, tinfo, &period,
5143                                                 &ppr_options, devinfo->role);
5144                         ahd_validate_offset(ahd, tinfo, period, &offset,
5145                                             tinfo->curr.width, devinfo->role);
5146                         if (bootverbose) {
5147                                 printk("(%s:%c:%d:%d): Received "
5148                                        "SDTR period %x, offset %x\n\t"
5149                                        "Filtered to period %x, offset %x\n",
5150                                        ahd_name(ahd), devinfo->channel,
5151                                        devinfo->target, devinfo->lun,
5152                                        ahd->msgin_buf[3], saved_offset,
5153                                        period, offset);
5154                         }
5155                         ahd_set_syncrate(ahd, devinfo, period,
5156                                          offset, ppr_options,
5157                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5158                                          /*paused*/TRUE);
5159
5160                         /*
5161                          * See if we initiated Sync Negotiation
5162                          * and didn't have to fall down to async
5163                          * transfers.
5164                          */
5165                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
5166                                 /* We started it */
5167                                 if (saved_offset != offset) {
5168                                         /* Went too low - force async */
5169                                         reject = TRUE;
5170                                 }
5171                         } else {
5172                                 /*
5173                                  * Send our own SDTR in reply
5174                                  */
5175                                 if (bootverbose
5176                                  && devinfo->role == ROLE_INITIATOR) {
5177                                         printk("(%s:%c:%d:%d): Target "
5178                                                "Initiated SDTR\n",
5179                                                ahd_name(ahd), devinfo->channel,
5180                                                devinfo->target, devinfo->lun);
5181                                 }
5182                                 ahd->msgout_index = 0;
5183                                 ahd->msgout_len = 0;
5184                                 ahd_construct_sdtr(ahd, devinfo,
5185                                                    period, offset);
5186                                 ahd->msgout_index = 0;
5187                                 response = TRUE;
5188                         }
5189                         done = MSGLOOP_MSGCOMPLETE;
5190                         break;
5191                 }
5192                 case MSG_EXT_WDTR:
5193                 {
5194                         u_int bus_width;
5195                         u_int saved_width;
5196                         u_int sending_reply;
5197
5198                         sending_reply = FALSE;
5199                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
5200                                 reject = TRUE;
5201                                 break;
5202                         }
5203
5204                         /*
5205                          * Wait until we have our arg before validating
5206                          * and acting on this message.
5207                          *
5208                          * Add one to MSG_EXT_WDTR_LEN to account for
5209                          * the extended message preamble.
5210                          */
5211                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
5212                                 break;
5213
5214                         bus_width = ahd->msgin_buf[3];
5215                         saved_width = bus_width;
5216                         ahd_validate_width(ahd, tinfo, &bus_width,
5217                                            devinfo->role);
5218                         if (bootverbose) {
5219                                 printk("(%s:%c:%d:%d): Received WDTR "
5220                                        "%x filtered to %x\n",
5221                                        ahd_name(ahd), devinfo->channel,
5222                                        devinfo->target, devinfo->lun,
5223                                        saved_width, bus_width);
5224                         }
5225
5226                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
5227                                 /*
5228                                  * Don't send a WDTR back to the
5229                                  * target, since we asked first.
5230                                  * If the width went higher than our
5231                                  * request, reject it.
5232                                  */
5233                                 if (saved_width > bus_width) {
5234                                         reject = TRUE;
5235                                         printk("(%s:%c:%d:%d): requested %dBit "
5236                                                "transfers.  Rejecting...\n",
5237                                                ahd_name(ahd), devinfo->channel,
5238                                                devinfo->target, devinfo->lun,
5239                                                8 * (0x01 << bus_width));
5240                                         bus_width = 0;
5241                                 }
5242                         } else {
5243                                 /*
5244                                  * Send our own WDTR in reply
5245                                  */
5246                                 if (bootverbose
5247                                  && devinfo->role == ROLE_INITIATOR) {
5248                                         printk("(%s:%c:%d:%d): Target "
5249                                                "Initiated WDTR\n",
5250                                                ahd_name(ahd), devinfo->channel,
5251                                                devinfo->target, devinfo->lun);
5252                                 }
5253                                 ahd->msgout_index = 0;
5254                                 ahd->msgout_len = 0;
5255                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
5256                                 ahd->msgout_index = 0;
5257                                 response = TRUE;
5258                                 sending_reply = TRUE;
5259                         }
5260                         /*
5261                          * After a wide message, we are async, but
5262                          * some devices don't seem to honor this portion
5263                          * of the spec.  Force a renegotiation of the
5264                          * sync component of our transfer agreement even
5265                          * if our goal is async.  By updating our width
5266                          * after forcing the negotiation, we avoid
5267                          * renegotiating for width.
5268                          */
5269                         ahd_update_neg_request(ahd, devinfo, tstate,
5270                                                tinfo, AHD_NEG_ALWAYS);
5271                         ahd_set_width(ahd, devinfo, bus_width,
5272                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5273                                       /*paused*/TRUE);
5274                         if (sending_reply == FALSE && reject == FALSE) {
5275
5276                                 /*
5277                                  * We will always have an SDTR to send.
5278                                  */
5279                                 ahd->msgout_index = 0;
5280                                 ahd->msgout_len = 0;
5281                                 ahd_build_transfer_msg(ahd, devinfo);
5282                                 ahd->msgout_index = 0;
5283                                 response = TRUE;
5284                         }
5285                         done = MSGLOOP_MSGCOMPLETE;
5286                         break;
5287                 }
5288                 case MSG_EXT_PPR:
5289                 {
5290                         u_int   period;
5291                         u_int   offset;
5292                         u_int   bus_width;
5293                         u_int   ppr_options;
5294                         u_int   saved_width;
5295                         u_int   saved_offset;
5296                         u_int   saved_ppr_options;
5297
5298                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
5299                                 reject = TRUE;
5300                                 break;
5301                         }
5302
5303                         /*
5304                          * Wait until we have all args before validating
5305                          * and acting on this message.
5306                          *
5307                          * Add one to MSG_EXT_PPR_LEN to account for
5308                          * the extended message preamble.
5309                          */
5310                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
5311                                 break;
5312
5313                         period = ahd->msgin_buf[3];
5314                         offset = ahd->msgin_buf[5];
5315                         bus_width = ahd->msgin_buf[6];
5316                         saved_width = bus_width;
5317                         ppr_options = ahd->msgin_buf[7];
5318                         /*
5319                          * According to the spec, a DT only
5320                          * period factor with no DT option
5321                          * set implies async.
5322                          */
5323                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
5324                          && period <= 9)
5325                                 offset = 0;
5326                         saved_ppr_options = ppr_options;
5327                         saved_offset = offset;
5328
5329                         /*
5330                          * Transfer options are only available if we
5331                          * are negotiating wide.
5332                          */
5333                         if (bus_width == 0)
5334                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
5335
5336                         ahd_validate_width(ahd, tinfo, &bus_width,
5337                                            devinfo->role);
5338                         ahd_devlimited_syncrate(ahd, tinfo, &period,
5339                                                 &ppr_options, devinfo->role);
5340                         ahd_validate_offset(ahd, tinfo, period, &offset,
5341                                             bus_width, devinfo->role);
5342
5343                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
5344                                 /*
5345                                  * If we are unable to do any of the
5346                                  * requested options (we went too low),
5347                                  * then we'll have to reject the message.
5348                                  */
5349                                 if (saved_width > bus_width
5350                                  || saved_offset != offset
5351                                  || saved_ppr_options != ppr_options) {
5352                                         reject = TRUE;
5353                                         period = 0;
5354                                         offset = 0;
5355                                         bus_width = 0;
5356                                         ppr_options = 0;
5357                                 }
5358                         } else {
5359                                 if (devinfo->role != ROLE_TARGET)
5360                                         printk("(%s:%c:%d:%d): Target "
5361                                                "Initiated PPR\n",
5362                                                ahd_name(ahd), devinfo->channel,
5363                                                devinfo->target, devinfo->lun);
5364                                 else
5365                                         printk("(%s:%c:%d:%d): Initiator "
5366                                                "Initiated PPR\n",
5367                                                ahd_name(ahd), devinfo->channel,
5368                                                devinfo->target, devinfo->lun);
5369                                 ahd->msgout_index = 0;
5370                                 ahd->msgout_len = 0;
5371                                 ahd_construct_ppr(ahd, devinfo, period, offset,
5372                                                   bus_width, ppr_options);
5373                                 ahd->msgout_index = 0;
5374                                 response = TRUE;
5375                         }
5376                         if (bootverbose) {
5377                                 printk("(%s:%c:%d:%d): Received PPR width %x, "
5378                                        "period %x, offset %x,options %x\n"
5379                                        "\tFiltered to width %x, period %x, "
5380                                        "offset %x, options %x\n",
5381                                        ahd_name(ahd), devinfo->channel,
5382                                        devinfo->target, devinfo->lun,
5383                                        saved_width, ahd->msgin_buf[3],
5384                                        saved_offset, saved_ppr_options,
5385                                        bus_width, period, offset, ppr_options);
5386                         }
5387                         ahd_set_width(ahd, devinfo, bus_width,
5388                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5389                                       /*paused*/TRUE);
5390                         ahd_set_syncrate(ahd, devinfo, period,
5391                                          offset, ppr_options,
5392                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5393                                          /*paused*/TRUE);
5394
5395                         done = MSGLOOP_MSGCOMPLETE;
5396                         break;
5397                 }
5398                 default:
5399                         /* Unknown extended message.  Reject it. */
5400                         reject = TRUE;
5401                         break;
5402                 }
5403                 break;
5404         }
5405 #ifdef AHD_TARGET_MODE
5406         case MSG_BUS_DEV_RESET:
5407                 ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
5408                                     CAM_BDR_SENT,
5409                                     "Bus Device Reset Received",
5410                                     /*verbose_level*/0);
5411                 ahd_restart(ahd);
5412                 done = MSGLOOP_TERMINATED;
5413                 break;
5414         case MSG_ABORT_TAG:
5415         case MSG_ABORT:
5416         case MSG_CLEAR_QUEUE:
5417         {
5418                 int tag;
5419
5420                 /* Target mode messages */
5421                 if (devinfo->role != ROLE_TARGET) {
5422                         reject = TRUE;
5423                         break;
5424                 }
5425                 tag = SCB_LIST_NULL;
5426                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
5427                         tag = ahd_inb(ahd, INITIATOR_TAG);
5428                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5429                                devinfo->lun, tag, ROLE_TARGET,
5430                                CAM_REQ_ABORTED);
5431
5432                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
5433                 if (tstate != NULL) {
5434                         struct ahd_tmode_lstate* lstate;
5435
5436                         lstate = tstate->enabled_luns[devinfo->lun];
5437                         if (lstate != NULL) {
5438                                 ahd_queue_lstate_event(ahd, lstate,
5439                                                        devinfo->our_scsiid,
5440                                                        ahd->msgin_buf[0],
5441                                                        /*arg*/tag);
5442                                 ahd_send_lstate_events(ahd, lstate);
5443                         }
5444                 }
5445                 ahd_restart(ahd);
5446                 done = MSGLOOP_TERMINATED;
5447                 break;
5448         }
5449 #endif
5450         case MSG_QAS_REQUEST:
5451 #ifdef AHD_DEBUG
5452                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
5453                         printk("%s: QAS request.  SCSISIGI == 0x%x\n",
5454                                ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
5455 #endif
5456                 ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
5457                 fallthrough;
5458         case MSG_TERM_IO_PROC:
5459         default:
5460                 reject = TRUE;
5461                 break;
5462         }
5463
5464         if (reject) {
5465                 /*
5466                  * Setup to reject the message.
5467                  */
5468                 ahd->msgout_index = 0;
5469                 ahd->msgout_len = 1;
5470                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
5471                 done = MSGLOOP_MSGCOMPLETE;
5472                 response = TRUE;
5473         }
5474
5475         if (done != MSGLOOP_IN_PROG && !response)
5476                 /* Clear the outgoing message buffer */
5477                 ahd->msgout_len = 0;
5478
5479         return (done);
5480 }
5481
5482 /*
5483  * Process a message reject message.
5484  */
5485 static int
5486 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5487 {
5488         /*
5489          * What we care about here is if we had an
5490          * outstanding SDTR or WDTR message for this
5491          * target.  If we did, this is a signal that
5492          * the target is refusing negotiation.
5493          */
5494         struct scb *scb;
5495         struct ahd_initiator_tinfo *tinfo;
5496         struct ahd_tmode_tstate *tstate;
5497         u_int scb_index;
5498         u_int last_msg;
5499         int   response = 0;
5500
5501         scb_index = ahd_get_scbptr(ahd);
5502         scb = ahd_lookup_scb(ahd, scb_index);
5503         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
5504                                     devinfo->our_scsiid,
5505                                     devinfo->target, &tstate);
5506         /* Might be necessary */
5507         last_msg = ahd_inb(ahd, LAST_MSG);
5508
5509         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
5510                 if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
5511                  && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
5512                         /*
5513                          * Target may not like our SPI-4 PPR Options.
5514                          * Attempt to negotiate 80MHz which will turn
5515                          * off these options.
5516                          */
5517                         if (bootverbose) {
5518                                 printk("(%s:%c:%d:%d): PPR Rejected. "
5519                                        "Trying simple U160 PPR\n",
5520                                        ahd_name(ahd), devinfo->channel,
5521                                        devinfo->target, devinfo->lun);
5522                         }
5523                         tinfo->goal.period = AHD_SYNCRATE_DT;
5524                         tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
5525                                                 |  MSG_EXT_PPR_QAS_REQ
5526                                                 |  MSG_EXT_PPR_DT_REQ;
5527                 } else {
5528                         /*
5529                          * Target does not support the PPR message.
5530                          * Attempt to negotiate SPI-2 style.
5531                          */
5532                         if (bootverbose) {
5533                                 printk("(%s:%c:%d:%d): PPR Rejected. "
5534                                        "Trying WDTR/SDTR\n",
5535                                        ahd_name(ahd), devinfo->channel,
5536                                        devinfo->target, devinfo->lun);
5537                         }
5538                         tinfo->goal.ppr_options = 0;
5539                         tinfo->curr.transport_version = 2;
5540                         tinfo->goal.transport_version = 2;
5541                 }
5542                 ahd->msgout_index = 0;
5543                 ahd->msgout_len = 0;
5544                 ahd_build_transfer_msg(ahd, devinfo);
5545                 ahd->msgout_index = 0;
5546                 response = 1;
5547         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
5548
5549                 /* note 8bit xfers */
5550                 printk("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
5551                        "8bit transfers\n", ahd_name(ahd),
5552                        devinfo->channel, devinfo->target, devinfo->lun);
5553                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5554                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5555                               /*paused*/TRUE);
5556                 /*
5557                  * No need to clear the sync rate.  If the target
5558                  * did not accept the command, our syncrate is
5559                  * unaffected.  If the target started the negotiation,
5560                  * but rejected our response, we already cleared the
5561                  * sync rate before sending our WDTR.
5562                  */
5563                 if (tinfo->goal.offset != tinfo->curr.offset) {
5564
5565                         /* Start the sync negotiation */
5566                         ahd->msgout_index = 0;
5567                         ahd->msgout_len = 0;
5568                         ahd_build_transfer_msg(ahd, devinfo);
5569                         ahd->msgout_index = 0;
5570                         response = 1;
5571                 }
5572         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
5573                 /* note asynch xfers and clear flag */
5574                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
5575                                  /*offset*/0, /*ppr_options*/0,
5576                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5577                                  /*paused*/TRUE);
5578                 printk("(%s:%c:%d:%d): refuses synchronous negotiation. "
5579                        "Using asynchronous transfers\n",
5580                        ahd_name(ahd), devinfo->channel,
5581                        devinfo->target, devinfo->lun);
5582         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
5583                 int tag_type;
5584                 int mask;
5585
5586                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
5587
5588                 if (tag_type == MSG_SIMPLE_TASK) {
5589                         printk("(%s:%c:%d:%d): refuses tagged commands.  "
5590                                "Performing non-tagged I/O\n", ahd_name(ahd),
5591                                devinfo->channel, devinfo->target, devinfo->lun);
5592                         ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_NONE);
5593                         mask = ~0x23;
5594                 } else {
5595                         printk("(%s:%c:%d:%d): refuses %s tagged commands.  "
5596                                "Performing simple queue tagged I/O only\n",
5597                                ahd_name(ahd), devinfo->channel, devinfo->target,
5598                                devinfo->lun, tag_type == MSG_ORDERED_TASK
5599                                ? "ordered" : "head of queue");
5600                         ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_BASIC);
5601                         mask = ~0x03;
5602                 }
5603
5604                 /*
5605                  * Resend the identify for this CCB as the target
5606                  * may believe that the selection is invalid otherwise.
5607                  */
5608                 ahd_outb(ahd, SCB_CONTROL,
5609                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
5610                 scb->hscb->control &= mask;
5611                 ahd_set_transaction_tag(scb, /*enabled*/FALSE,
5612                                         /*type*/MSG_SIMPLE_TASK);
5613                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
5614                 ahd_assert_atn(ahd);
5615                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
5616                              SCB_GET_TAG(scb));
5617
5618                 /*
5619                  * Requeue all tagged commands for this target
5620                  * currently in our possession so they can be
5621                  * converted to untagged commands.
5622                  */
5623                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
5624                                    SCB_GET_CHANNEL(ahd, scb),
5625                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
5626                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
5627                                    SEARCH_COMPLETE);
5628         } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
5629                 /*
5630                  * Most likely the device believes that we had
5631                  * previously negotiated packetized.
5632                  */
5633                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
5634                                |  MSG_FLAG_IU_REQ_CHANGED;
5635
5636                 ahd_force_renegotiation(ahd, devinfo);
5637                 ahd->msgout_index = 0;
5638                 ahd->msgout_len = 0;
5639                 ahd_build_transfer_msg(ahd, devinfo);
5640                 ahd->msgout_index = 0;
5641                 response = 1;
5642         } else {
5643                 /*
5644                  * Otherwise, we ignore it.
5645                  */
5646                 printk("%s:%c:%d: Message reject for %x -- ignored\n",
5647                        ahd_name(ahd), devinfo->channel, devinfo->target,
5648                        last_msg);
5649         }
5650         return (response);
5651 }
5652
5653 /*
5654  * Process an ingnore wide residue message.
5655  */
5656 static void
5657 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5658 {
5659         u_int scb_index;
5660         struct scb *scb;
5661
5662         scb_index = ahd_get_scbptr(ahd);
5663         scb = ahd_lookup_scb(ahd, scb_index);
5664         /*
5665          * XXX Actually check data direction in the sequencer?
5666          * Perhaps add datadir to some spare bits in the hscb?
5667          */
5668         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
5669          || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
5670                 /*
5671                  * Ignore the message if we haven't
5672                  * seen an appropriate data phase yet.
5673                  */
5674         } else {
5675                 /*
5676                  * If the residual occurred on the last
5677                  * transfer and the transfer request was
5678                  * expected to end on an odd count, do
5679                  * nothing.  Otherwise, subtract a byte
5680                  * and update the residual count accordingly.
5681                  */
5682                 uint32_t sgptr;
5683
5684                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
5685                 if ((sgptr & SG_LIST_NULL) != 0
5686                  && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5687                      & SCB_XFERLEN_ODD) != 0) {
5688                         /*
5689                          * If the residual occurred on the last
5690                          * transfer and the transfer request was
5691                          * expected to end on an odd count, do
5692                          * nothing.
5693                          */
5694                 } else {
5695                         uint32_t data_cnt;
5696                         uint64_t data_addr;
5697                         uint32_t sglen;
5698
5699                         /* Pull in the rest of the sgptr */
5700                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5701                         data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
5702                         if ((sgptr & SG_LIST_NULL) != 0) {
5703                                 /*
5704                                  * The residual data count is not updated
5705                                  * for the command run to completion case.
5706                                  * Explicitly zero the count.
5707                                  */
5708                                 data_cnt &= ~AHD_SG_LEN_MASK;
5709                         }
5710                         data_addr = ahd_inq(ahd, SHADDR);
5711                         data_cnt += 1;
5712                         data_addr -= 1;
5713                         sgptr &= SG_PTR_MASK;
5714                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5715                                 struct ahd_dma64_seg *sg;
5716
5717                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5718
5719                                 /*
5720                                  * The residual sg ptr points to the next S/G
5721                                  * to load so we must go back one.
5722                                  */
5723                                 sg--;
5724                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5725                                 if (sg != scb->sg_list
5726                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5727
5728                                         sg--;
5729                                         sglen = ahd_le32toh(sg->len);
5730                                         /*
5731                                          * Preserve High Address and SG_LIST
5732                                          * bits while setting the count to 1.
5733                                          */
5734                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5735                                         data_addr = ahd_le64toh(sg->addr)
5736                                                   + (sglen & AHD_SG_LEN_MASK)
5737                                                   - 1;
5738
5739                                         /*
5740                                          * Increment sg so it points to the
5741                                          * "next" sg.
5742                                          */
5743                                         sg++;
5744                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5745                                                                    sg);
5746                                 }
5747                         } else {
5748                                 struct ahd_dma_seg *sg;
5749
5750                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5751
5752                                 /*
5753                                  * The residual sg ptr points to the next S/G
5754                                  * to load so we must go back one.
5755                                  */
5756                                 sg--;
5757                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5758                                 if (sg != scb->sg_list
5759                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5760
5761                                         sg--;
5762                                         sglen = ahd_le32toh(sg->len);
5763                                         /*
5764                                          * Preserve High Address and SG_LIST
5765                                          * bits while setting the count to 1.
5766                                          */
5767                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5768                                         data_addr = ahd_le32toh(sg->addr)
5769                                                   + (sglen & AHD_SG_LEN_MASK)
5770                                                   - 1;
5771
5772                                         /*
5773                                          * Increment sg so it points to the
5774                                          * "next" sg.
5775                                          */
5776                                         sg++;
5777                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
5778                                                                   sg);
5779                                 }
5780                         }
5781                         /*
5782                          * Toggle the "oddness" of the transfer length
5783                          * to handle this mid-transfer ignore wide
5784                          * residue.  This ensures that the oddness is
5785                          * correct for subsequent data transfers.
5786                          */
5787                         ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
5788                             ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5789                             ^ SCB_XFERLEN_ODD);
5790
5791                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
5792                         ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
5793                         /*
5794                          * The FIFO's pointers will be updated if/when the
5795                          * sequencer re-enters a data phase.
5796                          */
5797                 }
5798         }
5799 }
5800
5801
5802 /*
5803  * Reinitialize the data pointers for the active transfer
5804  * based on its current residual.
5805  */
5806 static void
5807 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
5808 {
5809         struct           scb *scb;
5810         ahd_mode_state   saved_modes;
5811         u_int            scb_index;
5812         u_int            wait;
5813         uint32_t         sgptr;
5814         uint32_t         resid;
5815         uint64_t         dataptr;
5816
5817         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5818                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5819
5820         scb_index = ahd_get_scbptr(ahd);
5821         scb = ahd_lookup_scb(ahd, scb_index);
5822
5823         /*
5824          * Release and reacquire the FIFO so we
5825          * have a clean slate.
5826          */
5827         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5828         wait = 1000;
5829         while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5830                 ahd_delay(100);
5831         if (wait == 0) {
5832                 ahd_print_path(ahd, scb);
5833                 printk("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5834                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5835         }
5836         saved_modes = ahd_save_modes(ahd);
5837         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5838         ahd_outb(ahd, DFFSTAT,
5839                  ahd_inb(ahd, DFFSTAT)
5840                 | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5841
5842         /*
5843          * Determine initial values for data_addr and data_cnt
5844          * for resuming the data phase.
5845          */
5846         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5847         sgptr &= SG_PTR_MASK;
5848
5849         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5850               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5851               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5852
5853         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5854                 struct ahd_dma64_seg *sg;
5855
5856                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5857
5858                 /* The residual sg_ptr always points to the next sg */
5859                 sg--;
5860
5861                 dataptr = ahd_le64toh(sg->addr)
5862                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5863                         - resid;
5864                 ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5865         } else {
5866                 struct   ahd_dma_seg *sg;
5867
5868                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5869
5870                 /* The residual sg_ptr always points to the next sg */
5871                 sg--;
5872
5873                 dataptr = ahd_le32toh(sg->addr)
5874                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5875                         - resid;
5876                 ahd_outb(ahd, HADDR + 4,
5877                          (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5878         }
5879         ahd_outl(ahd, HADDR, dataptr);
5880         ahd_outb(ahd, HCNT + 2, resid >> 16);
5881         ahd_outb(ahd, HCNT + 1, resid >> 8);
5882         ahd_outb(ahd, HCNT, resid);
5883 }
5884
5885 /*
5886  * Handle the effects of issuing a bus device reset message.
5887  */
5888 static void
5889 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5890                     u_int lun, cam_status status, char *message,
5891                     int verbose_level)
5892 {
5893 #ifdef AHD_TARGET_MODE
5894         struct ahd_tmode_tstate* tstate;
5895 #endif
5896         int found;
5897
5898         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5899                                lun, SCB_LIST_NULL, devinfo->role,
5900                                status);
5901
5902 #ifdef AHD_TARGET_MODE
5903         /*
5904          * Send an immediate notify ccb to all target mord peripheral
5905          * drivers affected by this action.
5906          */
5907         tstate = ahd->enabled_targets[devinfo->our_scsiid];
5908         if (tstate != NULL) {
5909                 u_int cur_lun;
5910                 u_int max_lun;
5911
5912                 if (lun != CAM_LUN_WILDCARD) {
5913                         cur_lun = 0;
5914                         max_lun = AHD_NUM_LUNS - 1;
5915                 } else {
5916                         cur_lun = lun;
5917                         max_lun = lun;
5918                 }
5919                 for (;cur_lun <= max_lun; cur_lun++) {
5920                         struct ahd_tmode_lstate* lstate;
5921
5922                         lstate = tstate->enabled_luns[cur_lun];
5923                         if (lstate == NULL)
5924                                 continue;
5925
5926                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5927                                                MSG_BUS_DEV_RESET, /*arg*/0);
5928                         ahd_send_lstate_events(ahd, lstate);
5929                 }
5930         }
5931 #endif
5932
5933         /*
5934          * Go back to async/narrow transfers and renegotiate.
5935          */
5936         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5937                       AHD_TRANS_CUR, /*paused*/TRUE);
5938         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5939                          /*ppr_options*/0, AHD_TRANS_CUR,
5940                          /*paused*/TRUE);
5941
5942         if (status != CAM_SEL_TIMEOUT)
5943                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
5944                                CAM_LUN_WILDCARD, AC_SENT_BDR);
5945
5946         if (message != NULL && bootverbose)
5947                 printk("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5948                        message, devinfo->channel, devinfo->target, found);
5949 }
5950
5951 #ifdef AHD_TARGET_MODE
5952 static void
5953 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5954                        struct scb *scb)
5955 {
5956
5957         /*
5958          * To facilitate adding multiple messages together,
5959          * each routine should increment the index and len
5960          * variables instead of setting them explicitly.
5961          */
5962         ahd->msgout_index = 0;
5963         ahd->msgout_len = 0;
5964
5965         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
5966                 ahd_build_transfer_msg(ahd, devinfo);
5967         else
5968                 panic("ahd_intr: AWAITING target message with no message");
5969
5970         ahd->msgout_index = 0;
5971         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5972 }
5973 #endif
5974 /**************************** Initialization **********************************/
5975 static u_int
5976 ahd_sglist_size(struct ahd_softc *ahd)
5977 {
5978         bus_size_t list_size;
5979
5980         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
5981         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5982                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
5983         return (list_size);
5984 }
5985
5986 /*
5987  * Calculate the optimum S/G List allocation size.  S/G elements used
5988  * for a given transaction must be physically contiguous.  Assume the
5989  * OS will allocate full pages to us, so it doesn't make sense to request
5990  * less than a page.
5991  */
5992 static u_int
5993 ahd_sglist_allocsize(struct ahd_softc *ahd)
5994 {
5995         bus_size_t sg_list_increment;
5996         bus_size_t sg_list_size;
5997         bus_size_t max_list_size;
5998         bus_size_t best_list_size;
5999
6000         /* Start out with the minimum required for AHD_NSEG. */
6001         sg_list_increment = ahd_sglist_size(ahd);
6002         sg_list_size = sg_list_increment;
6003
6004         /* Get us as close as possible to a page in size. */
6005         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
6006                 sg_list_size += sg_list_increment;
6007
6008         /*
6009          * Try to reduce the amount of wastage by allocating
6010          * multiple pages.
6011          */
6012         best_list_size = sg_list_size;
6013         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
6014         if (max_list_size < 4 * PAGE_SIZE)
6015                 max_list_size = 4 * PAGE_SIZE;
6016         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
6017                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
6018         while ((sg_list_size + sg_list_increment) <= max_list_size
6019            &&  (sg_list_size % PAGE_SIZE) != 0) {
6020                 bus_size_t new_mod;
6021                 bus_size_t best_mod;
6022
6023                 sg_list_size += sg_list_increment;
6024                 new_mod = sg_list_size % PAGE_SIZE;
6025                 best_mod = best_list_size % PAGE_SIZE;
6026                 if (new_mod > best_mod || new_mod == 0) {
6027                         best_list_size = sg_list_size;
6028                 }
6029         }
6030         return (best_list_size);
6031 }
6032
6033 /*
6034  * Allocate a controller structure for a new device
6035  * and perform initial initializion.
6036  */
6037 struct ahd_softc *
6038 ahd_alloc(void *platform_arg, char *name)
6039 {
6040         struct  ahd_softc *ahd;
6041
6042         ahd = kzalloc(sizeof(*ahd), GFP_ATOMIC);
6043         if (!ahd) {
6044                 printk("aic7xxx: cannot malloc softc!\n");
6045                 kfree(name);
6046                 return NULL;
6047         }
6048
6049         ahd->seep_config = kmalloc(sizeof(*ahd->seep_config), GFP_ATOMIC);
6050         if (ahd->seep_config == NULL) {
6051                 kfree(ahd);
6052                 kfree(name);
6053                 return (NULL);
6054         }
6055         LIST_INIT(&ahd->pending_scbs);
6056         /* We don't know our unit number until the OSM sets it */
6057         ahd->name = name;
6058         ahd->unit = -1;
6059         ahd->description = NULL;
6060         ahd->bus_description = NULL;
6061         ahd->channel = 'A';
6062         ahd->chip = AHD_NONE;
6063         ahd->features = AHD_FENONE;
6064         ahd->bugs = AHD_BUGNONE;
6065         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
6066                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
6067         timer_setup(&ahd->stat_timer, ahd_stat_timer, 0);
6068         ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
6069         ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
6070         ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
6071         ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
6072         ahd->int_coalescing_stop_threshold =
6073             AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
6074
6075 #ifdef AHD_DEBUG
6076         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
6077                 printk("%s: scb size = 0x%x, hscb size = 0x%x\n",
6078                        ahd_name(ahd), (u_int)sizeof(struct scb),
6079                        (u_int)sizeof(struct hardware_scb));
6080         }
6081 #endif
6082         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
6083                 ahd_free(ahd);
6084                 ahd = NULL;
6085         }
6086         return (ahd);
6087 }
6088
6089 int
6090 ahd_softc_init(struct ahd_softc *ahd)
6091 {
6092
6093         ahd->unpause = 0;
6094         ahd->pause = PAUSE;
6095         return (0);
6096 }
6097
6098 void
6099 ahd_set_unit(struct ahd_softc *ahd, int unit)
6100 {
6101         ahd->unit = unit;
6102 }
6103
6104 void
6105 ahd_set_name(struct ahd_softc *ahd, char *name)
6106 {
6107         kfree(ahd->name);
6108         ahd->name = name;
6109 }
6110
6111 void
6112 ahd_free(struct ahd_softc *ahd)
6113 {
6114         int i;
6115
6116         switch (ahd->init_level) {
6117         default:
6118         case 5:
6119                 ahd_shutdown(ahd);
6120                 fallthrough;
6121         case 4:
6122                 ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
6123                                   ahd->shared_data_map.dmamap);
6124                 fallthrough;
6125         case 3:
6126                 ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
6127                                 ahd->shared_data_map.dmamap);
6128                 ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
6129                                    ahd->shared_data_map.dmamap);
6130                 fallthrough;
6131         case 2:
6132                 ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
6133                 break;
6134         case 1:
6135                 break;
6136         case 0:
6137                 break;
6138         }
6139
6140         ahd_platform_free(ahd);
6141         ahd_fini_scbdata(ahd);
6142         for (i = 0; i < AHD_NUM_TARGETS; i++) {
6143                 struct ahd_tmode_tstate *tstate;
6144
6145                 tstate = ahd->enabled_targets[i];
6146                 if (tstate != NULL) {
6147 #ifdef AHD_TARGET_MODE
6148                         int j;
6149
6150                         for (j = 0; j < AHD_NUM_LUNS; j++) {
6151                                 struct ahd_tmode_lstate *lstate;
6152
6153                                 lstate = tstate->enabled_luns[j];
6154                                 if (lstate != NULL) {
6155                                         xpt_free_path(lstate->path);
6156                                         kfree(lstate);
6157                                 }
6158                         }
6159 #endif
6160                         kfree(tstate);
6161                 }
6162         }
6163 #ifdef AHD_TARGET_MODE
6164         if (ahd->black_hole != NULL) {
6165                 xpt_free_path(ahd->black_hole->path);
6166                 kfree(ahd->black_hole);
6167         }
6168 #endif
6169         kfree(ahd->name);
6170         kfree(ahd->seep_config);
6171         kfree(ahd->saved_stack);
6172         kfree(ahd);
6173         return;
6174 }
6175
6176 static void
6177 ahd_shutdown(void *arg)
6178 {
6179         struct  ahd_softc *ahd;
6180
6181         ahd = (struct ahd_softc *)arg;
6182
6183         /*
6184          * Stop periodic timer callbacks.
6185          */
6186         del_timer_sync(&ahd->stat_timer);
6187
6188         /* This will reset most registers to 0, but not all */
6189         ahd_reset(ahd, /*reinit*/FALSE);
6190 }
6191
6192 /*
6193  * Reset the controller and record some information about it
6194  * that is only available just after a reset.  If "reinit" is
6195  * non-zero, this reset occurred after initial configuration
6196  * and the caller requests that the chip be fully reinitialized
6197  * to a runable state.  Chip interrupts are *not* enabled after
6198  * a reinitialization.  The caller must enable interrupts via
6199  * ahd_intr_enable().
6200  */
6201 int
6202 ahd_reset(struct ahd_softc *ahd, int reinit)
6203 {
6204         u_int    sxfrctl1;
6205         int      wait;
6206         uint32_t cmd;
6207
6208         /*
6209          * Preserve the value of the SXFRCTL1 register for all channels.
6210          * It contains settings that affect termination and we don't want
6211          * to disturb the integrity of the bus.
6212          */
6213         ahd_pause(ahd);
6214         ahd_update_modes(ahd);
6215         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6216         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
6217
6218         cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
6219         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6220                 uint32_t mod_cmd;
6221
6222                 /*
6223                  * A4 Razor #632
6224                  * During the assertion of CHIPRST, the chip
6225                  * does not disable its parity logic prior to
6226                  * the start of the reset.  This may cause a
6227                  * parity error to be detected and thus a
6228                  * spurious SERR or PERR assertion.  Disable
6229                  * PERR and SERR responses during the CHIPRST.
6230                  */
6231                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
6232                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6233                                      mod_cmd, /*bytes*/2);
6234         }
6235         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
6236
6237         /*
6238          * Ensure that the reset has finished.  We delay 1000us
6239          * prior to reading the register to make sure the chip
6240          * has sufficiently completed its reset to handle register
6241          * accesses.
6242          */
6243         wait = 1000;
6244         do {
6245                 ahd_delay(1000);
6246         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
6247
6248         if (wait == 0) {
6249                 printk("%s: WARNING - Failed chip reset!  "
6250                        "Trying to initialize anyway.\n", ahd_name(ahd));
6251         }
6252         ahd_outb(ahd, HCNTRL, ahd->pause);
6253
6254         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6255                 /*
6256                  * Clear any latched PCI error status and restore
6257                  * previous SERR and PERR response enables.
6258                  */
6259                 ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
6260                                      0xFF, /*bytes*/1);
6261                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6262                                      cmd, /*bytes*/2);
6263         }
6264
6265         /*
6266          * Mode should be SCSI after a chip reset, but lets
6267          * set it just to be safe.  We touch the MODE_PTR
6268          * register directly so as to bypass the lazy update
6269          * code in ahd_set_modes().
6270          */
6271         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6272         ahd_outb(ahd, MODE_PTR,
6273                  ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
6274
6275         /*
6276          * Restore SXFRCTL1.
6277          *
6278          * We must always initialize STPWEN to 1 before we
6279          * restore the saved values.  STPWEN is initialized
6280          * to a tri-state condition which can only be cleared
6281          * by turning it on.
6282          */
6283         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
6284         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
6285
6286         /* Determine chip configuration */
6287         ahd->features &= ~AHD_WIDE;
6288         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
6289                 ahd->features |= AHD_WIDE;
6290
6291         /*
6292          * If a recovery action has forced a chip reset,
6293          * re-initialize the chip to our liking.
6294          */
6295         if (reinit != 0)
6296                 ahd_chip_init(ahd);
6297
6298         return (0);
6299 }
6300
6301 /*
6302  * Determine the number of SCBs available on the controller
6303  */
6304 static int
6305 ahd_probe_scbs(struct ahd_softc *ahd) {
6306         int i;
6307
6308         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
6309                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
6310         for (i = 0; i < AHD_SCB_MAX; i++) {
6311                 int j;
6312
6313                 ahd_set_scbptr(ahd, i);
6314                 ahd_outw(ahd, SCB_BASE, i);
6315                 for (j = 2; j < 64; j++)
6316                         ahd_outb(ahd, SCB_BASE+j, 0);
6317                 /* Start out life as unallocated (needing an abort) */
6318                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
6319                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
6320                         break;
6321                 ahd_set_scbptr(ahd, 0);
6322                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
6323                         break;
6324         }
6325         return (i);
6326 }
6327
6328 static void
6329 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
6330 {
6331         dma_addr_t *baddr;
6332
6333         baddr = (dma_addr_t *)arg;
6334         *baddr = segs->ds_addr;
6335 }
6336
6337 static void
6338 ahd_initialize_hscbs(struct ahd_softc *ahd)
6339 {
6340         int i;
6341
6342         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
6343                 ahd_set_scbptr(ahd, i);
6344
6345                 /* Clear the control byte. */
6346                 ahd_outb(ahd, SCB_CONTROL, 0);
6347
6348                 /* Set the next pointer */
6349                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
6350         }
6351 }
6352
6353 static int
6354 ahd_init_scbdata(struct ahd_softc *ahd)
6355 {
6356         struct  scb_data *scb_data;
6357         int     i;
6358
6359         scb_data = &ahd->scb_data;
6360         TAILQ_INIT(&scb_data->free_scbs);
6361         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
6362                 LIST_INIT(&scb_data->free_scb_lists[i]);
6363         LIST_INIT(&scb_data->any_dev_free_scb_list);
6364         SLIST_INIT(&scb_data->hscb_maps);
6365         SLIST_INIT(&scb_data->sg_maps);
6366         SLIST_INIT(&scb_data->sense_maps);
6367
6368         /* Determine the number of hardware SCBs and initialize them */
6369         scb_data->maxhscbs = ahd_probe_scbs(ahd);
6370         if (scb_data->maxhscbs == 0) {
6371                 printk("%s: No SCB space found\n", ahd_name(ahd));
6372                 return (ENXIO);
6373         }
6374
6375         ahd_initialize_hscbs(ahd);
6376
6377         /*
6378          * Create our DMA tags.  These tags define the kinds of device
6379          * accessible memory allocations and memory mappings we will
6380          * need to perform during normal operation.
6381          *
6382          * Unless we need to further restrict the allocation, we rely
6383          * on the restrictions of the parent dmat, hence the common
6384          * use of MAXADDR and MAXSIZE.
6385          */
6386
6387         /* DMA tag for our hardware scb structures */
6388         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6389                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6390                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6391                                /*highaddr*/BUS_SPACE_MAXADDR,
6392                                /*filter*/NULL, /*filterarg*/NULL,
6393                                PAGE_SIZE, /*nsegments*/1,
6394                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6395                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
6396                 goto error_exit;
6397         }
6398
6399         scb_data->init_level++;
6400
6401         /* DMA tag for our S/G structures. */
6402         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
6403                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6404                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6405                                /*highaddr*/BUS_SPACE_MAXADDR,
6406                                /*filter*/NULL, /*filterarg*/NULL,
6407                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
6408                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6409                                /*flags*/0, &scb_data->sg_dmat) != 0) {
6410                 goto error_exit;
6411         }
6412 #ifdef AHD_DEBUG
6413         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
6414                 printk("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
6415                        ahd_sglist_allocsize(ahd));
6416 #endif
6417
6418         scb_data->init_level++;
6419
6420         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
6421         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6422                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6423                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6424                                /*highaddr*/BUS_SPACE_MAXADDR,
6425                                /*filter*/NULL, /*filterarg*/NULL,
6426                                PAGE_SIZE, /*nsegments*/1,
6427                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6428                                /*flags*/0, &scb_data->sense_dmat) != 0) {
6429                 goto error_exit;
6430         }
6431
6432         scb_data->init_level++;
6433
6434         /* Perform initial CCB allocation */
6435         ahd_alloc_scbs(ahd);
6436
6437         if (scb_data->numscbs == 0) {
6438                 printk("%s: ahd_init_scbdata - "
6439                        "Unable to allocate initial scbs\n",
6440                        ahd_name(ahd));
6441                 goto error_exit;
6442         }
6443
6444         /*
6445          * Note that we were successful
6446          */
6447         return (0);
6448
6449 error_exit:
6450
6451         return (ENOMEM);
6452 }
6453
6454 static struct scb *
6455 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
6456 {
6457         struct scb *scb;
6458
6459         /*
6460          * Look on the pending list.
6461          */
6462         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
6463                 if (SCB_GET_TAG(scb) == tag)
6464                         return (scb);
6465         }
6466
6467         /*
6468          * Then on all of the collision free lists.
6469          */
6470         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6471                 struct scb *list_scb;
6472
6473                 list_scb = scb;
6474                 do {
6475                         if (SCB_GET_TAG(list_scb) == tag)
6476                                 return (list_scb);
6477                         list_scb = LIST_NEXT(list_scb, collision_links);
6478                 } while (list_scb);
6479         }
6480
6481         /*
6482          * And finally on the generic free list.
6483          */
6484         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
6485                 if (SCB_GET_TAG(scb) == tag)
6486                         return (scb);
6487         }
6488
6489         return (NULL);
6490 }
6491
6492 static void
6493 ahd_fini_scbdata(struct ahd_softc *ahd)
6494 {
6495         struct scb_data *scb_data;
6496
6497         scb_data = &ahd->scb_data;
6498         if (scb_data == NULL)
6499                 return;
6500
6501         switch (scb_data->init_level) {
6502         default:
6503         case 7:
6504         {
6505                 struct map_node *sns_map;
6506
6507                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
6508                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
6509                         ahd_dmamap_unload(ahd, scb_data->sense_dmat,
6510                                           sns_map->dmamap);
6511                         ahd_dmamem_free(ahd, scb_data->sense_dmat,
6512                                         sns_map->vaddr, sns_map->dmamap);
6513                         kfree(sns_map);
6514                 }
6515                 ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
6516         }
6517                 fallthrough;
6518         case 6:
6519         {
6520                 struct map_node *sg_map;
6521
6522                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
6523                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
6524                         ahd_dmamap_unload(ahd, scb_data->sg_dmat,
6525                                           sg_map->dmamap);
6526                         ahd_dmamem_free(ahd, scb_data->sg_dmat,
6527                                         sg_map->vaddr, sg_map->dmamap);
6528                         kfree(sg_map);
6529                 }
6530                 ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
6531         }
6532                 fallthrough;
6533         case 5:
6534         {
6535                 struct map_node *hscb_map;
6536
6537                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
6538                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
6539                         ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
6540                                           hscb_map->dmamap);
6541                         ahd_dmamem_free(ahd, scb_data->hscb_dmat,
6542                                         hscb_map->vaddr, hscb_map->dmamap);
6543                         kfree(hscb_map);
6544                 }
6545                 ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
6546         }
6547                 fallthrough;
6548         case 4:
6549         case 3:
6550         case 2:
6551         case 1:
6552         case 0:
6553                 break;
6554         }
6555 }
6556
6557 /*
6558  * DSP filter Bypass must be enabled until the first selection
6559  * after a change in bus mode (Razor #491 and #493).
6560  */
6561 static void
6562 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
6563 {
6564         ahd_mode_state saved_modes;
6565
6566         saved_modes = ahd_save_modes(ahd);
6567         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6568         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
6569                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
6570         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
6571 #ifdef AHD_DEBUG
6572         if ((ahd_debug & AHD_SHOW_MISC) != 0)
6573                 printk("%s: Setting up iocell workaround\n", ahd_name(ahd));
6574 #endif
6575         ahd_restore_modes(ahd, saved_modes);
6576         ahd->flags &= ~AHD_HAD_FIRST_SEL;
6577 }
6578
6579 static void
6580 ahd_iocell_first_selection(struct ahd_softc *ahd)
6581 {
6582         ahd_mode_state  saved_modes;
6583         u_int           sblkctl;
6584
6585         if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
6586                 return;
6587         saved_modes = ahd_save_modes(ahd);
6588         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6589         sblkctl = ahd_inb(ahd, SBLKCTL);
6590         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6591 #ifdef AHD_DEBUG
6592         if ((ahd_debug & AHD_SHOW_MISC) != 0)
6593                 printk("%s: iocell first selection\n", ahd_name(ahd));
6594 #endif
6595         if ((sblkctl & ENAB40) != 0) {
6596                 ahd_outb(ahd, DSPDATACTL,
6597                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
6598 #ifdef AHD_DEBUG
6599                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6600                         printk("%s: BYPASS now disabled\n", ahd_name(ahd));
6601 #endif
6602         }
6603         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
6604         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6605         ahd_restore_modes(ahd, saved_modes);
6606         ahd->flags |= AHD_HAD_FIRST_SEL;
6607 }
6608
6609 /*************************** SCB Management ***********************************/
6610 static void
6611 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
6612 {
6613         struct  scb_list *free_list;
6614         struct  scb_tailq *free_tailq;
6615         struct  scb *first_scb;
6616
6617         scb->flags |= SCB_ON_COL_LIST;
6618         AHD_SET_SCB_COL_IDX(scb, col_idx);
6619         free_list = &ahd->scb_data.free_scb_lists[col_idx];
6620         free_tailq = &ahd->scb_data.free_scbs;
6621         first_scb = LIST_FIRST(free_list);
6622         if (first_scb != NULL) {
6623                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
6624         } else {
6625                 LIST_INSERT_HEAD(free_list, scb, collision_links);
6626                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
6627         }
6628 }
6629
6630 static void
6631 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
6632 {
6633         struct  scb_list *free_list;
6634         struct  scb_tailq *free_tailq;
6635         struct  scb *first_scb;
6636         u_int   col_idx;
6637
6638         scb->flags &= ~SCB_ON_COL_LIST;
6639         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
6640         free_list = &ahd->scb_data.free_scb_lists[col_idx];
6641         free_tailq = &ahd->scb_data.free_scbs;
6642         first_scb = LIST_FIRST(free_list);
6643         if (first_scb == scb) {
6644                 struct scb *next_scb;
6645
6646                 /*
6647                  * Maintain order in the collision free
6648                  * lists for fairness if this device has
6649                  * other colliding tags active.
6650                  */
6651                 next_scb = LIST_NEXT(scb, collision_links);
6652                 if (next_scb != NULL) {
6653                         TAILQ_INSERT_AFTER(free_tailq, scb,
6654                                            next_scb, links.tqe);
6655                 }
6656                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
6657         }
6658         LIST_REMOVE(scb, collision_links);
6659 }
6660
6661 /*
6662  * Get a free scb. If there are none, see if we can allocate a new SCB.
6663  */
6664 struct scb *
6665 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
6666 {
6667         struct scb *scb;
6668         int tries;
6669
6670         tries = 0;
6671 look_again:
6672         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6673                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
6674                         ahd_rem_col_list(ahd, scb);
6675                         goto found;
6676                 }
6677         }
6678         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
6679
6680                 if (tries++ != 0)
6681                         return (NULL);
6682                 ahd_alloc_scbs(ahd);
6683                 goto look_again;
6684         }
6685         LIST_REMOVE(scb, links.le);
6686         if (col_idx != AHD_NEVER_COL_IDX
6687          && (scb->col_scb != NULL)
6688          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
6689                 LIST_REMOVE(scb->col_scb, links.le);
6690                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
6691         }
6692 found:
6693         scb->flags |= SCB_ACTIVE;
6694         return (scb);
6695 }
6696
6697 /*
6698  * Return an SCB resource to the free list.
6699  */
6700 void
6701 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
6702 {
6703         /* Clean up for the next user */
6704         scb->flags = SCB_FLAG_NONE;
6705         scb->hscb->control = 0;
6706         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
6707
6708         if (scb->col_scb == NULL) {
6709
6710                 /*
6711                  * No collision possible.  Just free normally.
6712                  */
6713                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6714                                  scb, links.le);
6715         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
6716
6717                 /*
6718                  * The SCB we might have collided with is on
6719                  * a free collision list.  Put both SCBs on
6720                  * the generic list.
6721                  */
6722                 ahd_rem_col_list(ahd, scb->col_scb);
6723                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6724                                  scb, links.le);
6725                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6726                                  scb->col_scb, links.le);
6727         } else if ((scb->col_scb->flags
6728                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
6729                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
6730
6731                 /*
6732                  * The SCB we might collide with on the next allocation
6733                  * is still active in a non-packetized, tagged, context.
6734                  * Put us on the SCB collision list.
6735                  */
6736                 ahd_add_col_list(ahd, scb,
6737                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
6738         } else {
6739                 /*
6740                  * The SCB we might collide with on the next allocation
6741                  * is either active in a packetized context, or free.
6742                  * Since we can't collide, put this SCB on the generic
6743                  * free list.
6744                  */
6745                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6746                                  scb, links.le);
6747         }
6748
6749         ahd_platform_scb_free(ahd, scb);
6750 }
6751
6752 static void
6753 ahd_alloc_scbs(struct ahd_softc *ahd)
6754 {
6755         struct scb_data *scb_data;
6756         struct scb      *next_scb;
6757         struct hardware_scb *hscb;
6758         struct map_node *hscb_map;
6759         struct map_node *sg_map;
6760         struct map_node *sense_map;
6761         uint8_t         *segs;
6762         uint8_t         *sense_data;
6763         dma_addr_t       hscb_busaddr;
6764         dma_addr_t       sg_busaddr;
6765         dma_addr_t       sense_busaddr;
6766         int              newcount;
6767         int              i;
6768
6769         scb_data = &ahd->scb_data;
6770         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
6771                 /* Can't allocate any more */
6772                 return;
6773
6774         if (scb_data->scbs_left != 0) {
6775                 int offset;
6776
6777                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
6778                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
6779                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
6780                 hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
6781         } else {
6782                 hscb_map = kmalloc(sizeof(*hscb_map), GFP_ATOMIC);
6783
6784                 if (hscb_map == NULL)
6785                         return;
6786
6787                 /* Allocate the next batch of hardware SCBs */
6788                 if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
6789                                      (void **)&hscb_map->vaddr,
6790                                      BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
6791                         kfree(hscb_map);
6792                         return;
6793                 }
6794
6795                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6796
6797                 ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6798                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6799                                 &hscb_map->physaddr, /*flags*/0);
6800
6801                 hscb = (struct hardware_scb *)hscb_map->vaddr;
6802                 hscb_busaddr = hscb_map->physaddr;
6803                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6804         }
6805
6806         if (scb_data->sgs_left != 0) {
6807                 int offset;
6808
6809                 offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6810                        - scb_data->sgs_left) * ahd_sglist_size(ahd);
6811                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
6812                 segs = sg_map->vaddr + offset;
6813                 sg_busaddr = sg_map->physaddr + offset;
6814         } else {
6815                 sg_map = kmalloc(sizeof(*sg_map), GFP_ATOMIC);
6816
6817                 if (sg_map == NULL)
6818                         return;
6819
6820                 /* Allocate the next batch of S/G lists */
6821                 if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
6822                                      (void **)&sg_map->vaddr,
6823                                      BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
6824                         kfree(sg_map);
6825                         return;
6826                 }
6827
6828                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6829
6830                 ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6831                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
6832                                 ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
6833
6834                 segs = sg_map->vaddr;
6835                 sg_busaddr = sg_map->physaddr;
6836                 scb_data->sgs_left =
6837                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6838 #ifdef AHD_DEBUG
6839                 if (ahd_debug & AHD_SHOW_MEMORY)
6840                         printk("Mapped SG data\n");
6841 #endif
6842         }
6843
6844         if (scb_data->sense_left != 0) {
6845                 int offset;
6846
6847                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6848                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
6849                 sense_data = sense_map->vaddr + offset;
6850                 sense_busaddr = sense_map->physaddr + offset;
6851         } else {
6852                 sense_map = kmalloc(sizeof(*sense_map), GFP_ATOMIC);
6853
6854                 if (sense_map == NULL)
6855                         return;
6856
6857                 /* Allocate the next batch of sense buffers */
6858                 if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
6859                                      (void **)&sense_map->vaddr,
6860                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6861                         kfree(sense_map);
6862                         return;
6863                 }
6864
6865                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6866
6867                 ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6868                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6869                                 &sense_map->physaddr, /*flags*/0);
6870
6871                 sense_data = sense_map->vaddr;
6872                 sense_busaddr = sense_map->physaddr;
6873                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6874 #ifdef AHD_DEBUG
6875                 if (ahd_debug & AHD_SHOW_MEMORY)
6876                         printk("Mapped sense data\n");
6877 #endif
6878         }
6879
6880         newcount = min(scb_data->sense_left, scb_data->scbs_left);
6881         newcount = min(newcount, scb_data->sgs_left);
6882         newcount = min(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6883         for (i = 0; i < newcount; i++) {
6884                 struct scb_platform_data *pdata;
6885                 u_int col_tag;
6886
6887                 next_scb = kmalloc(sizeof(*next_scb), GFP_ATOMIC);
6888                 if (next_scb == NULL)
6889                         break;
6890
6891                 pdata = kmalloc(sizeof(*pdata), GFP_ATOMIC);
6892                 if (pdata == NULL) {
6893                         kfree(next_scb);
6894                         break;
6895                 }
6896                 next_scb->platform_data = pdata;
6897                 next_scb->hscb_map = hscb_map;
6898                 next_scb->sg_map = sg_map;
6899                 next_scb->sense_map = sense_map;
6900                 next_scb->sg_list = segs;
6901                 next_scb->sense_data = sense_data;
6902                 next_scb->sense_busaddr = sense_busaddr;
6903                 memset(hscb, 0, sizeof(*hscb));
6904                 next_scb->hscb = hscb;
6905                 hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
6906
6907                 /*
6908                  * The sequencer always starts with the second entry.
6909                  * The first entry is embedded in the scb.
6910                  */
6911                 next_scb->sg_list_busaddr = sg_busaddr;
6912                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6913                         next_scb->sg_list_busaddr
6914                             += sizeof(struct ahd_dma64_seg);
6915                 else
6916                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6917                 next_scb->ahd_softc = ahd;
6918                 next_scb->flags = SCB_FLAG_NONE;
6919                 next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
6920                 col_tag = scb_data->numscbs ^ 0x100;
6921                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6922                 if (next_scb->col_scb != NULL)
6923                         next_scb->col_scb->col_scb = next_scb;
6924                 ahd_free_scb(ahd, next_scb);
6925                 hscb++;
6926                 hscb_busaddr += sizeof(*hscb);
6927                 segs += ahd_sglist_size(ahd);
6928                 sg_busaddr += ahd_sglist_size(ahd);
6929                 sense_data += AHD_SENSE_BUFSIZE;
6930                 sense_busaddr += AHD_SENSE_BUFSIZE;
6931                 scb_data->numscbs++;
6932                 scb_data->sense_left--;
6933                 scb_data->scbs_left--;
6934                 scb_data->sgs_left--;
6935         }
6936 }
6937
6938 void
6939 ahd_controller_info(struct ahd_softc *ahd, char *buf)
6940 {
6941         const char *speed;
6942         const char *type;
6943         int len;
6944
6945         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
6946         buf += len;
6947
6948         speed = "Ultra320 ";
6949         if ((ahd->features & AHD_WIDE) != 0) {
6950                 type = "Wide ";
6951         } else {
6952                 type = "Single ";
6953         }
6954         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
6955                       speed, type, ahd->channel, ahd->our_id);
6956         buf += len;
6957
6958         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
6959                 ahd->scb_data.maxhscbs);
6960 }
6961
6962 static const char *channel_strings[] = {
6963         "Primary Low",
6964         "Primary High",
6965         "Secondary Low",
6966         "Secondary High"
6967 };
6968
6969 static const char *termstat_strings[] = {
6970         "Terminated Correctly",
6971         "Over Terminated",
6972         "Under Terminated",
6973         "Not Configured"
6974 };
6975
6976 /***************************** Timer Facilities *******************************/
6977 static void
6978 ahd_timer_reset(struct timer_list *timer, int usec)
6979 {
6980         del_timer(timer);
6981         timer->expires = jiffies + (usec * HZ)/1000000;
6982         add_timer(timer);
6983 }
6984
6985 /*
6986  * Start the board, ready for normal operation
6987  */
6988 int
6989 ahd_init(struct ahd_softc *ahd)
6990 {
6991         uint8_t         *next_vaddr;
6992         dma_addr_t       next_baddr;
6993         size_t           driver_data_size;
6994         int              i;
6995         int              error;
6996         u_int            warn_user;
6997         uint8_t          current_sensing;
6998         uint8_t          fstat;
6999
7000         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7001
7002         ahd->stack_size = ahd_probe_stack_size(ahd);
7003         ahd->saved_stack = kmalloc_array(ahd->stack_size, sizeof(uint16_t),
7004                                          GFP_ATOMIC);
7005         if (ahd->saved_stack == NULL)
7006                 return (ENOMEM);
7007
7008         /*
7009          * Verify that the compiler hasn't over-aggressively
7010          * padded important structures.
7011          */
7012         if (sizeof(struct hardware_scb) != 64)
7013                 panic("Hardware SCB size is incorrect");
7014
7015 #ifdef AHD_DEBUG
7016         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
7017                 ahd->flags |= AHD_SEQUENCER_DEBUG;
7018 #endif
7019
7020         /*
7021          * Default to allowing initiator operations.
7022          */
7023         ahd->flags |= AHD_INITIATORROLE;
7024
7025         /*
7026          * Only allow target mode features if this unit has them enabled.
7027          */
7028         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
7029                 ahd->features &= ~AHD_TARGETMODE;
7030
7031         ahd->init_level++;
7032
7033         /*
7034          * DMA tag for our command fifos and other data in system memory
7035          * the card's sequencer must be able to access.  For initiator
7036          * roles, we need to allocate space for the qoutfifo.  When providing
7037          * for the target mode role, we must additionally provide space for
7038          * the incoming target command fifo.
7039          */
7040         driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
7041                          + sizeof(struct hardware_scb);
7042         if ((ahd->features & AHD_TARGETMODE) != 0)
7043                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7044         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
7045                 driver_data_size += PKT_OVERRUN_BUFSIZE;
7046         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
7047                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
7048                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
7049                                /*highaddr*/BUS_SPACE_MAXADDR,
7050                                /*filter*/NULL, /*filterarg*/NULL,
7051                                driver_data_size,
7052                                /*nsegments*/1,
7053                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
7054                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
7055                 return (ENOMEM);
7056         }
7057
7058         ahd->init_level++;
7059
7060         /* Allocation of driver data */
7061         if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
7062                              (void **)&ahd->shared_data_map.vaddr,
7063                              BUS_DMA_NOWAIT,
7064                              &ahd->shared_data_map.dmamap) != 0) {
7065                 return (ENOMEM);
7066         }
7067
7068         ahd->init_level++;
7069
7070         /* And permanently map it in */
7071         ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
7072                         ahd->shared_data_map.vaddr, driver_data_size,
7073                         ahd_dmamap_cb, &ahd->shared_data_map.physaddr,
7074                         /*flags*/0);
7075         ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
7076         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
7077         next_baddr = ahd->shared_data_map.physaddr
7078                    + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
7079         if ((ahd->features & AHD_TARGETMODE) != 0) {
7080                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
7081                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7082                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7083         }
7084
7085         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
7086                 ahd->overrun_buf = next_vaddr;
7087                 next_vaddr += PKT_OVERRUN_BUFSIZE;
7088                 next_baddr += PKT_OVERRUN_BUFSIZE;
7089         }
7090
7091         /*
7092          * We need one SCB to serve as the "next SCB".  Since the
7093          * tag identifier in this SCB will never be used, there is
7094          * no point in using a valid HSCB tag from an SCB pulled from
7095          * the standard free pool.  So, we allocate this "sentinel"
7096          * specially from the DMA safe memory chunk used for the QOUTFIFO.
7097          */
7098         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
7099         ahd->next_queued_hscb_map = &ahd->shared_data_map;
7100         ahd->next_queued_hscb->hscb_busaddr = ahd_htole32(next_baddr);
7101
7102         ahd->init_level++;
7103
7104         /* Allocate SCB data now that buffer_dmat is initialized */
7105         if (ahd_init_scbdata(ahd) != 0)
7106                 return (ENOMEM);
7107
7108         if ((ahd->flags & AHD_INITIATORROLE) == 0)
7109                 ahd->flags &= ~AHD_RESET_BUS_A;
7110
7111         /*
7112          * Before committing these settings to the chip, give
7113          * the OSM one last chance to modify our configuration.
7114          */
7115         ahd_platform_init(ahd);
7116
7117         /* Bring up the chip. */
7118         ahd_chip_init(ahd);
7119
7120         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7121
7122         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
7123                 goto init_done;
7124
7125         /*
7126          * Verify termination based on current draw and
7127          * warn user if the bus is over/under terminated.
7128          */
7129         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
7130                                    CURSENSE_ENB);
7131         if (error != 0) {
7132                 printk("%s: current sensing timeout 1\n", ahd_name(ahd));
7133                 goto init_done;
7134         }
7135         for (i = 20, fstat = FLX_FSTAT_BUSY;
7136              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
7137                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
7138                 if (error != 0) {
7139                         printk("%s: current sensing timeout 2\n",
7140                                ahd_name(ahd));
7141                         goto init_done;
7142                 }
7143         }
7144         if (i == 0) {
7145                 printk("%s: Timedout during current-sensing test\n",
7146                        ahd_name(ahd));
7147                 goto init_done;
7148         }
7149
7150         /* Latch Current Sensing status. */
7151         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
7152         if (error != 0) {
7153                 printk("%s: current sensing timeout 3\n", ahd_name(ahd));
7154                 goto init_done;
7155         }
7156
7157         /* Diable current sensing. */
7158         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
7159
7160 #ifdef AHD_DEBUG
7161         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
7162                 printk("%s: current_sensing == 0x%x\n",
7163                        ahd_name(ahd), current_sensing);
7164         }
7165 #endif
7166         warn_user = 0;
7167         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
7168                 u_int term_stat;
7169
7170                 term_stat = (current_sensing & FLX_CSTAT_MASK);
7171                 switch (term_stat) {
7172                 case FLX_CSTAT_OVER:
7173                 case FLX_CSTAT_UNDER:
7174                         warn_user++;
7175                         fallthrough;
7176                 case FLX_CSTAT_INVALID:
7177                 case FLX_CSTAT_OKAY:
7178                         if (warn_user == 0 && bootverbose == 0)
7179                                 break;
7180                         printk("%s: %s Channel %s\n", ahd_name(ahd),
7181                                channel_strings[i], termstat_strings[term_stat]);
7182                         break;
7183                 }
7184         }
7185         if (warn_user) {
7186                 printk("%s: WARNING. Termination is not configured correctly.\n"
7187                        "%s: WARNING. SCSI bus operations may FAIL.\n",
7188                        ahd_name(ahd), ahd_name(ahd));
7189         }
7190 init_done:
7191         ahd_restart(ahd);
7192         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
7193         return (0);
7194 }
7195
7196 /*
7197  * (Re)initialize chip state after a chip reset.
7198  */
7199 static void
7200 ahd_chip_init(struct ahd_softc *ahd)
7201 {
7202         uint32_t busaddr;
7203         u_int    sxfrctl1;
7204         u_int    scsiseq_template;
7205         u_int    wait;
7206         u_int    i;
7207         u_int    target;
7208
7209         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7210         /*
7211          * Take the LED out of diagnostic mode
7212          */
7213         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
7214
7215         /*
7216          * Return HS_MAILBOX to its default value.
7217          */
7218         ahd->hs_mailbox = 0;
7219         ahd_outb(ahd, HS_MAILBOX, 0);
7220
7221         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
7222         ahd_outb(ahd, IOWNID, ahd->our_id);
7223         ahd_outb(ahd, TOWNID, ahd->our_id);
7224         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
7225         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
7226         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
7227          && (ahd->seltime != STIMESEL_MIN)) {
7228                 /*
7229                  * The selection timer duration is twice as long
7230                  * as it should be.  Halve it by adding "1" to
7231                  * the user specified setting.
7232                  */
7233                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
7234         } else {
7235                 sxfrctl1 |= ahd->seltime;
7236         }
7237
7238         ahd_outb(ahd, SXFRCTL0, DFON);
7239         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
7240         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
7241
7242         /*
7243          * Now that termination is set, wait for up
7244          * to 500ms for our transceivers to settle.  If
7245          * the adapter does not have a cable attached,
7246          * the transceivers may never settle, so don't
7247          * complain if we fail here.
7248          */
7249         for (wait = 10000;
7250              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
7251              wait--)
7252                 ahd_delay(100);
7253
7254         /* Clear any false bus resets due to the transceivers settling */
7255         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
7256         ahd_outb(ahd, CLRINT, CLRSCSIINT);
7257
7258         /* Initialize mode specific S/G state. */
7259         for (i = 0; i < 2; i++) {
7260                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
7261                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
7262                 ahd_outb(ahd, SG_STATE, 0);
7263                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
7264                 ahd_outb(ahd, SEQIMODE,
7265                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
7266                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
7267         }
7268
7269         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
7270         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
7271         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
7272         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
7273         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
7274         if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
7275                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
7276         } else {
7277                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
7278         }
7279         ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
7280         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
7281                 /*
7282                  * Do not issue a target abort when a split completion
7283                  * error occurs.  Let our PCIX interrupt handler deal
7284                  * with it instead. H2A4 Razor #625
7285                  */
7286                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
7287
7288         if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
7289                 ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
7290
7291         /*
7292          * Tweak IOCELL settings.
7293          */
7294         if ((ahd->flags & AHD_HP_BOARD) != 0) {
7295                 for (i = 0; i < NUMDSPS; i++) {
7296                         ahd_outb(ahd, DSPSELECT, i);
7297                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
7298                 }
7299 #ifdef AHD_DEBUG
7300                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7301                         printk("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
7302                                WRTBIASCTL_HP_DEFAULT);
7303 #endif
7304         }
7305         ahd_setup_iocell_workaround(ahd);
7306
7307         /*
7308          * Enable LQI Manager interrupts.
7309          */
7310         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
7311                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
7312                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
7313         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
7314         /*
7315          * We choose to have the sequencer catch LQOPHCHGINPKT errors
7316          * manually for the command phase at the start of a packetized
7317          * selection case.  ENLQOBUSFREE should be made redundant by
7318          * the BUSFREE interrupt, but it seems that some LQOBUSFREE
7319          * events fail to assert the BUSFREE interrupt so we must
7320          * also enable LQOBUSFREE interrupts.
7321          */
7322         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
7323
7324         /*
7325          * Setup sequencer interrupt handlers.
7326          */
7327         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
7328         ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
7329
7330         /*
7331          * Setup SCB Offset registers.
7332          */
7333         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7334                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
7335                          pkt_long_lun));
7336         } else {
7337                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
7338         }
7339         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
7340         ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
7341         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
7342         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
7343                                        shared_data.idata.cdb));
7344         ahd_outb(ahd, QNEXTPTR,
7345                  offsetof(struct hardware_scb, next_hscb_busaddr));
7346         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
7347         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
7348         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7349                 ahd_outb(ahd, LUNLEN,
7350                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
7351         } else {
7352                 ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
7353         }
7354         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
7355         ahd_outb(ahd, MAXCMD, 0xFF);
7356         ahd_outb(ahd, SCBAUTOPTR,
7357                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
7358
7359         /* We haven't been enabled for target mode yet. */
7360         ahd_outb(ahd, MULTARGID, 0);
7361         ahd_outb(ahd, MULTARGID + 1, 0);
7362
7363         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7364         /* Initialize the negotiation table. */
7365         if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
7366                 /*
7367                  * Clear the spare bytes in the neg table to avoid
7368                  * spurious parity errors.
7369                  */
7370                 for (target = 0; target < AHD_NUM_TARGETS; target++) {
7371                         ahd_outb(ahd, NEGOADDR, target);
7372                         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
7373                         for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
7374                                 ahd_outb(ahd, ANNEXDAT, 0);
7375                 }
7376         }
7377         for (target = 0; target < AHD_NUM_TARGETS; target++) {
7378                 struct   ahd_devinfo devinfo;
7379                 struct   ahd_initiator_tinfo *tinfo;
7380                 struct   ahd_tmode_tstate *tstate;
7381
7382                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7383                                             target, &tstate);
7384                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7385                                     target, CAM_LUN_WILDCARD,
7386                                     'A', ROLE_INITIATOR);
7387                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
7388         }
7389
7390         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
7391         ahd_outb(ahd, CLRINT, CLRSCSIINT);
7392
7393 #ifdef NEEDS_MORE_TESTING
7394         /*
7395          * Always enable abort on incoming L_Qs if this feature is
7396          * supported.  We use this to catch invalid SCB references.
7397          */
7398         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
7399                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
7400         else
7401 #endif
7402                 ahd_outb(ahd, LQCTL1, 0);
7403
7404         /* All of our queues are empty */
7405         ahd->qoutfifonext = 0;
7406         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
7407         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
7408         for (i = 0; i < AHD_QOUT_SIZE; i++)
7409                 ahd->qoutfifo[i].valid_tag = 0;
7410         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
7411
7412         ahd->qinfifonext = 0;
7413         for (i = 0; i < AHD_QIN_SIZE; i++)
7414                 ahd->qinfifo[i] = SCB_LIST_NULL;
7415
7416         if ((ahd->features & AHD_TARGETMODE) != 0) {
7417                 /* All target command blocks start out invalid. */
7418                 for (i = 0; i < AHD_TMODE_CMDS; i++)
7419                         ahd->targetcmds[i].cmd_valid = 0;
7420                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
7421                 ahd->tqinfifonext = 1;
7422                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
7423                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
7424         }
7425
7426         /* Initialize Scratch Ram. */
7427         ahd_outb(ahd, SEQ_FLAGS, 0);
7428         ahd_outb(ahd, SEQ_FLAGS2, 0);
7429
7430         /* We don't have any waiting selections */
7431         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
7432         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
7433         ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
7434         ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
7435         for (i = 0; i < AHD_NUM_TARGETS; i++)
7436                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
7437
7438         /*
7439          * Nobody is waiting to be DMAed into the QOUTFIFO.
7440          */
7441         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
7442         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
7443         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
7444         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
7445         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
7446
7447         /*
7448          * The Freeze Count is 0.
7449          */
7450         ahd->qfreeze_cnt = 0;
7451         ahd_outw(ahd, QFREEZE_COUNT, 0);
7452         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
7453
7454         /*
7455          * Tell the sequencer where it can find our arrays in memory.
7456          */
7457         busaddr = ahd->shared_data_map.physaddr;
7458         ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
7459         ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
7460
7461         /*
7462          * Setup the allowed SCSI Sequences based on operational mode.
7463          * If we are a target, we'll enable select in operations once
7464          * we've had a lun enabled.
7465          */
7466         scsiseq_template = ENAUTOATNP;
7467         if ((ahd->flags & AHD_INITIATORROLE) != 0)
7468                 scsiseq_template |= ENRSELI;
7469         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
7470
7471         /* There are no busy SCBs yet. */
7472         for (target = 0; target < AHD_NUM_TARGETS; target++) {
7473                 int lun;
7474
7475                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
7476                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
7477         }
7478
7479         /*
7480          * Initialize the group code to command length table.
7481          * Vendor Unique codes are set to 0 so we only capture
7482          * the first byte of the cdb.  These can be overridden
7483          * when target mode is enabled.
7484          */
7485         ahd_outb(ahd, CMDSIZE_TABLE, 5);
7486         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
7487         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
7488         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
7489         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
7490         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
7491         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
7492         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
7493
7494         /* Tell the sequencer of our initial queue positions */
7495         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7496         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
7497         ahd->qinfifonext = 0;
7498         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7499         ahd_set_hescb_qoff(ahd, 0);
7500         ahd_set_snscb_qoff(ahd, 0);
7501         ahd_set_sescb_qoff(ahd, 0);
7502         ahd_set_sdscb_qoff(ahd, 0);
7503
7504         /*
7505          * Tell the sequencer which SCB will be the next one it receives.
7506          */
7507         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7508         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7509
7510         /*
7511          * Default to coalescing disabled.
7512          */
7513         ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
7514         ahd_outw(ahd, CMDS_PENDING, 0);
7515         ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
7516                                      ahd->int_coalescing_maxcmds,
7517                                      ahd->int_coalescing_mincmds);
7518         ahd_enable_coalescing(ahd, FALSE);
7519
7520         ahd_loadseq(ahd);
7521         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7522
7523         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
7524                 u_int negodat3 = ahd_inb(ahd, NEGCONOPTS);
7525
7526                 negodat3 |= ENSLOWCRC;
7527                 ahd_outb(ahd, NEGCONOPTS, negodat3);
7528                 negodat3 = ahd_inb(ahd, NEGCONOPTS);
7529                 if (!(negodat3 & ENSLOWCRC))
7530                         printk("aic79xx: failed to set the SLOWCRC bit\n");
7531                 else
7532                         printk("aic79xx: SLOWCRC bit set\n");
7533         }
7534 }
7535
7536 /*
7537  * Setup default device and controller settings.
7538  * This should only be called if our probe has
7539  * determined that no configuration data is available.
7540  */
7541 int
7542 ahd_default_config(struct ahd_softc *ahd)
7543 {
7544         int     targ;
7545
7546         ahd->our_id = 7;
7547
7548         /*
7549          * Allocate a tstate to house information for our
7550          * initiator presence on the bus as well as the user
7551          * data for any target mode initiator.
7552          */
7553         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7554                 printk("%s: unable to allocate ahd_tmode_tstate.  "
7555                        "Failing attach\n", ahd_name(ahd));
7556                 return (ENOMEM);
7557         }
7558
7559         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
7560                 struct   ahd_devinfo devinfo;
7561                 struct   ahd_initiator_tinfo *tinfo;
7562                 struct   ahd_tmode_tstate *tstate;
7563                 uint16_t target_mask;
7564
7565                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7566                                             targ, &tstate);
7567                 /*
7568                  * We support SPC2 and SPI4.
7569                  */
7570                 tinfo->user.protocol_version = 4;
7571                 tinfo->user.transport_version = 4;
7572
7573                 target_mask = 0x01 << targ;
7574                 ahd->user_discenable |= target_mask;
7575                 tstate->discenable |= target_mask;
7576                 ahd->user_tagenable |= target_mask;
7577 #ifdef AHD_FORCE_160
7578                 tinfo->user.period = AHD_SYNCRATE_DT;
7579 #else
7580                 tinfo->user.period = AHD_SYNCRATE_160;
7581 #endif
7582                 tinfo->user.offset = MAX_OFFSET;
7583                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
7584                                         | MSG_EXT_PPR_WR_FLOW
7585                                         | MSG_EXT_PPR_HOLD_MCS
7586                                         | MSG_EXT_PPR_IU_REQ
7587                                         | MSG_EXT_PPR_QAS_REQ
7588                                         | MSG_EXT_PPR_DT_REQ;
7589                 if ((ahd->features & AHD_RTI) != 0)
7590                         tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
7591
7592                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
7593
7594                 /*
7595                  * Start out Async/Narrow/Untagged and with
7596                  * conservative protocol support.
7597                  */
7598                 tinfo->goal.protocol_version = 2;
7599                 tinfo->goal.transport_version = 2;
7600                 tinfo->curr.protocol_version = 2;
7601                 tinfo->curr.transport_version = 2;
7602                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7603                                     targ, CAM_LUN_WILDCARD,
7604                                     'A', ROLE_INITIATOR);
7605                 tstate->tagenable &= ~target_mask;
7606                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7607                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7608                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7609                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7610                                  /*paused*/TRUE);
7611         }
7612         return (0);
7613 }
7614
7615 /*
7616  * Parse device configuration information.
7617  */
7618 int
7619 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
7620 {
7621         int targ;
7622         int max_targ;
7623
7624         max_targ = sc->max_targets & CFMAXTARG;
7625         ahd->our_id = sc->brtime_id & CFSCSIID;
7626
7627         /*
7628          * Allocate a tstate to house information for our
7629          * initiator presence on the bus as well as the user
7630          * data for any target mode initiator.
7631          */
7632         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7633                 printk("%s: unable to allocate ahd_tmode_tstate.  "
7634                        "Failing attach\n", ahd_name(ahd));
7635                 return (ENOMEM);
7636         }
7637
7638         for (targ = 0; targ < max_targ; targ++) {
7639                 struct   ahd_devinfo devinfo;
7640                 struct   ahd_initiator_tinfo *tinfo;
7641                 struct   ahd_transinfo *user_tinfo;
7642                 struct   ahd_tmode_tstate *tstate;
7643                 uint16_t target_mask;
7644
7645                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7646                                             targ, &tstate);
7647                 user_tinfo = &tinfo->user;
7648
7649                 /*
7650                  * We support SPC2 and SPI4.
7651                  */
7652                 tinfo->user.protocol_version = 4;
7653                 tinfo->user.transport_version = 4;
7654
7655                 target_mask = 0x01 << targ;
7656                 ahd->user_discenable &= ~target_mask;
7657                 tstate->discenable &= ~target_mask;
7658                 ahd->user_tagenable &= ~target_mask;
7659                 if (sc->device_flags[targ] & CFDISC) {
7660                         tstate->discenable |= target_mask;
7661                         ahd->user_discenable |= target_mask;
7662                         ahd->user_tagenable |= target_mask;
7663                 } else {
7664                         /*
7665                          * Cannot be packetized without disconnection.
7666                          */
7667                         sc->device_flags[targ] &= ~CFPACKETIZED;
7668                 }
7669
7670                 user_tinfo->ppr_options = 0;
7671                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
7672                 if (user_tinfo->period < CFXFER_ASYNC) {
7673                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
7674                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
7675                         user_tinfo->offset = MAX_OFFSET;
7676                 } else  {
7677                         user_tinfo->offset = 0;
7678                         user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
7679                 }
7680 #ifdef AHD_FORCE_160
7681                 if (user_tinfo->period <= AHD_SYNCRATE_160)
7682                         user_tinfo->period = AHD_SYNCRATE_DT;
7683 #endif
7684
7685                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
7686                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
7687                                                 |  MSG_EXT_PPR_WR_FLOW
7688                                                 |  MSG_EXT_PPR_HOLD_MCS
7689                                                 |  MSG_EXT_PPR_IU_REQ;
7690                         if ((ahd->features & AHD_RTI) != 0)
7691                                 user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
7692                 }
7693
7694                 if ((sc->device_flags[targ] & CFQAS) != 0)
7695                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
7696
7697                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
7698                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
7699                 else
7700                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
7701 #ifdef AHD_DEBUG
7702                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
7703                         printk("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
7704                                user_tinfo->period, user_tinfo->offset,
7705                                user_tinfo->ppr_options);
7706 #endif
7707                 /*
7708                  * Start out Async/Narrow/Untagged and with
7709                  * conservative protocol support.
7710                  */
7711                 tstate->tagenable &= ~target_mask;
7712                 tinfo->goal.protocol_version = 2;
7713                 tinfo->goal.transport_version = 2;
7714                 tinfo->curr.protocol_version = 2;
7715                 tinfo->curr.transport_version = 2;
7716                 ahd_compile_devinfo(&devinfo, ahd->our_id,
7717                                     targ, CAM_LUN_WILDCARD,
7718                                     'A', ROLE_INITIATOR);
7719                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7720                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7721                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7722                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7723                                  /*paused*/TRUE);
7724         }
7725
7726         ahd->flags &= ~AHD_SPCHK_ENB_A;
7727         if (sc->bios_control & CFSPARITY)
7728                 ahd->flags |= AHD_SPCHK_ENB_A;
7729
7730         ahd->flags &= ~AHD_RESET_BUS_A;
7731         if (sc->bios_control & CFRESETB)
7732                 ahd->flags |= AHD_RESET_BUS_A;
7733
7734         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
7735         if (sc->bios_control & CFEXTEND)
7736                 ahd->flags |= AHD_EXTENDED_TRANS_A;
7737
7738         ahd->flags &= ~AHD_BIOS_ENABLED;
7739         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
7740                 ahd->flags |= AHD_BIOS_ENABLED;
7741
7742         ahd->flags &= ~AHD_STPWLEVEL_A;
7743         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
7744                 ahd->flags |= AHD_STPWLEVEL_A;
7745
7746         return (0);
7747 }
7748
7749 /*
7750  * Parse device configuration information.
7751  */
7752 int
7753 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
7754 {
7755         int error;
7756
7757         error = ahd_verify_vpd_cksum(vpd);
7758         if (error == 0)
7759                 return (EINVAL);
7760         if ((vpd->bios_flags & VPDBOOTHOST) != 0)
7761                 ahd->flags |= AHD_BOOT_CHANNEL;
7762         return (0);
7763 }
7764
7765 void
7766 ahd_intr_enable(struct ahd_softc *ahd, int enable)
7767 {
7768         u_int hcntrl;
7769
7770         hcntrl = ahd_inb(ahd, HCNTRL);
7771         hcntrl &= ~INTEN;
7772         ahd->pause &= ~INTEN;
7773         ahd->unpause &= ~INTEN;
7774         if (enable) {
7775                 hcntrl |= INTEN;
7776                 ahd->pause |= INTEN;
7777                 ahd->unpause |= INTEN;
7778         }
7779         ahd_outb(ahd, HCNTRL, hcntrl);
7780 }
7781
7782 static void
7783 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7784                              u_int mincmds)
7785 {
7786         if (timer > AHD_TIMER_MAX_US)
7787                 timer = AHD_TIMER_MAX_US;
7788         ahd->int_coalescing_timer = timer;
7789
7790         if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7791                 maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7792         if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7793                 mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7794         ahd->int_coalescing_maxcmds = maxcmds;
7795         ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7796         ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7797         ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7798 }
7799
7800 static void
7801 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7802 {
7803
7804         ahd->hs_mailbox &= ~ENINT_COALESCE;
7805         if (enable)
7806                 ahd->hs_mailbox |= ENINT_COALESCE;
7807         ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7808         ahd_flush_device_writes(ahd);
7809         ahd_run_qoutfifo(ahd);
7810 }
7811
7812 /*
7813  * Ensure that the card is paused in a location
7814  * outside of all critical sections and that all
7815  * pending work is completed prior to returning.
7816  * This routine should only be called from outside
7817  * an interrupt context.
7818  */
7819 void
7820 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7821 {
7822         u_int intstat;
7823         u_int maxloops;
7824
7825         maxloops = 1000;
7826         ahd->flags |= AHD_ALL_INTERRUPTS;
7827         ahd_pause(ahd);
7828         /*
7829          * Freeze the outgoing selections.  We do this only
7830          * until we are safely paused without further selections
7831          * pending.
7832          */
7833         ahd->qfreeze_cnt--;
7834         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7835         ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7836         do {
7837
7838                 ahd_unpause(ahd);
7839                 /*
7840                  * Give the sequencer some time to service
7841                  * any active selections.
7842                  */
7843                 ahd_delay(500);
7844
7845                 ahd_intr(ahd);
7846                 ahd_pause(ahd);
7847                 intstat = ahd_inb(ahd, INTSTAT);
7848                 if ((intstat & INT_PEND) == 0) {
7849                         ahd_clear_critical_section(ahd);
7850                         intstat = ahd_inb(ahd, INTSTAT);
7851                 }
7852         } while (--maxloops
7853               && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7854               && ((intstat & INT_PEND) != 0
7855                || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7856                || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7857
7858         if (maxloops == 0) {
7859                 printk("Infinite interrupt loop, INTSTAT = %x",
7860                       ahd_inb(ahd, INTSTAT));
7861         }
7862         ahd->qfreeze_cnt++;
7863         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7864
7865         ahd_flush_qoutfifo(ahd);
7866
7867         ahd->flags &= ~AHD_ALL_INTERRUPTS;
7868 }
7869
7870 int __maybe_unused
7871 ahd_suspend(struct ahd_softc *ahd)
7872 {
7873         ahd_pause_and_flushwork(ahd);
7874
7875         if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7876                 ahd_unpause(ahd);
7877                 return (EBUSY);
7878         }
7879         ahd_shutdown(ahd);
7880         return (0);
7881 }
7882
7883 void __maybe_unused
7884 ahd_resume(struct ahd_softc *ahd)
7885 {
7886         ahd_reset(ahd, /*reinit*/TRUE);
7887         ahd_intr_enable(ahd, TRUE);
7888         ahd_restart(ahd);
7889 }
7890
7891 /************************** Busy Target Table *********************************/
7892 /*
7893  * Set SCBPTR to the SCB that contains the busy
7894  * table entry for TCL.  Return the offset into
7895  * the SCB that contains the entry for TCL.
7896  * saved_scbid is dereferenced and set to the
7897  * scbid that should be restored once manipualtion
7898  * of the TCL entry is complete.
7899  */
7900 static inline u_int
7901 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7902 {
7903         /*
7904          * Index to the SCB that contains the busy entry.
7905          */
7906         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7907         *saved_scbid = ahd_get_scbptr(ahd);
7908         ahd_set_scbptr(ahd, TCL_LUN(tcl)
7909                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7910
7911         /*
7912          * And now calculate the SCB offset to the entry.
7913          * Each entry is 2 bytes wide, hence the
7914          * multiplication by 2.
7915          */
7916         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
7917 }
7918
7919 /*
7920  * Return the untagged transaction id for a given target/channel lun.
7921  */
7922 static u_int
7923 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
7924 {
7925         u_int scbid;
7926         u_int scb_offset;
7927         u_int saved_scbptr;
7928
7929         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7930         scbid = ahd_inw_scbram(ahd, scb_offset);
7931         ahd_set_scbptr(ahd, saved_scbptr);
7932         return (scbid);
7933 }
7934
7935 static void
7936 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
7937 {
7938         u_int scb_offset;
7939         u_int saved_scbptr;
7940
7941         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7942         ahd_outw(ahd, scb_offset, scbid);
7943         ahd_set_scbptr(ahd, saved_scbptr);
7944 }
7945
7946 /************************** SCB and SCB queue management **********************/
7947 static int
7948 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
7949               char channel, int lun, u_int tag, role_t role)
7950 {
7951         int targ = SCB_GET_TARGET(ahd, scb);
7952         char chan = SCB_GET_CHANNEL(ahd, scb);
7953         int slun = SCB_GET_LUN(scb);
7954         int match;
7955
7956         match = ((chan == channel) || (channel == ALL_CHANNELS));
7957         if (match != 0)
7958                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
7959         if (match != 0)
7960                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
7961         if (match != 0) {
7962 #ifdef AHD_TARGET_MODE
7963                 int group;
7964
7965                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
7966                 if (role == ROLE_INITIATOR) {
7967                         match = (group != XPT_FC_GROUP_TMODE)
7968                               && ((tag == SCB_GET_TAG(scb))
7969                                || (tag == SCB_LIST_NULL));
7970                 } else if (role == ROLE_TARGET) {
7971                         match = (group == XPT_FC_GROUP_TMODE)
7972                               && ((tag == scb->io_ctx->csio.tag_id)
7973                                || (tag == SCB_LIST_NULL));
7974                 }
7975 #else /* !AHD_TARGET_MODE */
7976                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
7977 #endif /* AHD_TARGET_MODE */
7978         }
7979
7980         return match;
7981 }
7982
7983 static void
7984 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
7985 {
7986         int     target;
7987         char    channel;
7988         int     lun;
7989
7990         target = SCB_GET_TARGET(ahd, scb);
7991         lun = SCB_GET_LUN(scb);
7992         channel = SCB_GET_CHANNEL(ahd, scb);
7993
7994         ahd_search_qinfifo(ahd, target, channel, lun,
7995                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
7996                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7997
7998         ahd_platform_freeze_devq(ahd, scb);
7999 }
8000
8001 void
8002 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
8003 {
8004         struct scb      *prev_scb;
8005         ahd_mode_state   saved_modes;
8006
8007         saved_modes = ahd_save_modes(ahd);
8008         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8009         prev_scb = NULL;
8010         if (ahd_qinfifo_count(ahd) != 0) {
8011                 u_int prev_tag;
8012                 u_int prev_pos;
8013
8014                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
8015                 prev_tag = ahd->qinfifo[prev_pos];
8016                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
8017         }
8018         ahd_qinfifo_requeue(ahd, prev_scb, scb);
8019         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8020         ahd_restore_modes(ahd, saved_modes);
8021 }
8022
8023 static void
8024 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
8025                     struct scb *scb)
8026 {
8027         if (prev_scb == NULL) {
8028                 uint32_t busaddr;
8029
8030                 busaddr = ahd_le32toh(scb->hscb->hscb_busaddr);
8031                 ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8032         } else {
8033                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
8034                 ahd_sync_scb(ahd, prev_scb,
8035                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8036         }
8037         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
8038         ahd->qinfifonext++;
8039         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
8040         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8041 }
8042
8043 static int
8044 ahd_qinfifo_count(struct ahd_softc *ahd)
8045 {
8046         u_int qinpos;
8047         u_int wrap_qinpos;
8048         u_int wrap_qinfifonext;
8049
8050         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
8051         qinpos = ahd_get_snscb_qoff(ahd);
8052         wrap_qinpos = AHD_QIN_WRAP(qinpos);
8053         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
8054         if (wrap_qinfifonext >= wrap_qinpos)
8055                 return (wrap_qinfifonext - wrap_qinpos);
8056         else
8057                 return (wrap_qinfifonext
8058                       + ARRAY_SIZE(ahd->qinfifo) - wrap_qinpos);
8059 }
8060
8061 static void
8062 ahd_reset_cmds_pending(struct ahd_softc *ahd)
8063 {
8064         struct          scb *scb;
8065         ahd_mode_state  saved_modes;
8066         u_int           pending_cmds;
8067
8068         saved_modes = ahd_save_modes(ahd);
8069         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8070
8071         /*
8072          * Don't count any commands as outstanding that the
8073          * sequencer has already marked for completion.
8074          */
8075         ahd_flush_qoutfifo(ahd);
8076
8077         pending_cmds = 0;
8078         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
8079                 pending_cmds++;
8080         }
8081         ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
8082         ahd_restore_modes(ahd, saved_modes);
8083         ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
8084 }
8085
8086 static void
8087 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
8088 {
8089         cam_status ostat;
8090         cam_status cstat;
8091
8092         ostat = ahd_get_transaction_status(scb);
8093         if (ostat == CAM_REQ_INPROG)
8094                 ahd_set_transaction_status(scb, status);
8095         cstat = ahd_get_transaction_status(scb);
8096         if (cstat != CAM_REQ_CMP)
8097                 ahd_freeze_scb(scb);
8098         ahd_done(ahd, scb);
8099 }
8100
8101 int
8102 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
8103                    int lun, u_int tag, role_t role, uint32_t status,
8104                    ahd_search_action action)
8105 {
8106         struct scb      *scb;
8107         struct scb      *mk_msg_scb;
8108         struct scb      *prev_scb;
8109         ahd_mode_state   saved_modes;
8110         u_int            qinstart;
8111         u_int            qinpos;
8112         u_int            qintail;
8113         u_int            tid_next;
8114         u_int            tid_prev;
8115         u_int            scbid;
8116         u_int            seq_flags2;
8117         u_int            savedscbptr;
8118         uint32_t         busaddr;
8119         int              found;
8120         int              targets;
8121
8122         /* Must be in CCHAN mode */
8123         saved_modes = ahd_save_modes(ahd);
8124         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8125
8126         /*
8127          * Halt any pending SCB DMA.  The sequencer will reinitiate
8128          * this dma if the qinfifo is not empty once we unpause.
8129          */
8130         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
8131          == (CCARREN|CCSCBEN|CCSCBDIR)) {
8132                 ahd_outb(ahd, CCSCBCTL,
8133                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
8134                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
8135                         ;
8136         }
8137         /* Determine sequencer's position in the qinfifo. */
8138         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
8139         qinstart = ahd_get_snscb_qoff(ahd);
8140         qinpos = AHD_QIN_WRAP(qinstart);
8141         found = 0;
8142         prev_scb = NULL;
8143
8144         if (action == SEARCH_PRINT) {
8145                 printk("qinstart = %d qinfifonext = %d\nQINFIFO:",
8146                        qinstart, ahd->qinfifonext);
8147         }
8148
8149         /*
8150          * Start with an empty queue.  Entries that are not chosen
8151          * for removal will be re-added to the queue as we go.
8152          */
8153         ahd->qinfifonext = qinstart;
8154         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
8155         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8156
8157         while (qinpos != qintail) {
8158                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
8159                 if (scb == NULL) {
8160                         printk("qinpos = %d, SCB index = %d\n",
8161                                 qinpos, ahd->qinfifo[qinpos]);
8162                         panic("Loop 1\n");
8163                 }
8164
8165                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
8166                         /*
8167                          * We found an scb that needs to be acted on.
8168                          */
8169                         found++;
8170                         switch (action) {
8171                         case SEARCH_COMPLETE:
8172                                 if ((scb->flags & SCB_ACTIVE) == 0)
8173                                         printk("Inactive SCB in qinfifo\n");
8174                                 ahd_done_with_status(ahd, scb, status);
8175                                 fallthrough;
8176                         case SEARCH_REMOVE:
8177                                 break;
8178                         case SEARCH_PRINT:
8179                                 printk(" 0x%x", ahd->qinfifo[qinpos]);
8180                                 fallthrough;
8181                         case SEARCH_COUNT:
8182                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
8183                                 prev_scb = scb;
8184                                 break;
8185                         }
8186                 } else {
8187                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
8188                         prev_scb = scb;
8189                 }
8190                 qinpos = AHD_QIN_WRAP(qinpos+1);
8191         }
8192
8193         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8194
8195         if (action == SEARCH_PRINT)
8196                 printk("\nWAITING_TID_QUEUES:\n");
8197
8198         /*
8199          * Search waiting for selection lists.  We traverse the
8200          * list of "their ids" waiting for selection and, if
8201          * appropriate, traverse the SCBs of each "their id"
8202          * looking for matches.
8203          */
8204         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8205         seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
8206         if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
8207                 scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
8208                 mk_msg_scb = ahd_lookup_scb(ahd, scbid);
8209         } else
8210                 mk_msg_scb = NULL;
8211         savedscbptr = ahd_get_scbptr(ahd);
8212         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
8213         tid_prev = SCB_LIST_NULL;
8214         targets = 0;
8215         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
8216                 u_int tid_head;
8217                 u_int tid_tail;
8218
8219                 targets++;
8220                 if (targets > AHD_NUM_TARGETS)
8221                         panic("TID LIST LOOP");
8222
8223                 if (scbid >= ahd->scb_data.numscbs) {
8224                         printk("%s: Waiting TID List inconsistency. "
8225                                "SCB index == 0x%x, yet numscbs == 0x%x.",
8226                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8227                         ahd_dump_card_state(ahd);
8228                         panic("for safety");
8229                 }
8230                 scb = ahd_lookup_scb(ahd, scbid);
8231                 if (scb == NULL) {
8232                         printk("%s: SCB = 0x%x Not Active!\n",
8233                                ahd_name(ahd), scbid);
8234                         panic("Waiting TID List traversal\n");
8235                 }
8236                 ahd_set_scbptr(ahd, scbid);
8237                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
8238                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8239                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
8240                         tid_prev = scbid;
8241                         continue;
8242                 }
8243
8244                 /*
8245                  * We found a list of scbs that needs to be searched.
8246                  */
8247                 if (action == SEARCH_PRINT)
8248                         printk("       %d ( ", SCB_GET_TARGET(ahd, scb));
8249                 tid_head = scbid;
8250                 found += ahd_search_scb_list(ahd, target, channel,
8251                                              lun, tag, role, status,
8252                                              action, &tid_head, &tid_tail,
8253                                              SCB_GET_TARGET(ahd, scb));
8254                 /*
8255                  * Check any MK_MESSAGE SCB that is still waiting to
8256                  * enter this target's waiting for selection queue.
8257                  */
8258                 if (mk_msg_scb != NULL
8259                  && ahd_match_scb(ahd, mk_msg_scb, target, channel,
8260                                   lun, tag, role)) {
8261
8262                         /*
8263                          * We found an scb that needs to be acted on.
8264                          */
8265                         found++;
8266                         switch (action) {
8267                         case SEARCH_COMPLETE:
8268                                 if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
8269                                         printk("Inactive SCB pending MK_MSG\n");
8270                                 ahd_done_with_status(ahd, mk_msg_scb, status);
8271                                 fallthrough;
8272                         case SEARCH_REMOVE:
8273                         {
8274                                 u_int tail_offset;
8275
8276                                 printk("Removing MK_MSG scb\n");
8277
8278                                 /*
8279                                  * Reset our tail to the tail of the
8280                                  * main per-target list.
8281                                  */
8282                                 tail_offset = WAITING_SCB_TAILS
8283                                     + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
8284                                 ahd_outw(ahd, tail_offset, tid_tail);
8285
8286                                 seq_flags2 &= ~PENDING_MK_MESSAGE;
8287                                 ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8288                                 ahd_outw(ahd, CMDS_PENDING,
8289                                          ahd_inw(ahd, CMDS_PENDING)-1);
8290                                 mk_msg_scb = NULL;
8291                                 break;
8292                         }
8293                         case SEARCH_PRINT:
8294                                 printk(" 0x%x", SCB_GET_TAG(scb));
8295                                 fallthrough;
8296                         case SEARCH_COUNT:
8297                                 break;
8298                         }
8299                 }
8300
8301                 if (mk_msg_scb != NULL
8302                  && SCBID_IS_NULL(tid_head)
8303                  && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8304                                   SCB_LIST_NULL, ROLE_UNKNOWN)) {
8305
8306                         /*
8307                          * When removing the last SCB for a target
8308                          * queue with a pending MK_MESSAGE scb, we
8309                          * must queue the MK_MESSAGE scb.
8310                          */
8311                         printk("Queueing mk_msg_scb\n");
8312                         tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
8313                         seq_flags2 &= ~PENDING_MK_MESSAGE;
8314                         ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8315                         mk_msg_scb = NULL;
8316                 }
8317                 if (tid_head != scbid)
8318                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
8319                 if (!SCBID_IS_NULL(tid_head))
8320                         tid_prev = tid_head;
8321                 if (action == SEARCH_PRINT)
8322                         printk(")\n");
8323         }
8324
8325         /* Restore saved state. */
8326         ahd_set_scbptr(ahd, savedscbptr);
8327         ahd_restore_modes(ahd, saved_modes);
8328         return (found);
8329 }
8330
8331 static int
8332 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
8333                     int lun, u_int tag, role_t role, uint32_t status,
8334                     ahd_search_action action, u_int *list_head,
8335                     u_int *list_tail, u_int tid)
8336 {
8337         struct  scb *scb;
8338         u_int   scbid;
8339         u_int   next;
8340         u_int   prev;
8341         int     found;
8342
8343         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8344         found = 0;
8345         prev = SCB_LIST_NULL;
8346         next = *list_head;
8347         *list_tail = SCB_LIST_NULL;
8348         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
8349                 if (scbid >= ahd->scb_data.numscbs) {
8350                         printk("%s:SCB List inconsistency. "
8351                                "SCB == 0x%x, yet numscbs == 0x%x.",
8352                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8353                         ahd_dump_card_state(ahd);
8354                         panic("for safety");
8355                 }
8356                 scb = ahd_lookup_scb(ahd, scbid);
8357                 if (scb == NULL) {
8358                         printk("%s: SCB = %d Not Active!\n",
8359                                ahd_name(ahd), scbid);
8360                         panic("Waiting List traversal\n");
8361                 }
8362                 ahd_set_scbptr(ahd, scbid);
8363                 *list_tail = scbid;
8364                 next = ahd_inw_scbram(ahd, SCB_NEXT);
8365                 if (ahd_match_scb(ahd, scb, target, channel,
8366                                   lun, SCB_LIST_NULL, role) == 0) {
8367                         prev = scbid;
8368                         continue;
8369                 }
8370                 found++;
8371                 switch (action) {
8372                 case SEARCH_COMPLETE:
8373                         if ((scb->flags & SCB_ACTIVE) == 0)
8374                                 printk("Inactive SCB in Waiting List\n");
8375                         ahd_done_with_status(ahd, scb, status);
8376                         fallthrough;
8377                 case SEARCH_REMOVE:
8378                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
8379                         *list_tail = prev;
8380                         if (SCBID_IS_NULL(prev))
8381                                 *list_head = next;
8382                         break;
8383                 case SEARCH_PRINT:
8384                         printk("0x%x ", scbid);
8385                         fallthrough;
8386                 case SEARCH_COUNT:
8387                         prev = scbid;
8388                         break;
8389                 }
8390                 if (found > AHD_SCB_MAX)
8391                         panic("SCB LIST LOOP");
8392         }
8393         if (action == SEARCH_COMPLETE
8394          || action == SEARCH_REMOVE)
8395                 ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
8396         return (found);
8397 }
8398
8399 static void
8400 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
8401                     u_int tid_cur, u_int tid_next)
8402 {
8403         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8404
8405         if (SCBID_IS_NULL(tid_cur)) {
8406
8407                 /* Bypass current TID list */
8408                 if (SCBID_IS_NULL(tid_prev)) {
8409                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
8410                 } else {
8411                         ahd_set_scbptr(ahd, tid_prev);
8412                         ahd_outw(ahd, SCB_NEXT2, tid_next);
8413                 }
8414                 if (SCBID_IS_NULL(tid_next))
8415                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
8416         } else {
8417
8418                 /* Stitch through tid_cur */
8419                 if (SCBID_IS_NULL(tid_prev)) {
8420                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
8421                 } else {
8422                         ahd_set_scbptr(ahd, tid_prev);
8423                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
8424                 }
8425                 ahd_set_scbptr(ahd, tid_cur);
8426                 ahd_outw(ahd, SCB_NEXT2, tid_next);
8427
8428                 if (SCBID_IS_NULL(tid_next))
8429                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
8430         }
8431 }
8432
8433 /*
8434  * Manipulate the waiting for selection list and return the
8435  * scb that follows the one that we remove.
8436  */
8437 static u_int
8438 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
8439              u_int prev, u_int next, u_int tid)
8440 {
8441         u_int tail_offset;
8442
8443         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8444         if (!SCBID_IS_NULL(prev)) {
8445                 ahd_set_scbptr(ahd, prev);
8446                 ahd_outw(ahd, SCB_NEXT, next);
8447         }
8448
8449         /*
8450          * SCBs that have MK_MESSAGE set in them may
8451          * cause the tail pointer to be updated without
8452          * setting the next pointer of the previous tail.
8453          * Only clear the tail if the removed SCB was
8454          * the tail.
8455          */
8456         tail_offset = WAITING_SCB_TAILS + (2 * tid);
8457         if (SCBID_IS_NULL(next)
8458          && ahd_inw(ahd, tail_offset) == scbid)
8459                 ahd_outw(ahd, tail_offset, prev);
8460
8461         ahd_add_scb_to_free_list(ahd, scbid);
8462         return (next);
8463 }
8464
8465 /*
8466  * Add the SCB as selected by SCBPTR onto the on chip list of
8467  * free hardware SCBs.  This list is empty/unused if we are not
8468  * performing SCB paging.
8469  */
8470 static void
8471 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
8472 {
8473 /* XXX Need some other mechanism to designate "free". */
8474         /*
8475          * Invalidate the tag so that our abort
8476          * routines don't think it's active.
8477         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
8478          */
8479 }
8480
8481 /******************************** Error Handling ******************************/
8482 /*
8483  * Abort all SCBs that match the given description (target/channel/lun/tag),
8484  * setting their status to the passed in status if the status has not already
8485  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
8486  * is paused before it is called.
8487  */
8488 static int
8489 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
8490                int lun, u_int tag, role_t role, uint32_t status)
8491 {
8492         struct          scb *scbp;
8493         struct          scb *scbp_next;
8494         u_int           i, j;
8495         u_int           maxtarget;
8496         u_int           minlun;
8497         u_int           maxlun;
8498         int             found;
8499         ahd_mode_state  saved_modes;
8500
8501         /* restore this when we're done */
8502         saved_modes = ahd_save_modes(ahd);
8503         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8504
8505         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
8506                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
8507
8508         /*
8509          * Clean out the busy target table for any untagged commands.
8510          */
8511         i = 0;
8512         maxtarget = 16;
8513         if (target != CAM_TARGET_WILDCARD) {
8514                 i = target;
8515                 if (channel == 'B')
8516                         i += 8;
8517                 maxtarget = i + 1;
8518         }
8519
8520         if (lun == CAM_LUN_WILDCARD) {
8521                 minlun = 0;
8522                 maxlun = AHD_NUM_LUNS_NONPKT;
8523         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
8524                 minlun = maxlun = 0;
8525         } else {
8526                 minlun = lun;
8527                 maxlun = lun + 1;
8528         }
8529
8530         if (role != ROLE_TARGET) {
8531                 for (;i < maxtarget; i++) {
8532                         for (j = minlun;j < maxlun; j++) {
8533                                 u_int scbid;
8534                                 u_int tcl;
8535
8536                                 tcl = BUILD_TCL_RAW(i, 'A', j);
8537                                 scbid = ahd_find_busy_tcl(ahd, tcl);
8538                                 scbp = ahd_lookup_scb(ahd, scbid);
8539                                 if (scbp == NULL
8540                                  || ahd_match_scb(ahd, scbp, target, channel,
8541                                                   lun, tag, role) == 0)
8542                                         continue;
8543                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
8544                         }
8545                 }
8546         }
8547
8548         /*
8549          * Don't abort commands that have already completed,
8550          * but haven't quite made it up to the host yet.
8551          */
8552         ahd_flush_qoutfifo(ahd);
8553
8554         /*
8555          * Go through the pending CCB list and look for
8556          * commands for this target that are still active.
8557          * These are other tagged commands that were
8558          * disconnected when the reset occurred.
8559          */
8560         scbp_next = LIST_FIRST(&ahd->pending_scbs);
8561         while (scbp_next != NULL) {
8562                 scbp = scbp_next;
8563                 scbp_next = LIST_NEXT(scbp, pending_links);
8564                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
8565                         cam_status ostat;
8566
8567                         ostat = ahd_get_transaction_status(scbp);
8568                         if (ostat == CAM_REQ_INPROG)
8569                                 ahd_set_transaction_status(scbp, status);
8570                         if (ahd_get_transaction_status(scbp) != CAM_REQ_CMP)
8571                                 ahd_freeze_scb(scbp);
8572                         if ((scbp->flags & SCB_ACTIVE) == 0)
8573                                 printk("Inactive SCB on pending list\n");
8574                         ahd_done(ahd, scbp);
8575                         found++;
8576                 }
8577         }
8578         ahd_restore_modes(ahd, saved_modes);
8579         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
8580         ahd->flags |= AHD_UPDATE_PEND_CMDS;
8581         return found;
8582 }
8583
8584 static void
8585 ahd_reset_current_bus(struct ahd_softc *ahd)
8586 {
8587         uint8_t scsiseq;
8588
8589         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8590         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
8591         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
8592         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
8593         ahd_flush_device_writes(ahd);
8594         ahd_delay(AHD_BUSRESET_DELAY);
8595         /* Turn off the bus reset */
8596         ahd_outb(ahd, SCSISEQ0, scsiseq);
8597         ahd_flush_device_writes(ahd);
8598         ahd_delay(AHD_BUSRESET_DELAY);
8599         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
8600                 /*
8601                  * 2A Razor #474
8602                  * Certain chip state is not cleared for
8603                  * SCSI bus resets that we initiate, so
8604                  * we must reset the chip.
8605                  */
8606                 ahd_reset(ahd, /*reinit*/TRUE);
8607                 ahd_intr_enable(ahd, /*enable*/TRUE);
8608                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8609         }
8610
8611         ahd_clear_intstat(ahd);
8612 }
8613
8614 int
8615 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
8616 {
8617         struct  ahd_devinfo caminfo;
8618         u_int   initiator;
8619         u_int   target;
8620         u_int   max_scsiid;
8621         int     found;
8622         u_int   fifo;
8623         u_int   next_fifo;
8624         uint8_t scsiseq;
8625
8626         /*
8627          * Check if the last bus reset is cleared
8628          */
8629         if (ahd->flags & AHD_BUS_RESET_ACTIVE) {
8630                 printk("%s: bus reset still active\n",
8631                        ahd_name(ahd));
8632                 return 0;
8633         }
8634         ahd->flags |= AHD_BUS_RESET_ACTIVE;
8635
8636         ahd->pending_device = NULL;
8637
8638         ahd_compile_devinfo(&caminfo,
8639                             CAM_TARGET_WILDCARD,
8640                             CAM_TARGET_WILDCARD,
8641                             CAM_LUN_WILDCARD,
8642                             channel, ROLE_UNKNOWN);
8643         ahd_pause(ahd);
8644
8645         /* Make sure the sequencer is in a safe location. */
8646         ahd_clear_critical_section(ahd);
8647
8648         /*
8649          * Run our command complete fifos to ensure that we perform
8650          * completion processing on any commands that 'completed'
8651          * before the reset occurred.
8652          */
8653         ahd_run_qoutfifo(ahd);
8654 #ifdef AHD_TARGET_MODE
8655         if ((ahd->flags & AHD_TARGETROLE) != 0) {
8656                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
8657         }
8658 #endif
8659         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8660
8661         /*
8662          * Disable selections so no automatic hardware
8663          * functions will modify chip state.
8664          */
8665         ahd_outb(ahd, SCSISEQ0, 0);
8666         ahd_outb(ahd, SCSISEQ1, 0);
8667
8668         /*
8669          * Safely shut down our DMA engines.  Always start with
8670          * the FIFO that is not currently active (if any are
8671          * actively connected).
8672          */
8673         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
8674         if (next_fifo > CURRFIFO_1)
8675                 /* If disconneced, arbitrarily start with FIFO1. */
8676                 next_fifo = fifo = 0;
8677         do {
8678                 next_fifo ^= CURRFIFO_1;
8679                 ahd_set_modes(ahd, next_fifo, next_fifo);
8680                 ahd_outb(ahd, DFCNTRL,
8681                          ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
8682                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
8683                         ahd_delay(10);
8684                 /*
8685                  * Set CURRFIFO to the now inactive channel.
8686                  */
8687                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8688                 ahd_outb(ahd, DFFSTAT, next_fifo);
8689         } while (next_fifo != fifo);
8690
8691         /*
8692          * Reset the bus if we are initiating this reset
8693          */
8694         ahd_clear_msg_state(ahd);
8695         ahd_outb(ahd, SIMODE1,
8696                  ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
8697
8698         if (initiate_reset)
8699                 ahd_reset_current_bus(ahd);
8700
8701         ahd_clear_intstat(ahd);
8702
8703         /*
8704          * Clean up all the state information for the
8705          * pending transactions on this bus.
8706          */
8707         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
8708                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
8709                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
8710
8711         /*
8712          * Cleanup anything left in the FIFOs.
8713          */
8714         ahd_clear_fifo(ahd, 0);
8715         ahd_clear_fifo(ahd, 1);
8716
8717         /*
8718          * Clear SCSI interrupt status
8719          */
8720         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8721
8722         /*
8723          * Reenable selections
8724          */
8725         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8726         scsiseq = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8727         ahd_outb(ahd, SCSISEQ1, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
8728
8729         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8730 #ifdef AHD_TARGET_MODE
8731         /*
8732          * Send an immediate notify ccb to all target more peripheral
8733          * drivers affected by this action.
8734          */
8735         for (target = 0; target <= max_scsiid; target++) {
8736                 struct ahd_tmode_tstate* tstate;
8737                 u_int lun;
8738
8739                 tstate = ahd->enabled_targets[target];
8740                 if (tstate == NULL)
8741                         continue;
8742                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
8743                         struct ahd_tmode_lstate* lstate;
8744
8745                         lstate = tstate->enabled_luns[lun];
8746                         if (lstate == NULL)
8747                                 continue;
8748
8749                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
8750                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
8751                         ahd_send_lstate_events(ahd, lstate);
8752                 }
8753         }
8754 #endif
8755         /*
8756          * Revert to async/narrow transfers until we renegotiate.
8757          */
8758         for (target = 0; target <= max_scsiid; target++) {
8759
8760                 if (ahd->enabled_targets[target] == NULL)
8761                         continue;
8762                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
8763                         struct ahd_devinfo devinfo;
8764
8765                         ahd_compile_devinfo(&devinfo, target, initiator,
8766                                             CAM_LUN_WILDCARD,
8767                                             'A', ROLE_UNKNOWN);
8768                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
8769                                       AHD_TRANS_CUR, /*paused*/TRUE);
8770                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
8771                                          /*offset*/0, /*ppr_options*/0,
8772                                          AHD_TRANS_CUR, /*paused*/TRUE);
8773                 }
8774         }
8775
8776         /* Notify the XPT that a bus reset occurred */
8777         ahd_send_async(ahd, caminfo.channel, CAM_TARGET_WILDCARD,
8778                        CAM_LUN_WILDCARD, AC_BUS_RESET);
8779
8780         ahd_restart(ahd);
8781
8782         return (found);
8783 }
8784
8785 /**************************** Statistics Processing ***************************/
8786 static void
8787 ahd_stat_timer(struct timer_list *t)
8788 {
8789         struct  ahd_softc *ahd = from_timer(ahd, t, stat_timer);
8790         u_long  s;
8791         int     enint_coal;
8792
8793         ahd_lock(ahd, &s);
8794
8795         enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8796         if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8797                 enint_coal |= ENINT_COALESCE;
8798         else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8799                 enint_coal &= ~ENINT_COALESCE;
8800
8801         if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8802                 ahd_enable_coalescing(ahd, enint_coal);
8803 #ifdef AHD_DEBUG
8804                 if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8805                         printk("%s: Interrupt coalescing "
8806                                "now %sabled. Cmds %d\n",
8807                                ahd_name(ahd),
8808                                (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8809                                ahd->cmdcmplt_total);
8810 #endif
8811         }
8812
8813         ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8814         ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8815         ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8816         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
8817         ahd_unlock(ahd, &s);
8818 }
8819
8820 /****************************** Status Processing *****************************/
8821
8822 static void
8823 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8824 {
8825         struct  hardware_scb *hscb;
8826         int     paused;
8827
8828         /*
8829          * The sequencer freezes its select-out queue
8830          * anytime a SCSI status error occurs.  We must
8831          * handle the error and increment our qfreeze count
8832          * to allow the sequencer to continue.  We don't
8833          * bother clearing critical sections here since all
8834          * operations are on data structures that the sequencer
8835          * is not touching once the queue is frozen.
8836          */
8837         hscb = scb->hscb;
8838
8839         if (ahd_is_paused(ahd)) {
8840                 paused = 1;
8841         } else {
8842                 paused = 0;
8843                 ahd_pause(ahd);
8844         }
8845
8846         /* Freeze the queue until the client sees the error. */
8847         ahd_freeze_devq(ahd, scb);
8848         ahd_freeze_scb(scb);
8849         ahd->qfreeze_cnt++;
8850         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8851
8852         if (paused == 0)
8853                 ahd_unpause(ahd);
8854
8855         /* Don't want to clobber the original sense code */
8856         if ((scb->flags & SCB_SENSE) != 0) {
8857                 /*
8858                  * Clear the SCB_SENSE Flag and perform
8859                  * a normal command completion.
8860                  */
8861                 scb->flags &= ~SCB_SENSE;
8862                 ahd_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8863                 ahd_done(ahd, scb);
8864                 return;
8865         }
8866         ahd_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8867         ahd_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8868         switch (hscb->shared_data.istatus.scsi_status) {
8869         case STATUS_PKT_SENSE:
8870         {
8871                 struct scsi_status_iu_header *siu;
8872
8873                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8874                 siu = (struct scsi_status_iu_header *)scb->sense_data;
8875                 ahd_set_scsi_status(scb, siu->status);
8876 #ifdef AHD_DEBUG
8877                 if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8878                         ahd_print_path(ahd, scb);
8879                         printk("SCB 0x%x Received PKT Status of 0x%x\n",
8880                                SCB_GET_TAG(scb), siu->status);
8881                         printk("\tflags = 0x%x, sense len = 0x%x, "
8882                                "pktfail = 0x%x\n",
8883                                siu->flags, scsi_4btoul(siu->sense_length),
8884                                scsi_4btoul(siu->pkt_failures_length));
8885                 }
8886 #endif
8887                 if ((siu->flags & SIU_RSPVALID) != 0) {
8888                         ahd_print_path(ahd, scb);
8889                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8890                                 printk("Unable to parse pkt_failures\n");
8891                         } else {
8892
8893                                 switch (SIU_PKTFAIL_CODE(siu)) {
8894                                 case SIU_PFC_NONE:
8895                                         printk("No packet failure found\n");
8896                                         break;
8897                                 case SIU_PFC_CIU_FIELDS_INVALID:
8898                                         printk("Invalid Command IU Field\n");
8899                                         break;
8900                                 case SIU_PFC_TMF_NOT_SUPPORTED:
8901                                         printk("TMF not supported\n");
8902                                         break;
8903                                 case SIU_PFC_TMF_FAILED:
8904                                         printk("TMF failed\n");
8905                                         break;
8906                                 case SIU_PFC_INVALID_TYPE_CODE:
8907                                         printk("Invalid L_Q Type code\n");
8908                                         break;
8909                                 case SIU_PFC_ILLEGAL_REQUEST:
8910                                         printk("Illegal request\n");
8911                                         break;
8912                                 default:
8913                                         break;
8914                                 }
8915                         }
8916                         if (siu->status == SCSI_STATUS_OK)
8917                                 ahd_set_transaction_status(scb,
8918                                                            CAM_REQ_CMP_ERR);
8919                 }
8920                 if ((siu->flags & SIU_SNSVALID) != 0) {
8921                         scb->flags |= SCB_PKT_SENSE;
8922 #ifdef AHD_DEBUG
8923                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
8924                                 printk("Sense data available\n");
8925 #endif
8926                 }
8927                 ahd_done(ahd, scb);
8928                 break;
8929         }
8930         case SCSI_STATUS_CMD_TERMINATED:
8931         case SCSI_STATUS_CHECK_COND:
8932         {
8933                 struct ahd_devinfo devinfo;
8934                 struct ahd_dma_seg *sg;
8935                 struct scsi_sense *sc;
8936                 struct ahd_initiator_tinfo *targ_info;
8937                 struct ahd_tmode_tstate *tstate;
8938                 struct ahd_transinfo *tinfo;
8939 #ifdef AHD_DEBUG
8940                 if (ahd_debug & AHD_SHOW_SENSE) {
8941                         ahd_print_path(ahd, scb);
8942                         printk("SCB %d: requests Check Status\n",
8943                                SCB_GET_TAG(scb));
8944                 }
8945 #endif
8946
8947                 if (ahd_perform_autosense(scb) == 0)
8948                         break;
8949
8950                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
8951                                     SCB_GET_TARGET(ahd, scb),
8952                                     SCB_GET_LUN(scb),
8953                                     SCB_GET_CHANNEL(ahd, scb),
8954                                     ROLE_INITIATOR);
8955                 targ_info = ahd_fetch_transinfo(ahd,
8956                                                 devinfo.channel,
8957                                                 devinfo.our_scsiid,
8958                                                 devinfo.target,
8959                                                 &tstate);
8960                 tinfo = &targ_info->curr;
8961                 sg = scb->sg_list;
8962                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
8963                 /*
8964                  * Save off the residual if there is one.
8965                  */
8966                 ahd_update_residual(ahd, scb);
8967 #ifdef AHD_DEBUG
8968                 if (ahd_debug & AHD_SHOW_SENSE) {
8969                         ahd_print_path(ahd, scb);
8970                         printk("Sending Sense\n");
8971                 }
8972 #endif
8973                 scb->sg_count = 0;
8974                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
8975                                   ahd_get_sense_bufsize(ahd, scb),
8976                                   /*last*/TRUE);
8977                 sc->opcode = REQUEST_SENSE;
8978                 sc->byte2 = 0;
8979                 if (tinfo->protocol_version <= SCSI_REV_2
8980                  && SCB_GET_LUN(scb) < 8)
8981                         sc->byte2 = SCB_GET_LUN(scb) << 5;
8982                 sc->unused[0] = 0;
8983                 sc->unused[1] = 0;
8984                 sc->length = ahd_get_sense_bufsize(ahd, scb);
8985                 sc->control = 0;
8986
8987                 /*
8988                  * We can't allow the target to disconnect.
8989                  * This will be an untagged transaction and
8990                  * having the target disconnect will make this
8991                  * transaction indestinguishable from outstanding
8992                  * tagged transactions.
8993                  */
8994                 hscb->control = 0;
8995
8996                 /*
8997                  * This request sense could be because the
8998                  * the device lost power or in some other
8999                  * way has lost our transfer negotiations.
9000                  * Renegotiate if appropriate.  Unit attention
9001                  * errors will be reported before any data
9002                  * phases occur.
9003                  */
9004                 if (ahd_get_residual(scb) == ahd_get_transfer_length(scb)) {
9005                         ahd_update_neg_request(ahd, &devinfo,
9006                                                tstate, targ_info,
9007                                                AHD_NEG_IF_NON_ASYNC);
9008                 }
9009                 if (tstate->auto_negotiate & devinfo.target_mask) {
9010                         hscb->control |= MK_MESSAGE;
9011                         scb->flags &=
9012                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
9013                         scb->flags |= SCB_AUTO_NEGOTIATE;
9014                 }
9015                 hscb->cdb_len = sizeof(*sc);
9016                 ahd_setup_data_scb(ahd, scb);
9017                 scb->flags |= SCB_SENSE;
9018                 ahd_queue_scb(ahd, scb);
9019                 break;
9020         }
9021         case SCSI_STATUS_OK:
9022                 printk("%s: Interrupted for status of 0???\n",
9023                        ahd_name(ahd));
9024                 fallthrough;
9025         default:
9026                 ahd_done(ahd, scb);
9027                 break;
9028         }
9029 }
9030
9031 static void
9032 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
9033 {
9034         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
9035                 ahd_handle_scsi_status(ahd, scb);
9036         } else {
9037                 ahd_calc_residual(ahd, scb);
9038                 ahd_done(ahd, scb);
9039         }
9040 }
9041
9042 /*
9043  * Calculate the residual for a just completed SCB.
9044  */
9045 static void
9046 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
9047 {
9048         struct hardware_scb *hscb;
9049         struct initiator_status *spkt;
9050         uint32_t sgptr;
9051         uint32_t resid_sgptr;
9052         uint32_t resid;
9053
9054         /*
9055          * 5 cases.
9056          * 1) No residual.
9057          *    SG_STATUS_VALID clear in sgptr.
9058          * 2) Transferless command
9059          * 3) Never performed any transfers.
9060          *    sgptr has SG_FULL_RESID set.
9061          * 4) No residual but target did not
9062          *    save data pointers after the
9063          *    last transfer, so sgptr was
9064          *    never updated.
9065          * 5) We have a partial residual.
9066          *    Use residual_sgptr to determine
9067          *    where we are.
9068          */
9069
9070         hscb = scb->hscb;
9071         sgptr = ahd_le32toh(hscb->sgptr);
9072         if ((sgptr & SG_STATUS_VALID) == 0)
9073                 /* Case 1 */
9074                 return;
9075         sgptr &= ~SG_STATUS_VALID;
9076
9077         if ((sgptr & SG_LIST_NULL) != 0)
9078                 /* Case 2 */
9079                 return;
9080
9081         /*
9082          * Residual fields are the same in both
9083          * target and initiator status packets,
9084          * so we can always use the initiator fields
9085          * regardless of the role for this SCB.
9086          */
9087         spkt = &hscb->shared_data.istatus;
9088         resid_sgptr = ahd_le32toh(spkt->residual_sgptr);
9089         if ((sgptr & SG_FULL_RESID) != 0) {
9090                 /* Case 3 */
9091                 resid = ahd_get_transfer_length(scb);
9092         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
9093                 /* Case 4 */
9094                 return;
9095         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
9096                 ahd_print_path(ahd, scb);
9097                 printk("data overrun detected Tag == 0x%x.\n",
9098                        SCB_GET_TAG(scb));
9099                 ahd_freeze_devq(ahd, scb);
9100                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
9101                 ahd_freeze_scb(scb);
9102                 return;
9103         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
9104                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
9105                 /* NOTREACHED */
9106         } else {
9107                 struct ahd_dma_seg *sg;
9108
9109                 /*
9110                  * Remainder of the SG where the transfer
9111                  * stopped.
9112                  */
9113                 resid = ahd_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
9114                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
9115
9116                 /* The residual sg_ptr always points to the next sg */
9117                 sg--;
9118
9119                 /*
9120                  * Add up the contents of all residual
9121                  * SG segments that are after the SG where
9122                  * the transfer stopped.
9123                  */
9124                 while ((ahd_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
9125                         sg++;
9126                         resid += ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
9127                 }
9128         }
9129         if ((scb->flags & SCB_SENSE) == 0)
9130                 ahd_set_residual(scb, resid);
9131         else
9132                 ahd_set_sense_residual(scb, resid);
9133
9134 #ifdef AHD_DEBUG
9135         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
9136                 ahd_print_path(ahd, scb);
9137                 printk("Handled %sResidual of %d bytes\n",
9138                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
9139         }
9140 #endif
9141 }
9142
9143 /******************************* Target Mode **********************************/
9144 #ifdef AHD_TARGET_MODE
9145 /*
9146  * Add a target mode event to this lun's queue
9147  */
9148 static void
9149 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
9150                        u_int initiator_id, u_int event_type, u_int event_arg)
9151 {
9152         struct ahd_tmode_event *event;
9153         int pending;
9154
9155         xpt_freeze_devq(lstate->path, /*count*/1);
9156         if (lstate->event_w_idx >= lstate->event_r_idx)
9157                 pending = lstate->event_w_idx - lstate->event_r_idx;
9158         else
9159                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
9160                         - (lstate->event_r_idx - lstate->event_w_idx);
9161
9162         if (event_type == EVENT_TYPE_BUS_RESET
9163          || event_type == MSG_BUS_DEV_RESET) {
9164                 /*
9165                  * Any earlier events are irrelevant, so reset our buffer.
9166                  * This has the effect of allowing us to deal with reset
9167                  * floods (an external device holding down the reset line)
9168                  * without losing the event that is really interesting.
9169                  */
9170                 lstate->event_r_idx = 0;
9171                 lstate->event_w_idx = 0;
9172                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
9173         }
9174
9175         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
9176                 xpt_print_path(lstate->path);
9177                 printk("immediate event %x:%x lost\n",
9178                        lstate->event_buffer[lstate->event_r_idx].event_type,
9179                        lstate->event_buffer[lstate->event_r_idx].event_arg);
9180                 lstate->event_r_idx++;
9181                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9182                         lstate->event_r_idx = 0;
9183                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
9184         }
9185
9186         event = &lstate->event_buffer[lstate->event_w_idx];
9187         event->initiator_id = initiator_id;
9188         event->event_type = event_type;
9189         event->event_arg = event_arg;
9190         lstate->event_w_idx++;
9191         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9192                 lstate->event_w_idx = 0;
9193 }
9194
9195 /*
9196  * Send any target mode events queued up waiting
9197  * for immediate notify resources.
9198  */
9199 void
9200 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
9201 {
9202         struct ccb_hdr *ccbh;
9203         struct ccb_immed_notify *inot;
9204
9205         while (lstate->event_r_idx != lstate->event_w_idx
9206             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
9207                 struct ahd_tmode_event *event;
9208
9209                 event = &lstate->event_buffer[lstate->event_r_idx];
9210                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
9211                 inot = (struct ccb_immed_notify *)ccbh;
9212                 switch (event->event_type) {
9213                 case EVENT_TYPE_BUS_RESET:
9214                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
9215                         break;
9216                 default:
9217                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
9218                         inot->message_args[0] = event->event_type;
9219                         inot->message_args[1] = event->event_arg;
9220                         break;
9221                 }
9222                 inot->initiator_id = event->initiator_id;
9223                 inot->sense_len = 0;
9224                 xpt_done((union ccb *)inot);
9225                 lstate->event_r_idx++;
9226                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9227                         lstate->event_r_idx = 0;
9228         }
9229 }
9230 #endif
9231
9232 /******************** Sequencer Program Patching/Download *********************/
9233
9234 #ifdef AHD_DUMP_SEQ
9235 void
9236 ahd_dumpseq(struct ahd_softc* ahd)
9237 {
9238         int i;
9239         int max_prog;
9240
9241         max_prog = 2048;
9242
9243         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9244         ahd_outw(ahd, PRGMCNT, 0);
9245         for (i = 0; i < max_prog; i++) {
9246                 uint8_t ins_bytes[4];
9247
9248                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
9249                 printk("0x%08x\n", ins_bytes[0] << 24
9250                                  | ins_bytes[1] << 16
9251                                  | ins_bytes[2] << 8
9252                                  | ins_bytes[3]);
9253         }
9254 }
9255 #endif
9256
9257 static void
9258 ahd_loadseq(struct ahd_softc *ahd)
9259 {
9260         struct  cs cs_table[NUM_CRITICAL_SECTIONS];
9261         u_int   begin_set[NUM_CRITICAL_SECTIONS];
9262         u_int   end_set[NUM_CRITICAL_SECTIONS];
9263         const struct patch *cur_patch;
9264         u_int   cs_count;
9265         u_int   cur_cs;
9266         u_int   i;
9267         int     downloaded;
9268         u_int   skip_addr;
9269         u_int   sg_prefetch_cnt;
9270         u_int   sg_prefetch_cnt_limit;
9271         u_int   sg_prefetch_align;
9272         u_int   sg_size;
9273         u_int   cacheline_mask;
9274         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
9275
9276         if (bootverbose)
9277                 printk("%s: Downloading Sequencer Program...",
9278                        ahd_name(ahd));
9279
9280 #if DOWNLOAD_CONST_COUNT != 8
9281 #error "Download Const Mismatch"
9282 #endif
9283         /*
9284          * Start out with 0 critical sections
9285          * that apply to this firmware load.
9286          */
9287         cs_count = 0;
9288         cur_cs = 0;
9289         memset(begin_set, 0, sizeof(begin_set));
9290         memset(end_set, 0, sizeof(end_set));
9291
9292         /*
9293          * Setup downloadable constant table.
9294          *
9295          * The computation for the S/G prefetch variables is
9296          * a bit complicated.  We would like to always fetch
9297          * in terms of cachelined sized increments.  However,
9298          * if the cacheline is not an even multiple of the
9299          * SG element size or is larger than our SG RAM, using
9300          * just the cache size might leave us with only a portion
9301          * of an SG element at the tail of a prefetch.  If the
9302          * cacheline is larger than our S/G prefetch buffer less
9303          * the size of an SG element, we may round down to a cacheline
9304          * that doesn't contain any or all of the S/G of interest
9305          * within the bounds of our S/G ram.  Provide variables to
9306          * the sequencer that will allow it to handle these edge
9307          * cases.
9308          */
9309         /* Start by aligning to the nearest cacheline. */
9310         sg_prefetch_align = ahd->pci_cachesize;
9311         if (sg_prefetch_align == 0)
9312                 sg_prefetch_align = 8;
9313         /* Round down to the nearest power of 2. */
9314         while (powerof2(sg_prefetch_align) == 0)
9315                 sg_prefetch_align--;
9316
9317         cacheline_mask = sg_prefetch_align - 1;
9318
9319         /*
9320          * If the cacheline boundary is greater than half our prefetch RAM
9321          * we risk not being able to fetch even a single complete S/G
9322          * segment if we align to that boundary.
9323          */
9324         if (sg_prefetch_align > CCSGADDR_MAX/2)
9325                 sg_prefetch_align = CCSGADDR_MAX/2;
9326         /* Start by fetching a single cacheline. */
9327         sg_prefetch_cnt = sg_prefetch_align;
9328         /*
9329          * Increment the prefetch count by cachelines until
9330          * at least one S/G element will fit.
9331          */
9332         sg_size = sizeof(struct ahd_dma_seg);
9333         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
9334                 sg_size = sizeof(struct ahd_dma64_seg);
9335         while (sg_prefetch_cnt < sg_size)
9336                 sg_prefetch_cnt += sg_prefetch_align;
9337         /*
9338          * If the cacheline is not an even multiple of
9339          * the S/G size, we may only get a partial S/G when
9340          * we align. Add a cacheline if this is the case.
9341          */
9342         if ((sg_prefetch_align % sg_size) != 0
9343          && (sg_prefetch_cnt < CCSGADDR_MAX))
9344                 sg_prefetch_cnt += sg_prefetch_align;
9345         /*
9346          * Lastly, compute a value that the sequencer can use
9347          * to determine if the remainder of the CCSGRAM buffer
9348          * has a full S/G element in it.
9349          */
9350         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
9351         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
9352         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
9353         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
9354         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
9355         download_consts[SG_SIZEOF] = sg_size;
9356         download_consts[PKT_OVERRUN_BUFOFFSET] =
9357                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
9358         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
9359         download_consts[CACHELINE_MASK] = cacheline_mask;
9360         cur_patch = patches;
9361         downloaded = 0;
9362         skip_addr = 0;
9363         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9364         ahd_outw(ahd, PRGMCNT, 0);
9365
9366         for (i = 0; i < sizeof(seqprog)/4; i++) {
9367                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
9368                         /*
9369                          * Don't download this instruction as it
9370                          * is in a patch that was removed.
9371                          */
9372                         continue;
9373                 }
9374                 /*
9375                  * Move through the CS table until we find a CS
9376                  * that might apply to this instruction.
9377                  */
9378                 for (; cur_cs < NUM_CRITICAL_SECTIONS; cur_cs++) {
9379                         if (critical_sections[cur_cs].end <= i) {
9380                                 if (begin_set[cs_count] == TRUE
9381                                  && end_set[cs_count] == FALSE) {
9382                                         cs_table[cs_count].end = downloaded;
9383                                         end_set[cs_count] = TRUE;
9384                                         cs_count++;
9385                                 }
9386                                 continue;
9387                         }
9388                         if (critical_sections[cur_cs].begin <= i
9389                          && begin_set[cs_count] == FALSE) {
9390                                 cs_table[cs_count].begin = downloaded;
9391                                 begin_set[cs_count] = TRUE;
9392                         }
9393                         break;
9394                 }
9395                 ahd_download_instr(ahd, i, download_consts);
9396                 downloaded++;
9397         }
9398
9399         ahd->num_critical_sections = cs_count;
9400         if (cs_count != 0) {
9401
9402                 cs_count *= sizeof(struct cs);
9403                 ahd->critical_sections = kmemdup(cs_table, cs_count, GFP_ATOMIC);
9404                 if (ahd->critical_sections == NULL)
9405                         panic("ahd_loadseq: Could not malloc");
9406         }
9407         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
9408
9409         if (bootverbose) {
9410                 printk(" %d instructions downloaded\n", downloaded);
9411                 printk("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
9412                        ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
9413         }
9414 }
9415
9416 static int
9417 ahd_check_patch(struct ahd_softc *ahd, const struct patch **start_patch,
9418                 u_int start_instr, u_int *skip_addr)
9419 {
9420         const struct patch *cur_patch;
9421         const struct patch *last_patch;
9422         u_int   num_patches;
9423
9424         num_patches = ARRAY_SIZE(patches);
9425         last_patch = &patches[num_patches];
9426         cur_patch = *start_patch;
9427
9428         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
9429
9430                 if (cur_patch->patch_func(ahd) == 0) {
9431
9432                         /* Start rejecting code */
9433                         *skip_addr = start_instr + cur_patch->skip_instr;
9434                         cur_patch += cur_patch->skip_patch;
9435                 } else {
9436                         /* Accepted this patch.  Advance to the next
9437                          * one and wait for our intruction pointer to
9438                          * hit this point.
9439                          */
9440                         cur_patch++;
9441                 }
9442         }
9443
9444         *start_patch = cur_patch;
9445         if (start_instr < *skip_addr)
9446                 /* Still skipping */
9447                 return (0);
9448
9449         return (1);
9450 }
9451
9452 static u_int
9453 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
9454 {
9455         const struct patch *cur_patch;
9456         int address_offset;
9457         u_int skip_addr;
9458         u_int i;
9459
9460         address_offset = 0;
9461         cur_patch = patches;
9462         skip_addr = 0;
9463
9464         for (i = 0; i < address;) {
9465
9466                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
9467
9468                 if (skip_addr > i) {
9469                         int end_addr;
9470
9471                         end_addr = min(address, skip_addr);
9472                         address_offset += end_addr - i;
9473                         i = skip_addr;
9474                 } else {
9475                         i++;
9476                 }
9477         }
9478         return (address - address_offset);
9479 }
9480
9481 static void
9482 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
9483 {
9484         union   ins_formats instr;
9485         struct  ins_format1 *fmt1_ins;
9486         struct  ins_format3 *fmt3_ins;
9487         u_int   opcode;
9488
9489         /*
9490          * The firmware is always compiled into a little endian format.
9491          */
9492         instr.integer = ahd_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
9493
9494         fmt1_ins = &instr.format1;
9495         fmt3_ins = NULL;
9496
9497         /* Pull the opcode */
9498         opcode = instr.format1.opcode;
9499         switch (opcode) {
9500         case AIC_OP_JMP:
9501         case AIC_OP_JC:
9502         case AIC_OP_JNC:
9503         case AIC_OP_CALL:
9504         case AIC_OP_JNE:
9505         case AIC_OP_JNZ:
9506         case AIC_OP_JE:
9507         case AIC_OP_JZ:
9508         {
9509                 fmt3_ins = &instr.format3;
9510                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
9511         }
9512                 fallthrough;
9513         case AIC_OP_OR:
9514         case AIC_OP_AND:
9515         case AIC_OP_XOR:
9516         case AIC_OP_ADD:
9517         case AIC_OP_ADC:
9518         case AIC_OP_BMOV:
9519                 if (fmt1_ins->parity != 0) {
9520                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
9521                 }
9522                 fmt1_ins->parity = 0;
9523                 fallthrough;
9524         case AIC_OP_ROL:
9525         {
9526                 int i, count;
9527
9528                 /* Calculate odd parity for the instruction */
9529                 for (i = 0, count = 0; i < 31; i++) {
9530                         uint32_t mask;
9531
9532                         mask = 0x01 << i;
9533                         if ((instr.integer & mask) != 0)
9534                                 count++;
9535                 }
9536                 if ((count & 0x01) == 0)
9537                         instr.format1.parity = 1;
9538
9539                 /* The sequencer is a little endian cpu */
9540                 instr.integer = ahd_htole32(instr.integer);
9541                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
9542                 break;
9543         }
9544         default:
9545                 panic("Unknown opcode encountered in seq program");
9546                 break;
9547         }
9548 }
9549
9550 static int
9551 ahd_probe_stack_size(struct ahd_softc *ahd)
9552 {
9553         int last_probe;
9554
9555         last_probe = 0;
9556         while (1) {
9557                 int i;
9558
9559                 /*
9560                  * We avoid using 0 as a pattern to avoid
9561                  * confusion if the stack implementation
9562                  * "back-fills" with zeros when "poping'
9563                  * entries.
9564                  */
9565                 for (i = 1; i <= last_probe+1; i++) {
9566                        ahd_outb(ahd, STACK, i & 0xFF);
9567                        ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
9568                 }
9569
9570                 /* Verify */
9571                 for (i = last_probe+1; i > 0; i--) {
9572                         u_int stack_entry;
9573
9574                         stack_entry = ahd_inb(ahd, STACK)
9575                                     |(ahd_inb(ahd, STACK) << 8);
9576                         if (stack_entry != i)
9577                                 goto sized;
9578                 }
9579                 last_probe++;
9580         }
9581 sized:
9582         return (last_probe);
9583 }
9584
9585 int
9586 ahd_print_register(const ahd_reg_parse_entry_t *table, u_int num_entries,
9587                    const char *name, u_int address, u_int value,
9588                    u_int *cur_column, u_int wrap_point)
9589 {
9590         int     printed;
9591         u_int   printed_mask;
9592
9593         if (cur_column != NULL && *cur_column >= wrap_point) {
9594                 printk("\n");
9595                 *cur_column = 0;
9596         }
9597         printed = printk("%s[0x%x]", name, value);
9598         if (table == NULL) {
9599                 printed += printk(" ");
9600                 *cur_column += printed;
9601                 return (printed);
9602         }
9603         printed_mask = 0;
9604         while (printed_mask != 0xFF) {
9605                 int entry;
9606
9607                 for (entry = 0; entry < num_entries; entry++) {
9608                         if (((value & table[entry].mask)
9609                           != table[entry].value)
9610                          || ((printed_mask & table[entry].mask)
9611                           == table[entry].mask))
9612                                 continue;
9613
9614                         printed += printk("%s%s",
9615                                           printed_mask == 0 ? ":(" : "|",
9616                                           table[entry].name);
9617                         printed_mask |= table[entry].mask;
9618
9619                         break;
9620                 }
9621                 if (entry >= num_entries)
9622                         break;
9623         }
9624         if (printed_mask != 0)
9625                 printed += printk(") ");
9626         else
9627                 printed += printk(" ");
9628         if (cur_column != NULL)
9629                 *cur_column += printed;
9630         return (printed);
9631 }
9632
9633 void
9634 ahd_dump_card_state(struct ahd_softc *ahd)
9635 {
9636         struct scb      *scb;
9637         ahd_mode_state   saved_modes;
9638         u_int            dffstat;
9639         int              paused;
9640         u_int            scb_index;
9641         u_int            saved_scb_index;
9642         u_int            cur_col;
9643         int              i;
9644
9645         if (ahd_is_paused(ahd)) {
9646                 paused = 1;
9647         } else {
9648                 paused = 0;
9649                 ahd_pause(ahd);
9650         }
9651         saved_modes = ahd_save_modes(ahd);
9652         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9653         printk(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
9654                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
9655                ahd_name(ahd),
9656                ahd_inw(ahd, CURADDR),
9657                ahd_build_mode_state(ahd, ahd->saved_src_mode,
9658                                     ahd->saved_dst_mode));
9659         if (paused)
9660                 printk("Card was paused\n");
9661
9662         if (ahd_check_cmdcmpltqueues(ahd))
9663                 printk("Completions are pending\n");
9664
9665         /*
9666          * Mode independent registers.
9667          */
9668         cur_col = 0;
9669         ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
9670         ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
9671         ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
9672         ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
9673         ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
9674         ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
9675         ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
9676         ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
9677         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
9678         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
9679         ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
9680         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
9681         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
9682         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
9683         ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
9684         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
9685         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
9686         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
9687         ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
9688         ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
9689                                        &cur_col, 50);
9690         ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
9691         ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
9692                                     &cur_col, 50);
9693         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
9694         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
9695         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
9696         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
9697         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
9698         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
9699         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
9700         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
9701         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
9702         ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
9703         ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
9704         ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
9705         printk("\n");
9706         printk("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
9707                "CURRSCB 0x%x NEXTSCB 0x%x\n",
9708                ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
9709                ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
9710                ahd_inw(ahd, NEXTSCB));
9711         cur_col = 0;
9712         /* QINFIFO */
9713         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
9714                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
9715                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
9716         saved_scb_index = ahd_get_scbptr(ahd);
9717         printk("Pending list:");
9718         i = 0;
9719         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9720                 if (i++ > AHD_SCB_MAX)
9721                         break;
9722                 cur_col = printk("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
9723                                  ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
9724                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9725                 ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
9726                                       &cur_col, 60);
9727                 ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
9728                                      &cur_col, 60);
9729         }
9730         printk("\nTotal %d\n", i);
9731
9732         printk("Kernel Free SCB list: ");
9733         i = 0;
9734         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
9735                 struct scb *list_scb;
9736
9737                 list_scb = scb;
9738                 do {
9739                         printk("%d ", SCB_GET_TAG(list_scb));
9740                         list_scb = LIST_NEXT(list_scb, collision_links);
9741                 } while (list_scb && i++ < AHD_SCB_MAX);
9742         }
9743
9744         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
9745                 if (i++ > AHD_SCB_MAX)
9746                         break;
9747                 printk("%d ", SCB_GET_TAG(scb));
9748         }
9749         printk("\n");
9750
9751         printk("Sequencer Complete DMA-inprog list: ");
9752         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9753         i = 0;
9754         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9755                 ahd_set_scbptr(ahd, scb_index);
9756                 printk("%d ", scb_index);
9757                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9758         }
9759         printk("\n");
9760
9761         printk("Sequencer Complete list: ");
9762         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9763         i = 0;
9764         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9765                 ahd_set_scbptr(ahd, scb_index);
9766                 printk("%d ", scb_index);
9767                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9768         }
9769         printk("\n");
9770
9771
9772         printk("Sequencer DMA-Up and Complete list: ");
9773         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9774         i = 0;
9775         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9776                 ahd_set_scbptr(ahd, scb_index);
9777                 printk("%d ", scb_index);
9778                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9779         }
9780         printk("\n");
9781         printk("Sequencer On QFreeze and Complete list: ");
9782         scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9783         i = 0;
9784         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9785                 ahd_set_scbptr(ahd, scb_index);
9786                 printk("%d ", scb_index);
9787                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9788         }
9789         printk("\n");
9790         ahd_set_scbptr(ahd, saved_scb_index);
9791         dffstat = ahd_inb(ahd, DFFSTAT);
9792         for (i = 0; i < 2; i++) {
9793 #ifdef AHD_DEBUG
9794                 struct scb *fifo_scb;
9795 #endif
9796                 u_int       fifo_scbptr;
9797
9798                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9799                 fifo_scbptr = ahd_get_scbptr(ahd);
9800                 printk("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9801                        ahd_name(ahd), i,
9802                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9803                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9804                 cur_col = 0;
9805                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9806                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9807                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9808                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9809                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9810                                           &cur_col, 50);
9811                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9812                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9813                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9814                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9815                 if (cur_col > 50) {
9816                         printk("\n");
9817                         cur_col = 0;
9818                 }
9819                 cur_col += printk("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9820                                   ahd_inl(ahd, SHADDR+4),
9821                                   ahd_inl(ahd, SHADDR),
9822                                   (ahd_inb(ahd, SHCNT)
9823                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
9824                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
9825                 if (cur_col > 50) {
9826                         printk("\n");
9827                         cur_col = 0;
9828                 }
9829                 cur_col += printk("HADDR = 0x%x%x, HCNT = 0x%x ",
9830                                   ahd_inl(ahd, HADDR+4),
9831                                   ahd_inl(ahd, HADDR),
9832                                   (ahd_inb(ahd, HCNT)
9833                                 | (ahd_inb(ahd, HCNT + 1) << 8)
9834                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
9835                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9836 #ifdef AHD_DEBUG
9837                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
9838                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9839                         if (fifo_scb != NULL)
9840                                 ahd_dump_sglist(fifo_scb);
9841                 }
9842 #endif
9843         }
9844         printk("\nLQIN: ");
9845         for (i = 0; i < 20; i++)
9846                 printk("0x%x ", ahd_inb(ahd, LQIN + i));
9847         printk("\n");
9848         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9849         printk("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9850                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9851                ahd_inb(ahd, OPTIONMODE));
9852         printk("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9853                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9854                ahd_inb(ahd, MAXCMDCNT));
9855         printk("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9856                ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9857                ahd_inb(ahd, SAVED_LUN));
9858         ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9859         printk("\n");
9860         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9861         cur_col = 0;
9862         ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9863         printk("\n");
9864         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9865         printk("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9866                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9867                ahd_inw(ahd, DINDEX));
9868         printk("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9869                ahd_name(ahd), ahd_get_scbptr(ahd),
9870                ahd_inw_scbram(ahd, SCB_NEXT),
9871                ahd_inw_scbram(ahd, SCB_NEXT2));
9872         printk("CDB %x %x %x %x %x %x\n",
9873                ahd_inb_scbram(ahd, SCB_CDB_STORE),
9874                ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9875                ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9876                ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9877                ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9878                ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9879         printk("STACK:");
9880         for (i = 0; i < ahd->stack_size; i++) {
9881                 ahd->saved_stack[i] =
9882                     ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9883                 printk(" 0x%x", ahd->saved_stack[i]);
9884         }
9885         for (i = ahd->stack_size-1; i >= 0; i--) {
9886                 ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9887                 ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9888         }
9889         printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9890         ahd_restore_modes(ahd, saved_modes);
9891         if (paused == 0)
9892                 ahd_unpause(ahd);
9893 }
9894
9895 #if 0
9896 void
9897 ahd_dump_scbs(struct ahd_softc *ahd)
9898 {
9899         ahd_mode_state saved_modes;
9900         u_int          saved_scb_index;
9901         int            i;
9902
9903         saved_modes = ahd_save_modes(ahd);
9904         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9905         saved_scb_index = ahd_get_scbptr(ahd);
9906         for (i = 0; i < AHD_SCB_MAX; i++) {
9907                 ahd_set_scbptr(ahd, i);
9908                 printk("%3d", i);
9909                 printk("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9910                        ahd_inb_scbram(ahd, SCB_CONTROL),
9911                        ahd_inb_scbram(ahd, SCB_SCSIID),
9912                        ahd_inw_scbram(ahd, SCB_NEXT),
9913                        ahd_inw_scbram(ahd, SCB_NEXT2),
9914                        ahd_inl_scbram(ahd, SCB_SGPTR),
9915                        ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9916         }
9917         printk("\n");
9918         ahd_set_scbptr(ahd, saved_scb_index);
9919         ahd_restore_modes(ahd, saved_modes);
9920 }
9921 #endif  /*  0  */
9922
9923 /**************************** Flexport Logic **********************************/
9924 /*
9925  * Read count 16bit words from 16bit word address start_addr from the
9926  * SEEPROM attached to the controller, into buf, using the controller's
9927  * SEEPROM reading state machine.  Optionally treat the data as a byte
9928  * stream in terms of byte order.
9929  */
9930 int
9931 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9932                  u_int start_addr, u_int count, int bytestream)
9933 {
9934         u_int cur_addr;
9935         u_int end_addr;
9936         int   error;
9937
9938         /*
9939          * If we never make it through the loop even once,
9940          * we were passed invalid arguments.
9941          */
9942         error = EINVAL;
9943         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9944         end_addr = start_addr + count;
9945         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9946
9947                 ahd_outb(ahd, SEEADR, cur_addr);
9948                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
9949
9950                 error = ahd_wait_seeprom(ahd);
9951                 if (error)
9952                         break;
9953                 if (bytestream != 0) {
9954                         uint8_t *bytestream_ptr;
9955
9956                         bytestream_ptr = (uint8_t *)buf;
9957                         *bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
9958                         *bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
9959                 } else {
9960                         /*
9961                          * ahd_inw() already handles machine byte order.
9962                          */
9963                         *buf = ahd_inw(ahd, SEEDAT);
9964                 }
9965                 buf++;
9966         }
9967         return (error);
9968 }
9969
9970 /*
9971  * Write count 16bit words from buf, into SEEPROM attache to the
9972  * controller starting at 16bit word address start_addr, using the
9973  * controller's SEEPROM writing state machine.
9974  */
9975 int
9976 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9977                   u_int start_addr, u_int count)
9978 {
9979         u_int cur_addr;
9980         u_int end_addr;
9981         int   error;
9982         int   retval;
9983
9984         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9985         error = ENOENT;
9986
9987         /* Place the chip into write-enable mode */
9988         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
9989         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
9990         error = ahd_wait_seeprom(ahd);
9991         if (error)
9992                 return (error);
9993
9994         /*
9995          * Write the data.  If we don't get through the loop at
9996          * least once, the arguments were invalid.
9997          */
9998         retval = EINVAL;
9999         end_addr = start_addr + count;
10000         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
10001                 ahd_outw(ahd, SEEDAT, *buf++);
10002                 ahd_outb(ahd, SEEADR, cur_addr);
10003                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
10004
10005                 retval = ahd_wait_seeprom(ahd);
10006                 if (retval)
10007                         break;
10008         }
10009
10010         /*
10011          * Disable writes.
10012          */
10013         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
10014         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
10015         error = ahd_wait_seeprom(ahd);
10016         if (error)
10017                 return (error);
10018         return (retval);
10019 }
10020
10021 /*
10022  * Wait ~100us for the serial eeprom to satisfy our request.
10023  */
10024 static int
10025 ahd_wait_seeprom(struct ahd_softc *ahd)
10026 {
10027         int cnt;
10028
10029         cnt = 5000;
10030         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
10031                 ahd_delay(5);
10032
10033         if (cnt == 0)
10034                 return (ETIMEDOUT);
10035         return (0);
10036 }
10037
10038 /*
10039  * Validate the two checksums in the per_channel
10040  * vital product data struct.
10041  */
10042 static int
10043 ahd_verify_vpd_cksum(struct vpd_config *vpd)
10044 {
10045         int i;
10046         int maxaddr;
10047         uint32_t checksum;
10048         uint8_t *vpdarray;
10049
10050         vpdarray = (uint8_t *)vpd;
10051         maxaddr = offsetof(struct vpd_config, vpd_checksum);
10052         checksum = 0;
10053         for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
10054                 checksum = checksum + vpdarray[i];
10055         if (checksum == 0
10056          || (-checksum & 0xFF) != vpd->vpd_checksum)
10057                 return (0);
10058
10059         checksum = 0;
10060         maxaddr = offsetof(struct vpd_config, checksum);
10061         for (i = offsetof(struct vpd_config, default_target_flags);
10062              i < maxaddr; i++)
10063                 checksum = checksum + vpdarray[i];
10064         if (checksum == 0
10065          || (-checksum & 0xFF) != vpd->checksum)
10066                 return (0);
10067         return (1);
10068 }
10069
10070 int
10071 ahd_verify_cksum(struct seeprom_config *sc)
10072 {
10073         int i;
10074         int maxaddr;
10075         uint32_t checksum;
10076         uint16_t *scarray;
10077
10078         maxaddr = (sizeof(*sc)/2) - 1;
10079         checksum = 0;
10080         scarray = (uint16_t *)sc;
10081
10082         for (i = 0; i < maxaddr; i++)
10083                 checksum = checksum + scarray[i];
10084         if (checksum == 0
10085          || (checksum & 0xFFFF) != sc->checksum) {
10086                 return (0);
10087         } else {
10088                 return (1);
10089         }
10090 }
10091
10092 int
10093 ahd_acquire_seeprom(struct ahd_softc *ahd)
10094 {
10095         /*
10096          * We should be able to determine the SEEPROM type
10097          * from the flexport logic, but unfortunately not
10098          * all implementations have this logic and there is
10099          * no programatic method for determining if the logic
10100          * is present.
10101          */
10102         return (1);
10103 #if 0
10104         uint8_t seetype;
10105         int     error;
10106
10107         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
10108         if (error != 0
10109          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
10110                 return (0);
10111         return (1);
10112 #endif
10113 }
10114
10115 void
10116 ahd_release_seeprom(struct ahd_softc *ahd)
10117 {
10118         /* Currently a no-op */
10119 }
10120
10121 /*
10122  * Wait at most 2 seconds for flexport arbitration to succeed.
10123  */
10124 static int
10125 ahd_wait_flexport(struct ahd_softc *ahd)
10126 {
10127         int cnt;
10128
10129         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10130         cnt = 1000000 * 2 / 5;
10131         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
10132                 ahd_delay(5);
10133
10134         if (cnt == 0)
10135                 return (ETIMEDOUT);
10136         return (0);
10137 }
10138
10139 int
10140 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
10141 {
10142         int error;
10143
10144         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10145         if (addr > 7)
10146                 panic("ahd_write_flexport: address out of range");
10147         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10148         error = ahd_wait_flexport(ahd);
10149         if (error != 0)
10150                 return (error);
10151         ahd_outb(ahd, BRDDAT, value);
10152         ahd_flush_device_writes(ahd);
10153         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
10154         ahd_flush_device_writes(ahd);
10155         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10156         ahd_flush_device_writes(ahd);
10157         ahd_outb(ahd, BRDCTL, 0);
10158         ahd_flush_device_writes(ahd);
10159         return (0);
10160 }
10161
10162 int
10163 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
10164 {
10165         int     error;
10166
10167         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10168         if (addr > 7)
10169                 panic("ahd_read_flexport: address out of range");
10170         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
10171         error = ahd_wait_flexport(ahd);
10172         if (error != 0)
10173                 return (error);
10174         *value = ahd_inb(ahd, BRDDAT);
10175         ahd_outb(ahd, BRDCTL, 0);
10176         ahd_flush_device_writes(ahd);
10177         return (0);
10178 }
10179
10180 /************************* Target Mode ****************************************/
10181 #ifdef AHD_TARGET_MODE
10182 cam_status
10183 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
10184                     struct ahd_tmode_tstate **tstate,
10185                     struct ahd_tmode_lstate **lstate,
10186                     int notfound_failure)
10187 {
10188
10189         if ((ahd->features & AHD_TARGETMODE) == 0)
10190                 return (CAM_REQ_INVALID);
10191
10192         /*
10193          * Handle the 'black hole' device that sucks up
10194          * requests to unattached luns on enabled targets.
10195          */
10196         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
10197          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
10198                 *tstate = NULL;
10199                 *lstate = ahd->black_hole;
10200         } else {
10201                 u_int max_id;
10202
10203                 max_id = (ahd->features & AHD_WIDE) ? 16 : 8;
10204                 if (ccb->ccb_h.target_id >= max_id)
10205                         return (CAM_TID_INVALID);
10206
10207                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
10208                         return (CAM_LUN_INVALID);
10209
10210                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
10211                 *lstate = NULL;
10212                 if (*tstate != NULL)
10213                         *lstate =
10214                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
10215         }
10216
10217         if (notfound_failure != 0 && *lstate == NULL)
10218                 return (CAM_PATH_INVALID);
10219
10220         return (CAM_REQ_CMP);
10221 }
10222
10223 void
10224 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
10225 {
10226 #if NOT_YET
10227         struct     ahd_tmode_tstate *tstate;
10228         struct     ahd_tmode_lstate *lstate;
10229         struct     ccb_en_lun *cel;
10230         cam_status status;
10231         u_int      target;
10232         u_int      lun;
10233         u_int      target_mask;
10234         u_long     s;
10235         char       channel;
10236
10237         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
10238                                      /*notfound_failure*/FALSE);
10239
10240         if (status != CAM_REQ_CMP) {
10241                 ccb->ccb_h.status = status;
10242                 return;
10243         }
10244
10245         if ((ahd->features & AHD_MULTIROLE) != 0) {
10246                 u_int      our_id;
10247
10248                 our_id = ahd->our_id;
10249                 if (ccb->ccb_h.target_id != our_id) {
10250                         if ((ahd->features & AHD_MULTI_TID) != 0
10251                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
10252                                 /*
10253                                  * Only allow additional targets if
10254                                  * the initiator role is disabled.
10255                                  * The hardware cannot handle a re-select-in
10256                                  * on the initiator id during a re-select-out
10257                                  * on a different target id.
10258                                  */
10259                                 status = CAM_TID_INVALID;
10260                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
10261                                 || ahd->enabled_luns > 0) {
10262                                 /*
10263                                  * Only allow our target id to change
10264                                  * if the initiator role is not configured
10265                                  * and there are no enabled luns which
10266                                  * are attached to the currently registered
10267                                  * scsi id.
10268                                  */
10269                                 status = CAM_TID_INVALID;
10270                         }
10271                 }
10272         }
10273
10274         if (status != CAM_REQ_CMP) {
10275                 ccb->ccb_h.status = status;
10276                 return;
10277         }
10278
10279         /*
10280          * We now have an id that is valid.
10281          * If we aren't in target mode, switch modes.
10282          */
10283         if ((ahd->flags & AHD_TARGETROLE) == 0
10284          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
10285                 u_long  s;
10286
10287                 printk("Configuring Target Mode\n");
10288                 ahd_lock(ahd, &s);
10289                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
10290                         ccb->ccb_h.status = CAM_BUSY;
10291                         ahd_unlock(ahd, &s);
10292                         return;
10293                 }
10294                 ahd->flags |= AHD_TARGETROLE;
10295                 if ((ahd->features & AHD_MULTIROLE) == 0)
10296                         ahd->flags &= ~AHD_INITIATORROLE;
10297                 ahd_pause(ahd);
10298                 ahd_loadseq(ahd);
10299                 ahd_restart(ahd);
10300                 ahd_unlock(ahd, &s);
10301         }
10302         cel = &ccb->cel;
10303         target = ccb->ccb_h.target_id;
10304         lun = ccb->ccb_h.target_lun;
10305         channel = SIM_CHANNEL(ahd, sim);
10306         target_mask = 0x01 << target;
10307         if (channel == 'B')
10308                 target_mask <<= 8;
10309
10310         if (cel->enable != 0) {
10311                 u_int scsiseq1;
10312
10313                 /* Are we already enabled?? */
10314                 if (lstate != NULL) {
10315                         xpt_print_path(ccb->ccb_h.path);
10316                         printk("Lun already enabled\n");
10317                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
10318                         return;
10319                 }
10320
10321                 if (cel->grp6_len != 0
10322                  || cel->grp7_len != 0) {
10323                         /*
10324                          * Don't (yet?) support vendor
10325                          * specific commands.
10326                          */
10327                         ccb->ccb_h.status = CAM_REQ_INVALID;
10328                         printk("Non-zero Group Codes\n");
10329                         return;
10330                 }
10331
10332                 /*
10333                  * Seems to be okay.
10334                  * Setup our data structures.
10335                  */
10336                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
10337                         tstate = ahd_alloc_tstate(ahd, target, channel);
10338                         if (tstate == NULL) {
10339                                 xpt_print_path(ccb->ccb_h.path);
10340                                 printk("Couldn't allocate tstate\n");
10341                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10342                                 return;
10343                         }
10344                 }
10345                 lstate = kzalloc(sizeof(*lstate), GFP_ATOMIC);
10346                 if (lstate == NULL) {
10347                         xpt_print_path(ccb->ccb_h.path);
10348                         printk("Couldn't allocate lstate\n");
10349                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10350                         return;
10351                 }
10352                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
10353                                          xpt_path_path_id(ccb->ccb_h.path),
10354                                          xpt_path_target_id(ccb->ccb_h.path),
10355                                          xpt_path_lun_id(ccb->ccb_h.path));
10356                 if (status != CAM_REQ_CMP) {
10357                         kfree(lstate);
10358                         xpt_print_path(ccb->ccb_h.path);
10359                         printk("Couldn't allocate path\n");
10360                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10361                         return;
10362                 }
10363                 SLIST_INIT(&lstate->accept_tios);
10364                 SLIST_INIT(&lstate->immed_notifies);
10365                 ahd_lock(ahd, &s);
10366                 ahd_pause(ahd);
10367                 if (target != CAM_TARGET_WILDCARD) {
10368                         tstate->enabled_luns[lun] = lstate;
10369                         ahd->enabled_luns++;
10370
10371                         if ((ahd->features & AHD_MULTI_TID) != 0) {
10372                                 u_int targid_mask;
10373
10374                                 targid_mask = ahd_inw(ahd, TARGID);
10375                                 targid_mask |= target_mask;
10376                                 ahd_outw(ahd, TARGID, targid_mask);
10377                                 ahd_update_scsiid(ahd, targid_mask);
10378                         } else {
10379                                 u_int our_id;
10380                                 char  channel;
10381
10382                                 channel = SIM_CHANNEL(ahd, sim);
10383                                 our_id = SIM_SCSI_ID(ahd, sim);
10384
10385                                 /*
10386                                  * This can only happen if selections
10387                                  * are not enabled
10388                                  */
10389                                 if (target != our_id) {
10390                                         u_int sblkctl;
10391                                         char  cur_channel;
10392                                         int   swap;
10393
10394                                         sblkctl = ahd_inb(ahd, SBLKCTL);
10395                                         cur_channel = (sblkctl & SELBUSB)
10396                                                     ? 'B' : 'A';
10397                                         if ((ahd->features & AHD_TWIN) == 0)
10398                                                 cur_channel = 'A';
10399                                         swap = cur_channel != channel;
10400                                         ahd->our_id = target;
10401
10402                                         if (swap)
10403                                                 ahd_outb(ahd, SBLKCTL,
10404                                                          sblkctl ^ SELBUSB);
10405
10406                                         ahd_outb(ahd, SCSIID, target);
10407
10408                                         if (swap)
10409                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
10410                                 }
10411                         }
10412                 } else
10413                         ahd->black_hole = lstate;
10414                 /* Allow select-in operations */
10415                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
10416                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10417                         scsiseq1 |= ENSELI;
10418                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10419                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10420                         scsiseq1 |= ENSELI;
10421                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10422                 }
10423                 ahd_unpause(ahd);
10424                 ahd_unlock(ahd, &s);
10425                 ccb->ccb_h.status = CAM_REQ_CMP;
10426                 xpt_print_path(ccb->ccb_h.path);
10427                 printk("Lun now enabled for target mode\n");
10428         } else {
10429                 struct scb *scb;
10430                 int i, empty;
10431
10432                 if (lstate == NULL) {
10433                         ccb->ccb_h.status = CAM_LUN_INVALID;
10434                         return;
10435                 }
10436
10437                 ahd_lock(ahd, &s);
10438
10439                 ccb->ccb_h.status = CAM_REQ_CMP;
10440                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
10441                         struct ccb_hdr *ccbh;
10442
10443                         ccbh = &scb->io_ctx->ccb_h;
10444                         if (ccbh->func_code == XPT_CONT_TARGET_IO
10445                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
10446                                 printk("CTIO pending\n");
10447                                 ccb->ccb_h.status = CAM_REQ_INVALID;
10448                                 ahd_unlock(ahd, &s);
10449                                 return;
10450                         }
10451                 }
10452
10453                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
10454                         printk("ATIOs pending\n");
10455                         ccb->ccb_h.status = CAM_REQ_INVALID;
10456                 }
10457
10458                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
10459                         printk("INOTs pending\n");
10460                         ccb->ccb_h.status = CAM_REQ_INVALID;
10461                 }
10462
10463                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
10464                         ahd_unlock(ahd, &s);
10465                         return;
10466                 }
10467
10468                 xpt_print_path(ccb->ccb_h.path);
10469                 printk("Target mode disabled\n");
10470                 xpt_free_path(lstate->path);
10471                 kfree(lstate);
10472
10473                 ahd_pause(ahd);
10474                 /* Can we clean up the target too? */
10475                 if (target != CAM_TARGET_WILDCARD) {
10476                         tstate->enabled_luns[lun] = NULL;
10477                         ahd->enabled_luns--;
10478                         for (empty = 1, i = 0; i < 8; i++)
10479                                 if (tstate->enabled_luns[i] != NULL) {
10480                                         empty = 0;
10481                                         break;
10482                                 }
10483
10484                         if (empty) {
10485                                 ahd_free_tstate(ahd, target, channel,
10486                                                 /*force*/FALSE);
10487                                 if (ahd->features & AHD_MULTI_TID) {
10488                                         u_int targid_mask;
10489
10490                                         targid_mask = ahd_inw(ahd, TARGID);
10491                                         targid_mask &= ~target_mask;
10492                                         ahd_outw(ahd, TARGID, targid_mask);
10493                                         ahd_update_scsiid(ahd, targid_mask);
10494                                 }
10495                         }
10496                 } else {
10497
10498                         ahd->black_hole = NULL;
10499
10500                         /*
10501                          * We can't allow selections without
10502                          * our black hole device.
10503                          */
10504                         empty = TRUE;
10505                 }
10506                 if (ahd->enabled_luns == 0) {
10507                         /* Disallow select-in */
10508                         u_int scsiseq1;
10509
10510                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10511                         scsiseq1 &= ~ENSELI;
10512                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10513                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10514                         scsiseq1 &= ~ENSELI;
10515                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
10516
10517                         if ((ahd->features & AHD_MULTIROLE) == 0) {
10518                                 printk("Configuring Initiator Mode\n");
10519                                 ahd->flags &= ~AHD_TARGETROLE;
10520                                 ahd->flags |= AHD_INITIATORROLE;
10521                                 ahd_pause(ahd);
10522                                 ahd_loadseq(ahd);
10523                                 ahd_restart(ahd);
10524                                 /*
10525                                  * Unpaused.  The extra unpause
10526                                  * that follows is harmless.
10527                                  */
10528                         }
10529                 }
10530                 ahd_unpause(ahd);
10531                 ahd_unlock(ahd, &s);
10532         }
10533 #endif
10534 }
10535
10536 static void
10537 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
10538 {
10539 #if NOT_YET
10540         u_int scsiid_mask;
10541         u_int scsiid;
10542
10543         if ((ahd->features & AHD_MULTI_TID) == 0)
10544                 panic("ahd_update_scsiid called on non-multitid unit\n");
10545
10546         /*
10547          * Since we will rely on the TARGID mask
10548          * for selection enables, ensure that OID
10549          * in SCSIID is not set to some other ID
10550          * that we don't want to allow selections on.
10551          */
10552         if ((ahd->features & AHD_ULTRA2) != 0)
10553                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
10554         else
10555                 scsiid = ahd_inb(ahd, SCSIID);
10556         scsiid_mask = 0x1 << (scsiid & OID);
10557         if ((targid_mask & scsiid_mask) == 0) {
10558                 u_int our_id;
10559
10560                 /* ffs counts from 1 */
10561                 our_id = ffs(targid_mask);
10562                 if (our_id == 0)
10563                         our_id = ahd->our_id;
10564                 else
10565                         our_id--;
10566                 scsiid &= TID;
10567                 scsiid |= our_id;
10568         }
10569         if ((ahd->features & AHD_ULTRA2) != 0)
10570                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
10571         else
10572                 ahd_outb(ahd, SCSIID, scsiid);
10573 #endif
10574 }
10575
10576 static void
10577 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
10578 {
10579         struct target_cmd *cmd;
10580
10581         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
10582         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
10583
10584                 /*
10585                  * Only advance through the queue if we
10586                  * have the resources to process the command.
10587                  */
10588                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
10589                         break;
10590
10591                 cmd->cmd_valid = 0;
10592                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
10593                                 ahd->shared_data_map.dmamap,
10594                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
10595                                 sizeof(struct target_cmd),
10596                                 BUS_DMASYNC_PREREAD);
10597                 ahd->tqinfifonext++;
10598
10599                 /*
10600                  * Lazily update our position in the target mode incoming
10601                  * command queue as seen by the sequencer.
10602                  */
10603                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
10604                         u_int hs_mailbox;
10605
10606                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
10607                         hs_mailbox &= ~HOST_TQINPOS;
10608                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
10609                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
10610                 }
10611         }
10612 }
10613
10614 static int
10615 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
10616 {
10617         struct    ahd_tmode_tstate *tstate;
10618         struct    ahd_tmode_lstate *lstate;
10619         struct    ccb_accept_tio *atio;
10620         uint8_t *byte;
10621         int       initiator;
10622         int       target;
10623         int       lun;
10624
10625         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
10626         target = SCSIID_OUR_ID(cmd->scsiid);
10627         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
10628
10629         byte = cmd->bytes;
10630         tstate = ahd->enabled_targets[target];
10631         lstate = NULL;
10632         if (tstate != NULL)
10633                 lstate = tstate->enabled_luns[lun];
10634
10635         /*
10636          * Commands for disabled luns go to the black hole driver.
10637          */
10638         if (lstate == NULL)
10639                 lstate = ahd->black_hole;
10640
10641         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
10642         if (atio == NULL) {
10643                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
10644                 /*
10645                  * Wait for more ATIOs from the peripheral driver for this lun.
10646                  */
10647                 return (1);
10648         } else
10649                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
10650 #ifdef AHD_DEBUG
10651         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10652                 printk("Incoming command from %d for %d:%d%s\n",
10653                        initiator, target, lun,
10654                        lstate == ahd->black_hole ? "(Black Holed)" : "");
10655 #endif
10656         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
10657
10658         if (lstate == ahd->black_hole) {
10659                 /* Fill in the wildcards */
10660                 atio->ccb_h.target_id = target;
10661                 atio->ccb_h.target_lun = lun;
10662         }
10663
10664         /*
10665          * Package it up and send it off to
10666          * whomever has this lun enabled.
10667          */
10668         atio->sense_len = 0;
10669         atio->init_id = initiator;
10670         if (byte[0] != 0xFF) {
10671                 /* Tag was included */
10672                 atio->tag_action = *byte++;
10673                 atio->tag_id = *byte++;
10674                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
10675         } else {
10676                 atio->ccb_h.flags = 0;
10677         }
10678         byte++;
10679
10680         /* Okay.  Now determine the cdb size based on the command code */
10681         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
10682         case 0:
10683                 atio->cdb_len = 6;
10684                 break;
10685         case 1:
10686         case 2:
10687                 atio->cdb_len = 10;
10688                 break;
10689         case 4:
10690                 atio->cdb_len = 16;
10691                 break;
10692         case 5:
10693                 atio->cdb_len = 12;
10694                 break;
10695         case 3:
10696         default:
10697                 /* Only copy the opcode. */
10698                 atio->cdb_len = 1;
10699                 printk("Reserved or VU command code type encountered\n");
10700                 break;
10701         }
10702
10703         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
10704
10705         atio->ccb_h.status |= CAM_CDB_RECVD;
10706
10707         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
10708                 /*
10709                  * We weren't allowed to disconnect.
10710                  * We're hanging on the bus until a
10711                  * continue target I/O comes in response
10712                  * to this accept tio.
10713                  */
10714 #ifdef AHD_DEBUG
10715                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10716                         printk("Received Immediate Command %d:%d:%d - %p\n",
10717                                initiator, target, lun, ahd->pending_device);
10718 #endif
10719                 ahd->pending_device = lstate;
10720                 ahd_freeze_ccb((union ccb *)atio);
10721                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
10722         }
10723         xpt_done((union ccb*)atio);
10724         return (0);
10725 }
10726
10727 #endif