drivers/scsi/aic7xxx/aic7xxx_core.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2002 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
  41  */
  42
  43 #include "aic7xxx_osm.h"
  44 #include "aic7xxx_inline.h"
  45 #include "aicasm/aicasm_insformat.h"
  46
  47 /***************************** Lookup Tables **********************************/
  48 static const char *const ahc_chip_names[] = {
  49         "NONE",
  50         "aic7770",
  51         "aic7850",
  52         "aic7855",
  53         "aic7859",
  54         "aic7860",
  55         "aic7870",
  56         "aic7880",
  57         "aic7895",
  58         "aic7895C",
  59         "aic7890/91",
  60         "aic7896/97",
  61         "aic7892",
  62         "aic7899"
  63 };
  64
  65 /*
  66  * Hardware error codes.
  67  */
  68 struct ahc_hard_error_entry {
  69         uint8_t errno;
  70         const char *errmesg;
  71 };
  72
  73 static const struct ahc_hard_error_entry ahc_hard_errors[] = {
  74         { ILLHADDR,     "Illegal Host Access" },
  75         { ILLSADDR,     "Illegal Sequencer Address referenced" },
  76         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  77         { SQPARERR,     "Sequencer Parity Error" },
  78         { DPARERR,      "Data-path Parity Error" },
  79         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  80         { PCIERRSTAT,   "PCI Error detected" },
  81         { CIOPARERR,    "CIOBUS Parity Error" },
  82 };
  83 static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
  84
  85 static const struct ahc_phase_table_entry ahc_phase_table[] =
  86 {
  87         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  88         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  89         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  90         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  91         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  92         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  93         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  94         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  95         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  96         { 0,            MSG_NOOP,               "in unknown phase"      }
  97 };
  98
  99 /*
 100  * In most cases we only wish to itterate over real phases, so
 101  * exclude the last element from the count.
 102  */
 103 static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
 104
 105 /*
 106  * Valid SCSIRATE values.  (p. 3-17)
 107  * Provides a mapping of tranfer periods in ns to the proper value to
 108  * stick in the scsixfer reg.
 109  */
 110 static const struct ahc_syncrate ahc_syncrates[] =
 111 {
 112       /* ultra2    fast/ultra  period     rate */
 113         { 0x42,      0x000,      9,      "80.0" },
 114         { 0x03,      0x000,     10,      "40.0" },
 115         { 0x04,      0x000,     11,      "33.0" },
 116         { 0x05,      0x100,     12,      "20.0" },
 117         { 0x06,      0x110,     15,      "16.0" },
 118         { 0x07,      0x120,     18,      "13.4" },
 119         { 0x08,      0x000,     25,      "10.0" },
 120         { 0x19,      0x010,     31,      "8.0"  },
 121         { 0x1a,      0x020,     37,      "6.67" },
 122         { 0x1b,      0x030,     43,      "5.7"  },
 123         { 0x1c,      0x040,     50,      "5.0"  },
 124         { 0x00,      0x050,     56,      "4.4"  },
 125         { 0x00,      0x060,     62,      "4.0"  },
 126         { 0x00,      0x070,     68,      "3.6"  },
 127         { 0x00,      0x000,      0,      NULL   }
 128 };
 129
 130 /* Our Sequencer Program */
 131 #include "aic7xxx_seq.h"
 132
 133 /**************************** Function Declarations ***************************/
 134 static void             ahc_force_renegotiation(struct ahc_softc *ahc,
 135                                                 struct ahc_devinfo *devinfo);
 136 static struct ahc_tmode_tstate*
 137                         ahc_alloc_tstate(struct ahc_softc *ahc,
 138                                          u_int scsi_id, char channel);
 139 #ifdef AHC_TARGET_MODE
 140 static void             ahc_free_tstate(struct ahc_softc *ahc,
 141                                         u_int scsi_id, char channel, int force);
 142 #endif
 143 static const struct ahc_syncrate*
 144                         ahc_devlimited_syncrate(struct ahc_softc *ahc,
 145                                                 struct ahc_initiator_tinfo *,
 146                                                 u_int *period,
 147                                                 u_int *ppr_options,
 148                                                 role_t role);
 149 static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
 150 static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
 151                                           struct ahc_devinfo *devinfo);
 152 static void             ahc_scb_devinfo(struct ahc_softc *ahc,
 153                                         struct ahc_devinfo *devinfo,
 154                                         struct scb *scb);
 155 static void             ahc_assert_atn(struct ahc_softc *ahc);
 156 static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 157                                                    struct ahc_devinfo *devinfo,
 158                                                    struct scb *scb);
 159 static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
 160                                                struct ahc_devinfo *devinfo);
 161 static void             ahc_construct_sdtr(struct ahc_softc *ahc,
 162                                            struct ahc_devinfo *devinfo,
 163                                            u_int period, u_int offset);
 164 static void             ahc_construct_wdtr(struct ahc_softc *ahc,
 165                                            struct ahc_devinfo *devinfo,
 166                                            u_int bus_width);
 167 static void             ahc_construct_ppr(struct ahc_softc *ahc,
 168                                           struct ahc_devinfo *devinfo,
 169                                           u_int period, u_int offset,
 170                                           u_int bus_width, u_int ppr_options);
 171 static void             ahc_clear_msg_state(struct ahc_softc *ahc);
 172 static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
 173 static void             ahc_handle_message_phase(struct ahc_softc *ahc);
 174 typedef enum {
 175         AHCMSG_1B,
 176         AHCMSG_2B,
 177         AHCMSG_EXT
 178 } ahc_msgtype;
 179 static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 180                                      u_int msgval, int full);
 181 static int              ahc_parse_msg(struct ahc_softc *ahc,
 182                                       struct ahc_devinfo *devinfo);
 183 static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
 184                                               struct ahc_devinfo *devinfo);
 185 static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 186                                                 struct ahc_devinfo *devinfo);
 187 static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 188 static void             ahc_handle_devreset(struct ahc_softc *ahc,
 189                                             struct ahc_devinfo *devinfo,
 190                                             cam_status status, char *message,
 191                                             int verbose_level);
 192 #ifdef AHC_TARGET_MODE
 193 static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
 194                                                struct ahc_devinfo *devinfo,
 195                                                struct scb *scb);
 196 #endif
 197
 198 static bus_dmamap_callback_t    ahc_dmamap_cb;
 199 static void             ahc_build_free_scb_list(struct ahc_softc *ahc);
 200 static int              ahc_init_scbdata(struct ahc_softc *ahc);
 201 static void             ahc_fini_scbdata(struct ahc_softc *ahc);
 202 static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
 203                                             struct scb *prev_scb,
 204                                             struct scb *scb);
 205 static int              ahc_qinfifo_count(struct ahc_softc *ahc);
 206 static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 207                                                    u_int prev, u_int scbptr);
 208 static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 209 static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
 210                                      u_int scbpos, u_int prev);
 211 static void             ahc_reset_current_bus(struct ahc_softc *ahc);
 212 #ifdef AHC_DUMP_SEQ
 213 static void             ahc_dumpseq(struct ahc_softc *ahc);
 214 #endif
 215 static int              ahc_loadseq(struct ahc_softc *ahc);
 216 static int              ahc_check_patch(struct ahc_softc *ahc,
 217                                         const struct patch **start_patch,
 218                                         u_int start_instr, u_int *skip_addr);
 219 static void             ahc_download_instr(struct ahc_softc *ahc,
 220                                            u_int instrptr, uint8_t *dconsts);
 221 #ifdef AHC_TARGET_MODE
 222 static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
 223                                                struct ahc_tmode_lstate *lstate,
 224                                                u_int initiator_id,
 225                                                u_int event_type,
 226                                                u_int event_arg);
 227 static void             ahc_update_scsiid(struct ahc_softc *ahc,
 228                                           u_int targid_mask);
 229 static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
 230                                               struct target_cmd *cmd);
 231 #endif
 232
 233 static u_int            ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl);
 234 static void             ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl);
 235 static void             ahc_busy_tcl(struct ahc_softc *ahc,
 236                                      u_int tcl, u_int busyid);
 237
 238 /************************** SCB and SCB queue management **********************/
 239 static void             ahc_run_untagged_queues(struct ahc_softc *ahc);
 240 static void             ahc_run_untagged_queue(struct ahc_softc *ahc,
 241                                                struct scb_tailq *queue);
 242
 243 /****************************** Initialization ********************************/
 244 static void              ahc_alloc_scbs(struct ahc_softc *ahc);
 245 static void              ahc_shutdown(void *arg);
 246
 247 /*************************** Interrupt Services *******************************/
 248 static void             ahc_clear_intstat(struct ahc_softc *ahc);
 249 static void             ahc_run_qoutfifo(struct ahc_softc *ahc);
 250 #ifdef AHC_TARGET_MODE
 251 static void             ahc_run_tqinfifo(struct ahc_softc *ahc, int paused);
 252 #endif
 253 static void             ahc_handle_brkadrint(struct ahc_softc *ahc);
 254 static void             ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat);
 255 static void             ahc_handle_scsiint(struct ahc_softc *ahc,
 256                                            u_int intstat);
 257 static void             ahc_clear_critical_section(struct ahc_softc *ahc);
 258
 259 /***************************** Error Recovery *********************************/
 260 static void             ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb);
 261 static int              ahc_abort_scbs(struct ahc_softc *ahc, int target,
 262                                        char channel, int lun, u_int tag,
 263                                        role_t role, uint32_t status);
 264 static void             ahc_calc_residual(struct ahc_softc *ahc,
 265                                           struct scb *scb);
 266
 267 /*********************** Untagged Transaction Routines ************************/
 268 static inline void      ahc_freeze_untagged_queues(struct ahc_softc *ahc);
 269 static inline void      ahc_release_untagged_queues(struct ahc_softc *ahc);
 270
 271 /*
 272  * Block our completion routine from starting the next untagged
 273  * transaction for this target or target lun.
 274  */
 275 static inline void
 276 ahc_freeze_untagged_queues(struct ahc_softc *ahc)
 277 {
 278         if ((ahc->flags & AHC_SCB_BTT) == 0)
 279                 ahc->untagged_queue_lock++;
 280 }
 281
 282 /*
 283  * Allow the next untagged transaction for this target or target lun
 284  * to be executed.  We use a counting semaphore to allow the lock
 285  * to be acquired recursively.  Once the count drops to zero, the
 286  * transaction queues will be run.
 287  */
 288 static inline void
 289 ahc_release_untagged_queues(struct ahc_softc *ahc)
 290 {
 291         if ((ahc->flags & AHC_SCB_BTT) == 0) {
 292                 ahc->untagged_queue_lock--;
 293                 if (ahc->untagged_queue_lock == 0)
 294                         ahc_run_untagged_queues(ahc);
 295         }
 296 }
 297
 298 /************************* Sequencer Execution Control ************************/
 299 /*
 300  * Work around any chip bugs related to halting sequencer execution.
 301  * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
 302  * reading a register that will set this signal and deassert it.
 303  * Without this workaround, if the chip is paused, by an interrupt or
 304  * manual pause while accessing scb ram, accesses to certain registers
 305  * will hang the system (infinite pci retries).
 306  */
 307 static void
 308 ahc_pause_bug_fix(struct ahc_softc *ahc)
 309 {
 310         if ((ahc->features & AHC_ULTRA2) != 0)
 311                 (void)ahc_inb(ahc, CCSCBCTL);
 312 }
 313
 314 /*
 315  * Determine whether the sequencer has halted code execution.
 316  * Returns non-zero status if the sequencer is stopped.
 317  */
 318 int
 319 ahc_is_paused(struct ahc_softc *ahc)
 320 {
 321         return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
 322 }
 323
 324 /*
 325  * Request that the sequencer stop and wait, indefinitely, for it
 326  * to stop.  The sequencer will only acknowledge that it is paused
 327  * once it has reached an instruction boundary and PAUSEDIS is
 328  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 329  * for critical sections.
 330  */
 331 void
 332 ahc_pause(struct ahc_softc *ahc)
 333 {
 334         ahc_outb(ahc, HCNTRL, ahc->pause);
 335
 336         /*
 337          * Since the sequencer can disable pausing in a critical section, we
 338          * must loop until it actually stops.
 339          */
 340         while (ahc_is_paused(ahc) == 0)
 341                 ;
 342
 343         ahc_pause_bug_fix(ahc);
 344 }
 345
 346 /*
 347  * Allow the sequencer to continue program execution.
 348  * We check here to ensure that no additional interrupt
 349  * sources that would cause the sequencer to halt have been
 350  * asserted.  If, for example, a SCSI bus reset is detected
 351  * while we are fielding a different, pausing, interrupt type,
 352  * we don't want to release the sequencer before going back
 353  * into our interrupt handler and dealing with this new
 354  * condition.
 355  */
 356 void
 357 ahc_unpause(struct ahc_softc *ahc)
 358 {
 359         if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
 360                 ahc_outb(ahc, HCNTRL, ahc->unpause);
 361 }
 362
 363 /************************** Memory mapping routines ***************************/
 364 static struct ahc_dma_seg *
 365 ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
 366 {
 367         int sg_index;
 368
 369         sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
 370         /* sg_list_phys points to entry 1, not 0 */
 371         sg_index++;
 372
 373         return (&scb->sg_list[sg_index]);
 374 }
 375
 376 static uint32_t
 377 ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
 378 {
 379         int sg_index;
 380
 381         /* sg_list_phys points to entry 1, not 0 */
 382         sg_index = sg - &scb->sg_list[1];
 383
 384         return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
 385 }
 386
 387 static uint32_t
 388 ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
 389 {
 390         return (ahc->scb_data->hscb_busaddr
 391                 + (sizeof(struct hardware_scb) * index));
 392 }
 393
 394 static void
 395 ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
 396 {
 397         ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
 398                         ahc->scb_data->hscb_dmamap,
 399                         /*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
 400                         /*len*/sizeof(*scb->hscb), op);
 401 }
 402
 403 void
 404 ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
 405 {
 406         if (scb->sg_count == 0)
 407                 return;
 408
 409         ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
 410                         /*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
 411                                 * sizeof(struct ahc_dma_seg),
 412                         /*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
 413 }
 414
 415 #ifdef AHC_TARGET_MODE
 416 static uint32_t
 417 ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
 418 {
 419         return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
 420 }
 421 #endif
 422
 423 /*********************** Miscellaneous Support Functions ***********************/
 424 /*
 425  * Determine whether the sequencer reported a residual
 426  * for this SCB/transaction.
 427  */
 428 static void
 429 ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
 430 {
 431         uint32_t sgptr;
 432
 433         sgptr = ahc_le32toh(scb->hscb->sgptr);
 434         if ((sgptr & SG_RESID_VALID) != 0)
 435                 ahc_calc_residual(ahc, scb);
 436 }
 437
 438 /*
 439  * Return pointers to the transfer negotiation information
 440  * for the specified our_id/remote_id pair.
 441  */
 442 struct ahc_initiator_tinfo *
 443 ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
 444                     u_int remote_id, struct ahc_tmode_tstate **tstate)
 445 {
 446         /*
 447          * Transfer data structures are stored from the perspective
 448          * of the target role.  Since the parameters for a connection
 449          * in the initiator role to a given target are the same as
 450          * when the roles are reversed, we pretend we are the target.
 451          */
 452         if (channel == 'B')
 453                 our_id += 8;
 454         *tstate = ahc->enabled_targets[our_id];
 455         return (&(*tstate)->transinfo[remote_id]);
 456 }
 457
 458 uint16_t
 459 ahc_inw(struct ahc_softc *ahc, u_int port)
 460 {
 461         uint16_t r = ahc_inb(ahc, port+1) << 8;
 462         return r | ahc_inb(ahc, port);
 463 }
 464
 465 void
 466 ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
 467 {
 468         ahc_outb(ahc, port, value & 0xFF);
 469         ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 470 }
 471
 472 uint32_t
 473 ahc_inl(struct ahc_softc *ahc, u_int port)
 474 {
 475         return ((ahc_inb(ahc, port))
 476               | (ahc_inb(ahc, port+1) << 8)
 477               | (ahc_inb(ahc, port+2) << 16)
 478               | (ahc_inb(ahc, port+3) << 24));
 479 }
 480
 481 void
 482 ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
 483 {
 484         ahc_outb(ahc, port, (value) & 0xFF);
 485         ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
 486         ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
 487         ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
 488 }
 489
 490 uint64_t
 491 ahc_inq(struct ahc_softc *ahc, u_int port)
 492 {
 493         return ((ahc_inb(ahc, port))
 494               | (ahc_inb(ahc, port+1) << 8)
 495               | (ahc_inb(ahc, port+2) << 16)
 496               | (ahc_inb(ahc, port+3) << 24)
 497               | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
 498               | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
 499               | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
 500               | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
 501 }
 502
 503 void
 504 ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
 505 {
 506         ahc_outb(ahc, port, value & 0xFF);
 507         ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 508         ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
 509         ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
 510         ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
 511         ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
 512         ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
 513         ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
 514 }
 515
 516 /*
 517  * Get a free scb. If there are none, see if we can allocate a new SCB.
 518  */
 519 struct scb *
 520 ahc_get_scb(struct ahc_softc *ahc)
 521 {
 522         struct scb *scb;
 523
 524         if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
 525                 ahc_alloc_scbs(ahc);
 526                 scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
 527                 if (scb == NULL)
 528                         return (NULL);
 529         }
 530         SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
 531         return (scb);
 532 }
 533
 534 /*
 535  * Return an SCB resource to the free list.
 536  */
 537 void
 538 ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
 539 {
 540         struct hardware_scb *hscb;
 541
 542         hscb = scb->hscb;
 543         /* Clean up for the next user */
 544         ahc->scb_data->scbindex[hscb->tag] = NULL;
 545         scb->flags = SCB_FREE;
 546         hscb->control = 0;
 547
 548         SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);
 549
 550         /* Notify the OSM that a resource is now available. */
 551         ahc_platform_scb_free(ahc, scb);
 552 }
 553
 554 struct scb *
 555 ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
 556 {
 557         struct scb* scb;
 558
 559         scb = ahc->scb_data->scbindex[tag];
 560         if (scb != NULL)
 561                 ahc_sync_scb(ahc, scb,
 562                              BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 563         return (scb);
 564 }
 565
 566 static void
 567 ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
 568 {
 569         struct hardware_scb *q_hscb;
 570         u_int  saved_tag;
 571
 572         /*
 573          * Our queuing method is a bit tricky.  The card
 574          * knows in advance which HSCB to download, and we
 575          * can't disappoint it.  To achieve this, the next
 576          * SCB to download is saved off in ahc->next_queued_scb.
 577          * When we are called to queue "an arbitrary scb",
 578          * we copy the contents of the incoming HSCB to the one
 579          * the sequencer knows about, swap HSCB pointers and
 580          * finally assign the SCB to the tag indexed location
 581          * in the scb_array.  This makes sure that we can still
 582          * locate the correct SCB by SCB_TAG.
 583          */
 584         q_hscb = ahc->next_queued_scb->hscb;
 585         saved_tag = q_hscb->tag;
 586         memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 587         if ((scb->flags & SCB_CDB32_PTR) != 0) {
 588                 q_hscb->shared_data.cdb_ptr =
 589                     ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
 590                               + offsetof(struct hardware_scb, cdb32));
 591         }
 592         q_hscb->tag = saved_tag;
 593         q_hscb->next = scb->hscb->tag;
 594
 595         /* Now swap HSCB pointers. */
 596         ahc->next_queued_scb->hscb = scb->hscb;
 597         scb->hscb = q_hscb;
 598
 599         /* Now define the mapping from tag to SCB in the scbindex */
 600         ahc->scb_data->scbindex[scb->hscb->tag] = scb;
 601 }
 602
 603 /*
 604  * Tell the sequencer about a new transaction to execute.
 605  */
 606 void
 607 ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
 608 {
 609         ahc_swap_with_next_hscb(ahc, scb);
 610
 611         if (scb->hscb->tag == SCB_LIST_NULL
 612          || scb->hscb->next == SCB_LIST_NULL)
 613                 panic("Attempt to queue invalid SCB tag %x:%x\n",
 614                       scb->hscb->tag, scb->hscb->next);
 615
 616         /*
 617          * Setup data "oddness".
 618          */
 619         scb->hscb->lun &= LID;
 620         if (ahc_get_transfer_length(scb) & 0x1)
 621                 scb->hscb->lun |= SCB_XFERLEN_ODD;
 622
 623         /*
 624          * Keep a history of SCBs we've downloaded in the qinfifo.
 625          */
 626         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
 627
 628         /*
 629          * Make sure our data is consistent from the
 630          * perspective of the adapter.
 631          */
 632         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 633
 634         /* Tell the adapter about the newly queued SCB */
 635         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
 636                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
 637         } else {
 638                 if ((ahc->features & AHC_AUTOPAUSE) == 0)
 639                         ahc_pause(ahc);
 640                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
 641                 if ((ahc->features & AHC_AUTOPAUSE) == 0)
 642                         ahc_unpause(ahc);
 643         }
 644 }
 645
 646 struct scsi_sense_data *
 647 ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
 648 {
 649         int offset;
 650
 651         offset = scb - ahc->scb_data->scbarray;
 652         return (&ahc->scb_data->sense[offset]);
 653 }
 654
 655 static uint32_t
 656 ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
 657 {
 658         int offset;
 659
 660         offset = scb - ahc->scb_data->scbarray;
 661         return (ahc->scb_data->sense_busaddr
 662               + (offset * sizeof(struct scsi_sense_data)));
 663 }
 664
 665 /************************** Interrupt Processing ******************************/
 666 static void
 667 ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
 668 {
 669         ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 670                         /*offset*/0, /*len*/256, op);
 671 }
 672
 673 static void
 674 ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
 675 {
 676 #ifdef AHC_TARGET_MODE
 677         if ((ahc->flags & AHC_TARGETROLE) != 0) {
 678                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 679                                 ahc->shared_data_dmamap,
 680                                 ahc_targetcmd_offset(ahc, 0),
 681                                 sizeof(struct target_cmd) * AHC_TMODE_CMDS,
 682                                 op);
 683         }
 684 #endif
 685 }
 686
 687 /*
 688  * See if the firmware has posted any completed commands
 689  * into our in-core command complete fifos.
 690  */
 691 #define AHC_RUN_QOUTFIFO 0x1
 692 #define AHC_RUN_TQINFIFO 0x2
 693 static u_int
 694 ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
 695 {
 696         u_int retval;
 697
 698         retval = 0;
 699         ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 700                         /*offset*/ahc->qoutfifonext, /*len*/1,
 701                         BUS_DMASYNC_POSTREAD);
 702         if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
 703                 retval |= AHC_RUN_QOUTFIFO;
 704 #ifdef AHC_TARGET_MODE
 705         if ((ahc->flags & AHC_TARGETROLE) != 0
 706          && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
 707                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 708                                 ahc->shared_data_dmamap,
 709                                 ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
 710                                 /*len*/sizeof(struct target_cmd),
 711                                 BUS_DMASYNC_POSTREAD);
 712                 if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
 713                         retval |= AHC_RUN_TQINFIFO;
 714         }
 715 #endif
 716         return (retval);
 717 }
 718
 719 /*
 720  * Catch an interrupt from the adapter
 721  */
 722 int
 723 ahc_intr(struct ahc_softc *ahc)
 724 {
 725         u_int   intstat;
 726
 727         if ((ahc->pause & INTEN) == 0) {
 728                 /*
 729                  * Our interrupt is not enabled on the chip
 730                  * and may be disabled for re-entrancy reasons,
 731                  * so just return.  This is likely just a shared
 732                  * interrupt.
 733                  */
 734                 return (0);
 735         }
 736         /*
 737          * Instead of directly reading the interrupt status register,
 738          * infer the cause of the interrupt by checking our in-core
 739          * completion queues.  This avoids a costly PCI bus read in
 740          * most cases.
 741          */
 742         if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
 743          && (ahc_check_cmdcmpltqueues(ahc) != 0))
 744                 intstat = CMDCMPLT;
 745         else {
 746                 intstat = ahc_inb(ahc, INTSTAT);
 747         }
 748
 749         if ((intstat & INT_PEND) == 0) {
 750 #if AHC_PCI_CONFIG > 0
 751                 if (ahc->unsolicited_ints > 500) {
 752                         ahc->unsolicited_ints = 0;
 753                         if ((ahc->chip & AHC_PCI) != 0
 754                          && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
 755                                 ahc->bus_intr(ahc);
 756                 }
 757 #endif
 758                 ahc->unsolicited_ints++;
 759                 return (0);
 760         }
 761         ahc->unsolicited_ints = 0;
 762
 763         if (intstat & CMDCMPLT) {
 764                 ahc_outb(ahc, CLRINT, CLRCMDINT);
 765
 766                 /*
 767                  * Ensure that the chip sees that we've cleared
 768                  * this interrupt before we walk the output fifo.
 769                  * Otherwise, we may, due to posted bus writes,
 770                  * clear the interrupt after we finish the scan,
 771                  * and after the sequencer has added new entries
 772                  * and asserted the interrupt again.
 773                  */
 774                 ahc_flush_device_writes(ahc);
 775                 ahc_run_qoutfifo(ahc);
 776 #ifdef AHC_TARGET_MODE
 777                 if ((ahc->flags & AHC_TARGETROLE) != 0)
 778                         ahc_run_tqinfifo(ahc, /*paused*/FALSE);
 779 #endif
 780         }
 781
 782         /*
 783          * Handle statuses that may invalidate our cached
 784          * copy of INTSTAT separately.
 785          */
 786         if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
 787                 /* Hot eject.  Do nothing */
 788         } else if (intstat & BRKADRINT) {
 789                 ahc_handle_brkadrint(ahc);
 790         } else if ((intstat & (SEQINT|SCSIINT)) != 0) {
 791
 792                 ahc_pause_bug_fix(ahc);
 793
 794                 if ((intstat & SEQINT) != 0)
 795                         ahc_handle_seqint(ahc, intstat);
 796
 797                 if ((intstat & SCSIINT) != 0)
 798                         ahc_handle_scsiint(ahc, intstat);
 799         }
 800         return (1);
 801 }
 802
 803 /************************* Sequencer Execution Control ************************/
 804 /*
 805  * Restart the sequencer program from address zero
 806  */
 807 static void
 808 ahc_restart(struct ahc_softc *ahc)
 809 {
 810         uint8_t sblkctl;
 811
 812         ahc_pause(ahc);
 813
 814         /* No more pending messages. */
 815         ahc_clear_msg_state(ahc);
 816
 817         ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
 818         ahc_outb(ahc, MSG_OUT, MSG_NOOP);       /* No message to send */
 819         ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 820         ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 821         ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 822         ahc_outb(ahc, SAVED_LUN, 0xFF);
 823
 824         /*
 825          * Ensure that the sequencer's idea of TQINPOS
 826          * matches our own.  The sequencer increments TQINPOS
 827          * only after it sees a DMA complete and a reset could
 828          * occur before the increment leaving the kernel to believe
 829          * the command arrived but the sequencer to not.
 830          */
 831         ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 832
 833         /* Always allow reselection */
 834         ahc_outb(ahc, SCSISEQ,
 835                  ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 836         if ((ahc->features & AHC_CMD_CHAN) != 0) {
 837                 /* Ensure that no DMA operations are in progress */
 838                 ahc_outb(ahc, CCSCBCNT, 0);
 839                 ahc_outb(ahc, CCSGCTL, 0);
 840                 ahc_outb(ahc, CCSCBCTL, 0);
 841         }
 842         /*
 843          * If we were in the process of DMA'ing SCB data into
 844          * an SCB, replace that SCB on the free list.  This prevents
 845          * an SCB leak.
 846          */
 847         if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 848                 ahc_add_curscb_to_free_list(ahc);
 849                 ahc_outb(ahc, SEQ_FLAGS2,
 850                          ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 851         }
 852
 853         /*
 854          * Clear any pending sequencer interrupt.  It is no
 855          * longer relevant since we're resetting the Program
 856          * Counter.
 857          */
 858         ahc_outb(ahc, CLRINT, CLRSEQINT);
 859
 860         ahc_outb(ahc, MWI_RESIDUAL, 0);
 861         ahc_outb(ahc, SEQCTL, ahc->seqctl);
 862         ahc_outb(ahc, SEQADDR0, 0);
 863         ahc_outb(ahc, SEQADDR1, 0);
 864
 865         /*
 866          * Take the LED out of diagnostic mode on PM resume, too
 867          */
 868         sblkctl = ahc_inb(ahc, SBLKCTL);
 869         ahc_outb(ahc, SBLKCTL, (sblkctl & ~(DIAGLEDEN|DIAGLEDON)));
 870
 871         ahc_unpause(ahc);
 872 }
 873
 874 /************************* Input/Output Queues ********************************/
 875 static void
 876 ahc_run_qoutfifo(struct ahc_softc *ahc)
 877 {
 878         struct scb *scb;
 879         u_int  scb_index;
 880
 881         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 882         while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 883
 884                 scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 885                 if ((ahc->qoutfifonext & 0x03) == 0x03) {
 886                         u_int modnext;
 887
 888                         /*
 889                          * Clear 32bits of QOUTFIFO at a time
 890                          * so that we don't clobber an incoming
 891                          * byte DMA to the array on architectures
 892                          * that only support 32bit load and store
 893                          * operations.
 894                          */
 895                         modnext = ahc->qoutfifonext & ~0x3;
 896                         *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 897                         ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 898                                         ahc->shared_data_dmamap,
 899                                         /*offset*/modnext, /*len*/4,
 900                                         BUS_DMASYNC_PREREAD);
 901                 }
 902                 ahc->qoutfifonext++;
 903
 904                 scb = ahc_lookup_scb(ahc, scb_index);
 905                 if (scb == NULL) {
 906                         printk("%s: WARNING no command for scb %d "
 907                                "(cmdcmplt)\nQOUTPOS = %d\n",
 908                                ahc_name(ahc), scb_index,
 909                                (ahc->qoutfifonext - 1) & 0xFF);
 910                         continue;
 911                 }
 912
 913                 /*
 914                  * Save off the residual
 915                  * if there is one.
 916                  */
 917                 ahc_update_residual(ahc, scb);
 918                 ahc_done(ahc, scb);
 919         }
 920 }
 921
 922 static void
 923 ahc_run_untagged_queues(struct ahc_softc *ahc)
 924 {
 925         int i;
 926
 927         for (i = 0; i < 16; i++)
 928                 ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 929 }
 930
 931 static void
 932 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 933 {
 934         struct scb *scb;
 935
 936         if (ahc->untagged_queue_lock != 0)
 937                 return;
 938
 939         if ((scb = TAILQ_FIRST(queue)) != NULL
 940          && (scb->flags & SCB_ACTIVE) == 0) {
 941                 scb->flags |= SCB_ACTIVE;
 942                 ahc_queue_scb(ahc, scb);
 943         }
 944 }
 945
 946 /************************* Interrupt Handling *********************************/
 947 static void
 948 ahc_handle_brkadrint(struct ahc_softc *ahc)
 949 {
 950         /*
 951          * We upset the sequencer :-(
 952          * Lookup the error message
 953          */
 954         int i;
 955         int error;
 956
 957         error = ahc_inb(ahc, ERROR);
 958         for (i = 0; error != 1 && i < num_errors; i++)
 959                 error >>= 1;
 960         printk("%s: brkadrint, %s at seqaddr = 0x%x\n",
 961                ahc_name(ahc), ahc_hard_errors[i].errmesg,
 962                ahc_inb(ahc, SEQADDR0) |
 963                (ahc_inb(ahc, SEQADDR1) << 8));
 964
 965         ahc_dump_card_state(ahc);
 966
 967         /* Tell everyone that this HBA is no longer available */
 968         ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 969                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 970                        CAM_NO_HBA);
 971
 972         /* Disable all interrupt sources by resetting the controller */
 973         ahc_shutdown(ahc);
 974 }
 975
 976 static void
 977 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 978 {
 979         struct scb *scb;
 980         struct ahc_devinfo devinfo;
 981
 982         ahc_fetch_devinfo(ahc, &devinfo);
 983
 984         /*
 985          * Clear the upper byte that holds SEQINT status
 986          * codes and clear the SEQINT bit. We will unpause
 987          * the sequencer, if appropriate, after servicing
 988          * the request.
 989          */
 990         ahc_outb(ahc, CLRINT, CLRSEQINT);
 991         switch (intstat & SEQINT_MASK) {
 992         case BAD_STATUS:
 993         {
 994                 u_int  scb_index;
 995                 struct hardware_scb *hscb;
 996
 997                 /*
 998                  * Set the default return value to 0 (don't
 999                  * send sense).  The sense code will change
1000                  * this if needed.
1001                  */
1002                 ahc_outb(ahc, RETURN_1, 0);
1003
1004                 /*
1005                  * The sequencer will notify us when a command
1006                  * has an error that would be of interest to
1007                  * the kernel.  This allows us to leave the sequencer
1008                  * running in the common case of command completes
1009                  * without error.  The sequencer will already have
1010                  * dma'd the SCB back up to us, so we can reference
1011                  * the in kernel copy directly.
1012                  */
1013                 scb_index = ahc_inb(ahc, SCB_TAG);
1014                 scb = ahc_lookup_scb(ahc, scb_index);
1015                 if (scb == NULL) {
1016                         ahc_print_devinfo(ahc, &devinfo);
1017                         printk("ahc_intr - referenced scb "
1018                                "not valid during seqint 0x%x scb(%d)\n",
1019                                intstat, scb_index);
1020                         ahc_dump_card_state(ahc);
1021                         panic("for safety");
1022                         goto unpause;
1023                 }
1024
1025                 hscb = scb->hscb;
1026
1027                 /* Don't want to clobber the original sense code */
1028                 if ((scb->flags & SCB_SENSE) != 0) {
1029                         /*
1030                          * Clear the SCB_SENSE Flag and have
1031                          * the sequencer do a normal command
1032                          * complete.
1033                          */
1034                         scb->flags &= ~SCB_SENSE;
1035                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1036                         break;
1037                 }
1038                 ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
1039                 /* Freeze the queue until the client sees the error. */
1040                 ahc_freeze_devq(ahc, scb);
1041                 ahc_freeze_scb(scb);
1042                 ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
1043                 switch (hscb->shared_data.status.scsi_status) {
1044                 case SCSI_STATUS_OK:
1045                         printk("%s: Interrupted for status of 0???\n",
1046                                ahc_name(ahc));
1047                         break;
1048                 case SCSI_STATUS_CMD_TERMINATED:
1049                 case SCSI_STATUS_CHECK_COND:
1050                 {
1051                         struct ahc_dma_seg *sg;
1052                         struct scsi_sense *sc;
1053                         struct ahc_initiator_tinfo *targ_info;
1054                         struct ahc_tmode_tstate *tstate;
1055                         struct ahc_transinfo *tinfo;
1056 #ifdef AHC_DEBUG
1057                         if (ahc_debug & AHC_SHOW_SENSE) {
1058                                 ahc_print_path(ahc, scb);
1059                                 printk("SCB %d: requests Check Status\n",
1060                                        scb->hscb->tag);
1061                         }
1062 #endif
1063
1064                         if (ahc_perform_autosense(scb) == 0)
1065                                 break;
1066
1067                         targ_info = ahc_fetch_transinfo(ahc,
1068                                                         devinfo.channel,
1069                                                         devinfo.our_scsiid,
1070                                                         devinfo.target,
1071                                                         &tstate);
1072                         tinfo = &targ_info->curr;
1073                         sg = scb->sg_list;
1074                         sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
1075                         /*
1076                          * Save off the residual if there is one.
1077                          */
1078                         ahc_update_residual(ahc, scb);
1079 #ifdef AHC_DEBUG
1080                         if (ahc_debug & AHC_SHOW_SENSE) {
1081                                 ahc_print_path(ahc, scb);
1082                                 printk("Sending Sense\n");
1083                         }
1084 #endif
1085                         sg->addr = ahc_get_sense_bufaddr(ahc, scb);
1086                         sg->len = ahc_get_sense_bufsize(ahc, scb);
1087                         sg->len |= AHC_DMA_LAST_SEG;
1088
1089                         /* Fixup byte order */
1090                         sg->addr = ahc_htole32(sg->addr);
1091                         sg->len = ahc_htole32(sg->len);
1092
1093                         sc->opcode = REQUEST_SENSE;
1094                         sc->byte2 = 0;
1095                         if (tinfo->protocol_version <= SCSI_REV_2
1096                          && SCB_GET_LUN(scb) < 8)
1097                                 sc->byte2 = SCB_GET_LUN(scb) << 5;
1098                         sc->unused[0] = 0;
1099                         sc->unused[1] = 0;
1100                         sc->length = sg->len;
1101                         sc->control = 0;
1102
1103                         /*
1104                          * We can't allow the target to disconnect.
1105                          * This will be an untagged transaction and
1106                          * having the target disconnect will make this
1107                          * transaction indestinguishable from outstanding
1108                          * tagged transactions.
1109                          */
1110                         hscb->control = 0;
1111
1112                         /*
1113                          * This request sense could be because the
1114                          * the device lost power or in some other
1115                          * way has lost our transfer negotiations.
1116                          * Renegotiate if appropriate.  Unit attention
1117                          * errors will be reported before any data
1118                          * phases occur.
1119                          */
1120                         if (ahc_get_residual(scb)
1121                          == ahc_get_transfer_length(scb)) {
1122                                 ahc_update_neg_request(ahc, &devinfo,
1123                                                        tstate, targ_info,
1124                                                        AHC_NEG_IF_NON_ASYNC);
1125                         }
1126                         if (tstate->auto_negotiate & devinfo.target_mask) {
1127                                 hscb->control |= MK_MESSAGE;
1128                                 scb->flags &= ~SCB_NEGOTIATE;
1129                                 scb->flags |= SCB_AUTO_NEGOTIATE;
1130                         }
1131                         hscb->cdb_len = sizeof(*sc);
1132                         hscb->dataptr = sg->addr;
1133                         hscb->datacnt = sg->len;
1134                         hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
1135                         hscb->sgptr = ahc_htole32(hscb->sgptr);
1136                         scb->sg_count = 1;
1137                         scb->flags |= SCB_SENSE;
1138                         ahc_qinfifo_requeue_tail(ahc, scb);
1139                         ahc_outb(ahc, RETURN_1, SEND_SENSE);
1140                         /*
1141                          * Ensure we have enough time to actually
1142                          * retrieve the sense.
1143                          */
1144                         ahc_scb_timer_reset(scb, 5 * 1000000);
1145                         break;
1146                 }
1147                 default:
1148                         break;
1149                 }
1150                 break;
1151         }
1152         case NO_MATCH:
1153         {
1154                 /* Ensure we don't leave the selection hardware on */
1155                 ahc_outb(ahc, SCSISEQ,
1156                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1157
1158                 printk("%s:%c:%d: no active SCB for reconnecting "
1159                        "target - issuing BUS DEVICE RESET\n",
1160                        ahc_name(ahc), devinfo.channel, devinfo.target);
1161                 printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1162                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
1163                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1164                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1165                 printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1166                        "SINDEX == 0x%x\n",
1167                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1168                        ahc_index_busy_tcl(ahc,
1169                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1170                                       ahc_inb(ahc, SAVED_LUN))),
1171                        ahc_inb(ahc, SINDEX));
1172                 printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1173                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1174                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1175                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1176                        ahc_inb(ahc, SCB_CONTROL));
1177                 printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1178                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1179                 printk("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
1180                 printk("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
1181                 ahc_dump_card_state(ahc);
1182                 ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
1183                 ahc->msgout_len = 1;
1184                 ahc->msgout_index = 0;
1185                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1186                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
1187                 ahc_assert_atn(ahc);
1188                 break;
1189         }
1190         case SEND_REJECT:
1191         {
1192                 u_int rejbyte = ahc_inb(ahc, ACCUM);
1193                 printk("%s:%c:%d: Warning - unknown message received from "
1194                        "target (0x%x).  Rejecting\n",
1195                        ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
1196                 break;
1197         }
1198         case PROTO_VIOLATION:
1199         {
1200                 ahc_handle_proto_violation(ahc);
1201                 break;
1202         }
1203         case IGN_WIDE_RES:
1204                 ahc_handle_ign_wide_residue(ahc, &devinfo);
1205                 break;
1206         case PDATA_REINIT:
1207                 ahc_reinitialize_dataptrs(ahc);
1208                 break;
1209         case BAD_PHASE:
1210         {
1211                 u_int lastphase;
1212
1213                 lastphase = ahc_inb(ahc, LASTPHASE);
1214                 printk("%s:%c:%d: unknown scsi bus phase %x, "
1215                        "lastphase = 0x%x.  Attempting to continue\n",
1216                        ahc_name(ahc), devinfo.channel, devinfo.target,
1217                        lastphase, ahc_inb(ahc, SCSISIGI));
1218                 break;
1219         }
1220         case MISSED_BUSFREE:
1221         {
1222                 u_int lastphase;
1223
1224                 lastphase = ahc_inb(ahc, LASTPHASE);
1225                 printk("%s:%c:%d: Missed busfree. "
1226                        "Lastphase = 0x%x, Curphase = 0x%x\n",
1227                        ahc_name(ahc), devinfo.channel, devinfo.target,
1228                        lastphase, ahc_inb(ahc, SCSISIGI));
1229                 ahc_restart(ahc);
1230                 return;
1231         }
1232         case HOST_MSG_LOOP:
1233         {
1234                 /*
1235                  * The sequencer has encountered a message phase
1236                  * that requires host assistance for completion.
1237                  * While handling the message phase(s), we will be
1238                  * notified by the sequencer after each byte is
1239                  * transferred so we can track bus phase changes.
1240                  *
1241                  * If this is the first time we've seen a HOST_MSG_LOOP
1242                  * interrupt, initialize the state of the host message
1243                  * loop.
1244                  */
1245                 if (ahc->msg_type == MSG_TYPE_NONE) {
1246                         struct scb *scb;
1247                         u_int scb_index;
1248                         u_int bus_phase;
1249
1250                         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1251                         if (bus_phase != P_MESGIN
1252                          && bus_phase != P_MESGOUT) {
1253                                 printk("ahc_intr: HOST_MSG_LOOP bad "
1254                                        "phase 0x%x\n",
1255                                       bus_phase);
1256                                 /*
1257                                  * Probably transitioned to bus free before
1258                                  * we got here.  Just punt the message.
1259                                  */
1260                                 ahc_clear_intstat(ahc);
1261                                 ahc_restart(ahc);
1262                                 return;
1263                         }
1264
1265                         scb_index = ahc_inb(ahc, SCB_TAG);
1266                         scb = ahc_lookup_scb(ahc, scb_index);
1267                         if (devinfo.role == ROLE_INITIATOR) {
1268                                 if (bus_phase == P_MESGOUT) {
1269                                         if (scb == NULL)
1270                                                 panic("HOST_MSG_LOOP with "
1271                                                       "invalid SCB %x\n",
1272                                                       scb_index);
1273
1274                                         ahc_setup_initiator_msgout(ahc,
1275                                                                    &devinfo,
1276                                                                    scb);
1277                                 } else {
1278                                         ahc->msg_type =
1279                                             MSG_TYPE_INITIATOR_MSGIN;
1280                                         ahc->msgin_index = 0;
1281                                 }
1282                         }
1283 #ifdef AHC_TARGET_MODE
1284                         else {
1285                                 if (bus_phase == P_MESGOUT) {
1286                                         ahc->msg_type =
1287                                             MSG_TYPE_TARGET_MSGOUT;
1288                                         ahc->msgin_index = 0;
1289                                 }
1290                                 else
1291                                         ahc_setup_target_msgin(ahc,
1292                                                                &devinfo,
1293                                                                scb);
1294                         }
1295 #endif
1296                 }
1297
1298                 ahc_handle_message_phase(ahc);
1299                 break;
1300         }
1301         case PERR_DETECTED:
1302         {
1303                 /*
1304                  * If we've cleared the parity error interrupt
1305                  * but the sequencer still believes that SCSIPERR
1306                  * is true, it must be that the parity error is
1307                  * for the currently presented byte on the bus,
1308                  * and we are not in a phase (data-in) where we will
1309                  * eventually ack this byte.  Ack the byte and
1310                  * throw it away in the hope that the target will
1311                  * take us to message out to deliver the appropriate
1312                  * error message.
1313                  */
1314                 if ((intstat & SCSIINT) == 0
1315                  && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
1316
1317                         if ((ahc->features & AHC_DT) == 0) {
1318                                 u_int curphase;
1319
1320                                 /*
1321                                  * The hardware will only let you ack bytes
1322                                  * if the expected phase in SCSISIGO matches
1323                                  * the current phase.  Make sure this is
1324                                  * currently the case.
1325                                  */
1326                                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1327                                 ahc_outb(ahc, LASTPHASE, curphase);
1328                                 ahc_outb(ahc, SCSISIGO, curphase);
1329                         }
1330                         if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
1331                                 int wait;
1332
1333                                 /*
1334                                  * In a data phase.  Faster to bitbucket
1335                                  * the data than to individually ack each
1336                                  * byte.  This is also the only strategy
1337                                  * that will work with AUTOACK enabled.
1338                                  */
1339                                 ahc_outb(ahc, SXFRCTL1,
1340                                          ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
1341                                 wait = 5000;
1342                                 while (--wait != 0) {
1343                                         if ((ahc_inb(ahc, SCSISIGI)
1344                                           & (CDI|MSGI)) != 0)
1345                                                 break;
1346                                         ahc_delay(100);
1347                                 }
1348                                 ahc_outb(ahc, SXFRCTL1,
1349                                          ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
1350                                 if (wait == 0) {
1351                                         struct  scb *scb;
1352                                         u_int   scb_index;
1353
1354                                         ahc_print_devinfo(ahc, &devinfo);
1355                                         printk("Unable to clear parity error.  "
1356                                                "Resetting bus.\n");
1357                                         scb_index = ahc_inb(ahc, SCB_TAG);
1358                                         scb = ahc_lookup_scb(ahc, scb_index);
1359                                         if (scb != NULL)
1360                                                 ahc_set_transaction_status(scb,
1361                                                     CAM_UNCOR_PARITY);
1362                                         ahc_reset_channel(ahc, devinfo.channel,
1363                                                           /*init reset*/TRUE);
1364                                 }
1365                         } else {
1366                                 ahc_inb(ahc, SCSIDATL);
1367                         }
1368                 }
1369                 break;
1370         }
1371         case DATA_OVERRUN:
1372         {
1373                 /*
1374                  * When the sequencer detects an overrun, it
1375                  * places the controller in "BITBUCKET" mode
1376                  * and allows the target to complete its transfer.
1377                  * Unfortunately, none of the counters get updated
1378                  * when the controller is in this mode, so we have
1379                  * no way of knowing how large the overrun was.
1380                  */
1381                 u_int scbindex = ahc_inb(ahc, SCB_TAG);
1382                 u_int lastphase = ahc_inb(ahc, LASTPHASE);
1383                 u_int i;
1384
1385                 scb = ahc_lookup_scb(ahc, scbindex);
1386                 for (i = 0; i < num_phases; i++) {
1387                         if (lastphase == ahc_phase_table[i].phase)
1388                                 break;
1389                 }
1390                 ahc_print_path(ahc, scb);
1391                 printk("data overrun detected %s."
1392                        "  Tag == 0x%x.\n",
1393                        ahc_phase_table[i].phasemsg,
1394                        scb->hscb->tag);
1395                 ahc_print_path(ahc, scb);
1396                 printk("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
1397                        ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
1398                        ahc_get_transfer_length(scb), scb->sg_count);
1399                 if (scb->sg_count > 0) {
1400                         for (i = 0; i < scb->sg_count; i++) {
1401
1402                                 printk("sg[%d] - Addr 0x%x%x : Length %d\n",
1403                                        i,
1404                                        (ahc_le32toh(scb->sg_list[i].len) >> 24
1405                                         & SG_HIGH_ADDR_BITS),
1406                                        ahc_le32toh(scb->sg_list[i].addr),
1407                                        ahc_le32toh(scb->sg_list[i].len)
1408                                        & AHC_SG_LEN_MASK);
1409                         }
1410                 }
1411                 /*
1412                  * Set this and it will take effect when the
1413                  * target does a command complete.
1414                  */
1415                 ahc_freeze_devq(ahc, scb);
1416                 if ((scb->flags & SCB_SENSE) == 0) {
1417                         ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1418                 } else {
1419                         scb->flags &= ~SCB_SENSE;
1420                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1421                 }
1422                 ahc_freeze_scb(scb);
1423
1424                 if ((ahc->features & AHC_ULTRA2) != 0) {
1425                         /*
1426                          * Clear the channel in case we return
1427                          * to data phase later.
1428                          */
1429                         ahc_outb(ahc, SXFRCTL0,
1430                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1431                         ahc_outb(ahc, SXFRCTL0,
1432                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1433                 }
1434                 if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
1435                         u_int dscommand1;
1436
1437                         /* Ensure HHADDR is 0 for future DMA operations. */
1438                         dscommand1 = ahc_inb(ahc, DSCOMMAND1);
1439                         ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
1440                         ahc_outb(ahc, HADDR, 0);
1441                         ahc_outb(ahc, DSCOMMAND1, dscommand1);
1442                 }
1443                 break;
1444         }
1445         case MKMSG_FAILED:
1446         {
1447                 u_int scbindex;
1448
1449                 printk("%s:%c:%d:%d: Attempt to issue message failed\n",
1450                        ahc_name(ahc), devinfo.channel, devinfo.target,
1451                        devinfo.lun);
1452                 scbindex = ahc_inb(ahc, SCB_TAG);
1453                 scb = ahc_lookup_scb(ahc, scbindex);
1454                 if (scb != NULL
1455                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
1456                         /*
1457                          * Ensure that we didn't put a second instance of this
1458                          * SCB into the QINFIFO.
1459                          */
1460                         ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
1461                                            SCB_GET_CHANNEL(ahc, scb),
1462                                            SCB_GET_LUN(scb), scb->hscb->tag,
1463                                            ROLE_INITIATOR, /*status*/0,
1464                                            SEARCH_REMOVE);
1465                 break;
1466         }
1467         case NO_FREE_SCB:
1468         {
1469                 printk("%s: No free or disconnected SCBs\n", ahc_name(ahc));
1470                 ahc_dump_card_state(ahc);
1471                 panic("for safety");
1472                 break;
1473         }
1474         case SCB_MISMATCH:
1475         {
1476                 u_int scbptr;
1477
1478                 scbptr = ahc_inb(ahc, SCBPTR);
1479                 printk("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
1480                        scbptr, ahc_inb(ahc, ARG_1),
1481                        ahc->scb_data->hscbs[scbptr].tag);
1482                 ahc_dump_card_state(ahc);
1483                 panic("for safety");
1484                 break;
1485         }
1486         case OUT_OF_RANGE:
1487         {
1488                 printk("%s: BTT calculation out of range\n", ahc_name(ahc));
1489                 printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1490                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
1491                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1492                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1493                 printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1494                        "SINDEX == 0x%x\n, A == 0x%x\n",
1495                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1496                        ahc_index_busy_tcl(ahc,
1497                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1498                                       ahc_inb(ahc, SAVED_LUN))),
1499                        ahc_inb(ahc, SINDEX),
1500                        ahc_inb(ahc, ACCUM));
1501                 printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1502                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1503                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1504                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1505                        ahc_inb(ahc, SCB_CONTROL));
1506                 printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1507                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1508                 ahc_dump_card_state(ahc);
1509                 panic("for safety");
1510                 break;
1511         }
1512         default:
1513                 printk("ahc_intr: seqint, "
1514                        "intstat == 0x%x, scsisigi = 0x%x\n",
1515                        intstat, ahc_inb(ahc, SCSISIGI));
1516                 break;
1517         }
1518 unpause:
1519         /*
1520          *  The sequencer is paused immediately on
1521          *  a SEQINT, so we should restart it when
1522          *  we're done.
1523          */
1524         ahc_unpause(ahc);
1525 }
1526
1527 static void
1528 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
1529 {
1530         u_int   scb_index;
1531         u_int   status0;
1532         u_int   status;
1533         struct  scb *scb;
1534         char    cur_channel;
1535         char    intr_channel;
1536
1537         if ((ahc->features & AHC_TWIN) != 0
1538          && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
1539                 cur_channel = 'B';
1540         else
1541                 cur_channel = 'A';
1542         intr_channel = cur_channel;
1543
1544         if ((ahc->features & AHC_ULTRA2) != 0)
1545                 status0 = ahc_inb(ahc, SSTAT0) & IOERR;
1546         else
1547                 status0 = 0;
1548         status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1549         if (status == 0 && status0 == 0) {
1550                 if ((ahc->features & AHC_TWIN) != 0) {
1551                         /* Try the other channel */
1552                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1553                         status = ahc_inb(ahc, SSTAT1)
1554                                & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1555                         intr_channel = (cur_channel == 'A') ? 'B' : 'A';
1556                 }
1557                 if (status == 0) {
1558                         printk("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
1559                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1560                         ahc_unpause(ahc);
1561                         return;
1562                 }
1563         }
1564
1565         /* Make sure the sequencer is in a safe location. */
1566         ahc_clear_critical_section(ahc);
1567
1568         scb_index = ahc_inb(ahc, SCB_TAG);
1569         scb = ahc_lookup_scb(ahc, scb_index);
1570         if (scb != NULL
1571          && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1572                 scb = NULL;
1573
1574         if ((ahc->features & AHC_ULTRA2) != 0
1575          && (status0 & IOERR) != 0) {
1576                 int now_lvd;
1577
1578                 now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1579                 printk("%s: Transceiver State Has Changed to %s mode\n",
1580                        ahc_name(ahc), now_lvd ? "LVD" : "SE");
1581                 ahc_outb(ahc, CLRSINT0, CLRIOERR);
1582                 /*
1583                  * When transitioning to SE mode, the reset line
1584                  * glitches, triggering an arbitration bug in some
1585                  * Ultra2 controllers.  This bug is cleared when we
1586                  * assert the reset line.  Since a reset glitch has
1587                  * already occurred with this transition and a
1588                  * transceiver state change is handled just like
1589                  * a bus reset anyway, asserting the reset line
1590                  * ourselves is safe.
1591                  */
1592                 ahc_reset_channel(ahc, intr_channel,
1593                                  /*Initiate Reset*/now_lvd == 0);
1594         } else if ((status & SCSIRSTI) != 0) {
1595                 printk("%s: Someone reset channel %c\n",
1596                         ahc_name(ahc), intr_channel);
1597                 if (intr_channel != cur_channel)
1598                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1599                 ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1600         } else if ((status & SCSIPERR) != 0) {
1601                 /*
1602                  * Determine the bus phase and queue an appropriate message.
1603                  * SCSIPERR is latched true as soon as a parity error
1604                  * occurs.  If the sequencer acked the transfer that
1605                  * caused the parity error and the currently presented
1606                  * transfer on the bus has correct parity, SCSIPERR will
1607                  * be cleared by CLRSCSIPERR.  Use this to determine if
1608                  * we should look at the last phase the sequencer recorded,
1609                  * or the current phase presented on the bus.
1610                  */
1611                 struct  ahc_devinfo devinfo;
1612                 u_int   mesg_out;
1613                 u_int   curphase;
1614                 u_int   errorphase;
1615                 u_int   lastphase;
1616                 u_int   scsirate;
1617                 u_int   i;
1618                 u_int   sstat2;
1619                 int     silent;
1620
1621                 lastphase = ahc_inb(ahc, LASTPHASE);
1622                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1623                 sstat2 = ahc_inb(ahc, SSTAT2);
1624                 ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1625                 /*
1626                  * For all phases save DATA, the sequencer won't
1627                  * automatically ack a byte that has a parity error
1628                  * in it.  So the only way that the current phase
1629                  * could be 'data-in' is if the parity error is for
1630                  * an already acked byte in the data phase.  During
1631                  * synchronous data-in transfers, we may actually
1632                  * ack bytes before latching the current phase in
1633                  * LASTPHASE, leading to the discrepancy between
1634                  * curphase and lastphase.
1635                  */
1636                 if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1637                  || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1638                         errorphase = curphase;
1639                 else
1640                         errorphase = lastphase;
1641
1642                 for (i = 0; i < num_phases; i++) {
1643                         if (errorphase == ahc_phase_table[i].phase)
1644                                 break;
1645                 }
1646                 mesg_out = ahc_phase_table[i].mesg_out;
1647                 silent = FALSE;
1648                 if (scb != NULL) {
1649                         if (SCB_IS_SILENT(scb))
1650                                 silent = TRUE;
1651                         else
1652                                 ahc_print_path(ahc, scb);
1653                         scb->flags |= SCB_TRANSMISSION_ERROR;
1654                 } else
1655                         printk("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1656                                SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1657                 scsirate = ahc_inb(ahc, SCSIRATE);
1658                 if (silent == FALSE) {
1659                         printk("parity error detected %s. "
1660                                "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1661                                ahc_phase_table[i].phasemsg,
1662                                ahc_inw(ahc, SEQADDR0),
1663                                scsirate);
1664                         if ((ahc->features & AHC_DT) != 0) {
1665                                 if ((sstat2 & CRCVALERR) != 0)
1666                                         printk("\tCRC Value Mismatch\n");
1667                                 if ((sstat2 & CRCENDERR) != 0)
1668                                         printk("\tNo terminal CRC packet "
1669                                                "received\n");
1670                                 if ((sstat2 & CRCREQERR) != 0)
1671                                         printk("\tIllegal CRC packet "
1672                                                "request\n");
1673                                 if ((sstat2 & DUAL_EDGE_ERR) != 0)
1674                                         printk("\tUnexpected %sDT Data Phase\n",
1675                                                (scsirate & SINGLE_EDGE)
1676                                              ? "" : "non-");
1677                         }
1678                 }
1679
1680                 if ((ahc->features & AHC_DT) != 0
1681                  && (sstat2 & DUAL_EDGE_ERR) != 0) {
1682                         /*
1683                          * This error applies regardless of
1684                          * data direction, so ignore the value
1685                          * in the phase table.
1686                          */
1687                         mesg_out = MSG_INITIATOR_DET_ERR;
1688                 }
1689
1690                 /*
1691                  * We've set the hardware to assert ATN if we
1692                  * get a parity error on "in" phases, so all we
1693                  * need to do is stuff the message buffer with
1694                  * the appropriate message.  "In" phases have set
1695                  * mesg_out to something other than MSG_NOP.
1696                  */
1697                 if (mesg_out != MSG_NOOP) {
1698                         if (ahc->msg_type != MSG_TYPE_NONE)
1699                                 ahc->send_msg_perror = TRUE;
1700                         else
1701                                 ahc_outb(ahc, MSG_OUT, mesg_out);
1702                 }
1703                 /*
1704                  * Force a renegotiation with this target just in
1705                  * case we are out of sync for some external reason
1706                  * unknown (or unreported) by the target.
1707                  */
1708                 ahc_fetch_devinfo(ahc, &devinfo);
1709                 ahc_force_renegotiation(ahc, &devinfo);
1710
1711                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1712                 ahc_unpause(ahc);
1713         } else if ((status & SELTO) != 0) {
1714                 u_int   scbptr;
1715
1716                 /* Stop the selection */
1717                 ahc_outb(ahc, SCSISEQ, 0);
1718
1719                 /* No more pending messages */
1720                 ahc_clear_msg_state(ahc);
1721
1722                 /* Clear interrupt state */
1723                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1724                 ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1725
1726                 /*
1727                  * Although the driver does not care about the
1728                  * 'Selection in Progress' status bit, the busy
1729                  * LED does.  SELINGO is only cleared by a successful
1730                  * selection, so we must manually clear it to insure
1731                  * the LED turns off just incase no future successful
1732                  * selections occur (e.g. no devices on the bus).
1733                  */
1734                 ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1735
1736                 scbptr = ahc_inb(ahc, WAITING_SCBH);
1737                 ahc_outb(ahc, SCBPTR, scbptr);
1738                 scb_index = ahc_inb(ahc, SCB_TAG);
1739
1740                 scb = ahc_lookup_scb(ahc, scb_index);
1741                 if (scb == NULL) {
1742                         printk("%s: ahc_intr - referenced scb not "
1743                                "valid during SELTO scb(%d, %d)\n",
1744                                ahc_name(ahc), scbptr, scb_index);
1745                         ahc_dump_card_state(ahc);
1746                 } else {
1747                         struct ahc_devinfo devinfo;
1748 #ifdef AHC_DEBUG
1749                         if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1750                                 ahc_print_path(ahc, scb);
1751                                 printk("Saw Selection Timeout for SCB 0x%x\n",
1752                                        scb_index);
1753                         }
1754 #endif
1755                         ahc_scb_devinfo(ahc, &devinfo, scb);
1756                         ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1757                         ahc_freeze_devq(ahc, scb);
1758
1759                         /*
1760                          * Cancel any pending transactions on the device
1761                          * now that it seems to be missing.  This will
1762                          * also revert us to async/narrow transfers until
1763                          * we can renegotiate with the device.
1764                          */
1765                         ahc_handle_devreset(ahc, &devinfo,
1766                                             CAM_SEL_TIMEOUT,
1767                                             "Selection Timeout",
1768                                             /*verbose_level*/1);
1769                 }
1770                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1771                 ahc_restart(ahc);
1772         } else if ((status & BUSFREE) != 0
1773                 && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1774                 struct  ahc_devinfo devinfo;
1775                 u_int   lastphase;
1776                 u_int   saved_scsiid;
1777                 u_int   saved_lun;
1778                 u_int   target;
1779                 u_int   initiator_role_id;
1780                 char    channel;
1781                 int     printerror;
1782
1783                 /*
1784                  * Clear our selection hardware as soon as possible.
1785                  * We may have an entry in the waiting Q for this target,
1786                  * that is affected by this busfree and we don't want to
1787                  * go about selecting the target while we handle the event.
1788                  */
1789                 ahc_outb(ahc, SCSISEQ,
1790                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1791
1792                 /*
1793                  * Disable busfree interrupts and clear the busfree
1794                  * interrupt status.  We do this here so that several
1795                  * bus transactions occur prior to clearing the SCSIINT
1796                  * latch.  It can take a bit for the clearing to take effect.
1797                  */
1798                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1799                 ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1800
1801                 /*
1802                  * Look at what phase we were last in.
1803                  * If its message out, chances are pretty good
1804                  * that the busfree was in response to one of
1805                  * our abort requests.
1806                  */
1807                 lastphase = ahc_inb(ahc, LASTPHASE);
1808                 saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1809                 saved_lun = ahc_inb(ahc, SAVED_LUN);
1810                 target = SCSIID_TARGET(ahc, saved_scsiid);
1811                 initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1812                 channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1813                 ahc_compile_devinfo(&devinfo, initiator_role_id,
1814                                     target, saved_lun, channel, ROLE_INITIATOR);
1815                 printerror = 1;
1816
1817                 if (lastphase == P_MESGOUT) {
1818                         u_int tag;
1819
1820                         tag = SCB_LIST_NULL;
1821                         if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1822                          || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1823                                 if (ahc->msgout_buf[ahc->msgout_index - 1]
1824                                  == MSG_ABORT_TAG)
1825                                         tag = scb->hscb->tag;
1826                                 ahc_print_path(ahc, scb);
1827                                 printk("SCB %d - Abort%s Completed.\n",
1828                                        scb->hscb->tag, tag == SCB_LIST_NULL ?
1829                                        "" : " Tag");
1830                                 ahc_abort_scbs(ahc, target, channel,
1831                                                saved_lun, tag,
1832                                                ROLE_INITIATOR,
1833                                                CAM_REQ_ABORTED);
1834                                 printerror = 0;
1835                         } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1836                                                 MSG_BUS_DEV_RESET, TRUE)) {
1837                                 ahc_compile_devinfo(&devinfo,
1838                                                     initiator_role_id,
1839                                                     target,
1840                                                     CAM_LUN_WILDCARD,
1841                                                     channel,
1842                                                     ROLE_INITIATOR);
1843                                 ahc_handle_devreset(ahc, &devinfo,
1844                                                     CAM_BDR_SENT,
1845                                                     "Bus Device Reset",
1846                                                     /*verbose_level*/0);
1847                                 printerror = 0;
1848                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1849                                                 MSG_EXT_PPR, FALSE)) {
1850                                 struct ahc_initiator_tinfo *tinfo;
1851                                 struct ahc_tmode_tstate *tstate;
1852
1853                                 /*
1854                                  * PPR Rejected.  Try non-ppr negotiation
1855                                  * and retry command.
1856                                  */
1857                                 tinfo = ahc_fetch_transinfo(ahc,
1858                                                             devinfo.channel,
1859                                                             devinfo.our_scsiid,
1860                                                             devinfo.target,
1861                                                             &tstate);
1862                                 tinfo->curr.transport_version = 2;
1863                                 tinfo->goal.transport_version = 2;
1864                                 tinfo->goal.ppr_options = 0;
1865                                 ahc_qinfifo_requeue_tail(ahc, scb);
1866                                 printerror = 0;
1867                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1868                                                 MSG_EXT_WDTR, FALSE)) {
1869                                 /*
1870                                  * Negotiation Rejected.  Go-narrow and
1871                                  * retry command.
1872                                  */
1873                                 ahc_set_width(ahc, &devinfo,
1874                                               MSG_EXT_WDTR_BUS_8_BIT,
1875                                               AHC_TRANS_CUR|AHC_TRANS_GOAL,
1876                                               /*paused*/TRUE);
1877                                 ahc_qinfifo_requeue_tail(ahc, scb);
1878                                 printerror = 0;
1879                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1880                                                 MSG_EXT_SDTR, FALSE)) {
1881                                 /*
1882                                  * Negotiation Rejected.  Go-async and
1883                                  * retry command.
1884                                  */
1885                                 ahc_set_syncrate(ahc, &devinfo,
1886                                                 /*syncrate*/NULL,
1887                                                 /*period*/0, /*offset*/0,
1888                                                 /*ppr_options*/0,
1889                                                 AHC_TRANS_CUR|AHC_TRANS_GOAL,
1890                                                 /*paused*/TRUE);
1891                                 ahc_qinfifo_requeue_tail(ahc, scb);
1892                                 printerror = 0;
1893                         }
1894                 }
1895                 if (printerror != 0) {
1896                         u_int i;
1897
1898                         if (scb != NULL) {
1899                                 u_int tag;
1900
1901                                 if ((scb->hscb->control & TAG_ENB) != 0)
1902                                         tag = scb->hscb->tag;
1903                                 else
1904                                         tag = SCB_LIST_NULL;
1905                                 ahc_print_path(ahc, scb);
1906                                 ahc_abort_scbs(ahc, target, channel,
1907                                                SCB_GET_LUN(scb), tag,
1908                                                ROLE_INITIATOR,
1909                                                CAM_UNEXP_BUSFREE);
1910                         } else {
1911                                 /*
1912                                  * We had not fully identified this connection,
1913                                  * so we cannot abort anything.
1914                                  */
1915                                 printk("%s: ", ahc_name(ahc));
1916                         }
1917                         for (i = 0; i < num_phases; i++) {
1918                                 if (lastphase == ahc_phase_table[i].phase)
1919                                         break;
1920                         }
1921                         if (lastphase != P_BUSFREE) {
1922                                 /*
1923                                  * Renegotiate with this device at the
1924                                  * next opportunity just in case this busfree
1925                                  * is due to a negotiation mismatch with the
1926                                  * device.
1927                                  */
1928                                 ahc_force_renegotiation(ahc, &devinfo);
1929                         }
1930                         printk("Unexpected busfree %s\n"
1931                                "SEQADDR == 0x%x\n",
1932                                ahc_phase_table[i].phasemsg,
1933                                ahc_inb(ahc, SEQADDR0)
1934                                 | (ahc_inb(ahc, SEQADDR1) << 8));
1935                 }
1936                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1937                 ahc_restart(ahc);
1938         } else {
1939                 printk("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1940                        ahc_name(ahc), status);
1941                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1942         }
1943 }
1944
1945 /*
1946  * Force renegotiation to occur the next time we initiate
1947  * a command to the current device.
1948  */
1949 static void
1950 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1951 {
1952         struct  ahc_initiator_tinfo *targ_info;
1953         struct  ahc_tmode_tstate *tstate;
1954
1955         targ_info = ahc_fetch_transinfo(ahc,
1956                                         devinfo->channel,
1957                                         devinfo->our_scsiid,
1958                                         devinfo->target,
1959                                         &tstate);
1960         ahc_update_neg_request(ahc, devinfo, tstate,
1961                                targ_info, AHC_NEG_IF_NON_ASYNC);
1962 }
1963
1964 #define AHC_MAX_STEPS 2000
1965 static void
1966 ahc_clear_critical_section(struct ahc_softc *ahc)
1967 {
1968         int     stepping;
1969         int     steps;
1970         u_int   simode0;
1971         u_int   simode1;
1972
1973         if (ahc->num_critical_sections == 0)
1974                 return;
1975
1976         stepping = FALSE;
1977         steps = 0;
1978         simode0 = 0;
1979         simode1 = 0;
1980         for (;;) {
1981                 struct  cs *cs;
1982                 u_int   seqaddr;
1983                 u_int   i;
1984
1985                 seqaddr = ahc_inb(ahc, SEQADDR0)
1986                         | (ahc_inb(ahc, SEQADDR1) << 8);
1987
1988                 /*
1989                  * Seqaddr represents the next instruction to execute,
1990                  * so we are really executing the instruction just
1991                  * before it.
1992                  */
1993                 if (seqaddr != 0)
1994                         seqaddr -= 1;
1995                 cs = ahc->critical_sections;
1996                 for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1997
1998                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1999                                 break;
2000                 }
2001
2002                 if (i == ahc->num_critical_sections)
2003                         break;
2004
2005                 if (steps > AHC_MAX_STEPS) {
2006                         printk("%s: Infinite loop in critical section\n",
2007                                ahc_name(ahc));
2008                         ahc_dump_card_state(ahc);
2009                         panic("critical section loop");
2010                 }
2011
2012                 steps++;
2013                 if (stepping == FALSE) {
2014
2015                         /*
2016                          * Disable all interrupt sources so that the
2017                          * sequencer will not be stuck by a pausing
2018                          * interrupt condition while we attempt to
2019                          * leave a critical section.
2020                          */
2021                         simode0 = ahc_inb(ahc, SIMODE0);
2022                         ahc_outb(ahc, SIMODE0, 0);
2023                         simode1 = ahc_inb(ahc, SIMODE1);
2024                         if ((ahc->features & AHC_DT) != 0)
2025                                 /*
2026                                  * On DT class controllers, we
2027                                  * use the enhanced busfree logic.
2028                                  * Unfortunately we cannot re-enable
2029                                  * busfree detection within the
2030                                  * current connection, so we must
2031                                  * leave it on while single stepping.
2032                                  */
2033                                 ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
2034                         else
2035                                 ahc_outb(ahc, SIMODE1, 0);
2036                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
2037                         ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
2038                         stepping = TRUE;
2039                 }
2040                 if ((ahc->features & AHC_DT) != 0) {
2041                         ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
2042                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
2043                 }
2044                 ahc_outb(ahc, HCNTRL, ahc->unpause);
2045                 while (!ahc_is_paused(ahc))
2046                         ahc_delay(200);
2047         }
2048         if (stepping) {
2049                 ahc_outb(ahc, SIMODE0, simode0);
2050                 ahc_outb(ahc, SIMODE1, simode1);
2051                 ahc_outb(ahc, SEQCTL, ahc->seqctl);
2052         }
2053 }
2054
2055 /*
2056  * Clear any pending interrupt status.
2057  */
2058 static void
2059 ahc_clear_intstat(struct ahc_softc *ahc)
2060 {
2061         /* Clear any interrupt conditions this may have caused */
2062         ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2063                                 |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
2064                                 CLRREQINIT);
2065         ahc_flush_device_writes(ahc);
2066         ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
2067         ahc_flush_device_writes(ahc);
2068         ahc_outb(ahc, CLRINT, CLRSCSIINT);
2069         ahc_flush_device_writes(ahc);
2070 }
2071
2072 /**************************** Debugging Routines ******************************/
2073 #ifdef AHC_DEBUG
2074 uint32_t ahc_debug = AHC_DEBUG_OPTS;
2075 #endif
2076
2077 #if 0 /* unused */
2078 static void
2079 ahc_print_scb(struct scb *scb)
2080 {
2081         int i;
2082
2083         struct hardware_scb *hscb = scb->hscb;
2084
2085         printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2086                (void *)scb,
2087                hscb->control,
2088                hscb->scsiid,
2089                hscb->lun,
2090                hscb->cdb_len);
2091         printk("Shared Data: ");
2092         for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
2093                 printk("%#02x", hscb->shared_data.cdb[i]);
2094         printk("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
2095                 ahc_le32toh(hscb->dataptr),
2096                 ahc_le32toh(hscb->datacnt),
2097                 ahc_le32toh(hscb->sgptr),
2098                 hscb->tag);
2099         if (scb->sg_count > 0) {
2100                 for (i = 0; i < scb->sg_count; i++) {
2101                         printk("sg[%d] - Addr 0x%x%x : Length %d\n",
2102                                i,
2103                                (ahc_le32toh(scb->sg_list[i].len) >> 24
2104                                 & SG_HIGH_ADDR_BITS),
2105                                ahc_le32toh(scb->sg_list[i].addr),
2106                                ahc_le32toh(scb->sg_list[i].len));
2107                 }
2108         }
2109 }
2110 #endif
2111
2112 /************************* Transfer Negotiation *******************************/
2113 /*
2114  * Allocate per target mode instance (ID we respond to as a target)
2115  * transfer negotiation data structures.
2116  */
2117 static struct ahc_tmode_tstate *
2118 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
2119 {
2120         struct ahc_tmode_tstate *master_tstate;
2121         struct ahc_tmode_tstate *tstate;
2122         int i;
2123
2124         master_tstate = ahc->enabled_targets[ahc->our_id];
2125         if (channel == 'B') {
2126                 scsi_id += 8;
2127                 master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
2128         }
2129         if (ahc->enabled_targets[scsi_id] != NULL
2130          && ahc->enabled_targets[scsi_id] != master_tstate)
2131                 panic("%s: ahc_alloc_tstate - Target already allocated",
2132                       ahc_name(ahc));
2133         tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
2134         if (tstate == NULL)
2135                 return (NULL);
2136
2137         /*
2138          * If we have allocated a master tstate, copy user settings from
2139          * the master tstate (taken from SRAM or the EEPROM) for this
2140          * channel, but reset our current and goal settings to async/narrow
2141          * until an initiator talks to us.
2142          */
2143         if (master_tstate != NULL) {
2144                 memcpy(tstate, master_tstate, sizeof(*tstate));
2145                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2146                 tstate->ultraenb = 0;
2147                 for (i = 0; i < AHC_NUM_TARGETS; i++) {
2148                         memset(&tstate->transinfo[i].curr, 0,
2149                               sizeof(tstate->transinfo[i].curr));
2150                         memset(&tstate->transinfo[i].goal, 0,
2151                               sizeof(tstate->transinfo[i].goal));
2152                 }
2153         } else
2154                 memset(tstate, 0, sizeof(*tstate));
2155         ahc->enabled_targets[scsi_id] = tstate;
2156         return (tstate);
2157 }
2158
2159 #ifdef AHC_TARGET_MODE
2160 /*
2161  * Free per target mode instance (ID we respond to as a target)
2162  * transfer negotiation data structures.
2163  */
2164 static void
2165 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
2166 {
2167         struct ahc_tmode_tstate *tstate;
2168
2169         /*
2170          * Don't clean up our "master" tstate.
2171          * It has our default user settings.
2172          */
2173         if (((channel == 'B' && scsi_id == ahc->our_id_b)
2174           || (channel == 'A' && scsi_id == ahc->our_id))
2175          && force == FALSE)
2176                 return;
2177
2178         if (channel == 'B')
2179                 scsi_id += 8;
2180         tstate = ahc->enabled_targets[scsi_id];
2181         kfree(tstate);
2182         ahc->enabled_targets[scsi_id] = NULL;
2183 }
2184 #endif
2185
2186 /*
2187  * Called when we have an active connection to a target on the bus,
2188  * this function finds the nearest syncrate to the input period limited
2189  * by the capabilities of the bus connectivity of and sync settings for
2190  * the target.
2191  */
2192 static const struct ahc_syncrate *
2193 ahc_devlimited_syncrate(struct ahc_softc *ahc,
2194                         struct ahc_initiator_tinfo *tinfo,
2195                         u_int *period, u_int *ppr_options, role_t role)
2196 {
2197         struct  ahc_transinfo *transinfo;
2198         u_int   maxsync;
2199
2200         if ((ahc->features & AHC_ULTRA2) != 0) {
2201                 if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
2202                  && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
2203                         maxsync = AHC_SYNCRATE_DT;
2204                 } else {
2205                         maxsync = AHC_SYNCRATE_ULTRA;
2206                         /* Can't do DT on an SE bus */
2207                         *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2208                 }
2209         } else if ((ahc->features & AHC_ULTRA) != 0) {
2210                 maxsync = AHC_SYNCRATE_ULTRA;
2211         } else {
2212                 maxsync = AHC_SYNCRATE_FAST;
2213         }
2214         /*
2215          * Never allow a value higher than our current goal
2216          * period otherwise we may allow a target initiated
2217          * negotiation to go above the limit as set by the
2218          * user.  In the case of an initiator initiated
2219          * sync negotiation, we limit based on the user
2220          * setting.  This allows the system to still accept
2221          * incoming negotiations even if target initiated
2222          * negotiation is not performed.
2223          */
2224         if (role == ROLE_TARGET)
2225                 transinfo = &tinfo->user;
2226         else
2227                 transinfo = &tinfo->goal;
2228         *ppr_options &= transinfo->ppr_options;
2229         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2230                 maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
2231                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2232         }
2233         if (transinfo->period == 0) {
2234                 *period = 0;
2235                 *ppr_options = 0;
2236                 return (NULL);
2237         }
2238         *period = max(*period, (u_int)transinfo->period);
2239         return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
2240 }
2241
2242 /*
2243  * Look up the valid period to SCSIRATE conversion in our table.
2244  * Return the period and offset that should be sent to the target
2245  * if this was the beginning of an SDTR.
2246  */
2247 const struct ahc_syncrate *
2248 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
2249                   u_int *ppr_options, u_int maxsync)
2250 {
2251         const struct ahc_syncrate *syncrate;
2252
2253         if ((ahc->features & AHC_DT) == 0)
2254                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2255
2256         /* Skip all DT only entries if DT is not available */
2257         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2258          && maxsync < AHC_SYNCRATE_ULTRA2)
2259                 maxsync = AHC_SYNCRATE_ULTRA2;
2260
2261         /* Now set the maxsync based on the card capabilities
2262          * DT is already done above */
2263         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2264             && maxsync < AHC_SYNCRATE_ULTRA)
2265                 maxsync = AHC_SYNCRATE_ULTRA;
2266         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2267             && maxsync < AHC_SYNCRATE_FAST)
2268                 maxsync = AHC_SYNCRATE_FAST;
2269
2270         for (syncrate = &ahc_syncrates[maxsync];
2271              syncrate->rate != NULL;
2272              syncrate++) {
2273
2274                 /*
2275                  * The Ultra2 table doesn't go as low
2276                  * as for the Fast/Ultra cards.
2277                  */
2278                 if ((ahc->features & AHC_ULTRA2) != 0
2279                  && (syncrate->sxfr_u2 == 0))
2280                         break;
2281
2282                 if (*period <= syncrate->period) {
2283                         /*
2284                          * When responding to a target that requests
2285                          * sync, the requested rate may fall between
2286                          * two rates that we can output, but still be
2287                          * a rate that we can receive.  Because of this,
2288                          * we want to respond to the target with
2289                          * the same rate that it sent to us even
2290                          * if the period we use to send data to it
2291                          * is lower.  Only lower the response period
2292                          * if we must.
2293                          */
2294                         if (syncrate == &ahc_syncrates[maxsync])
2295                                 *period = syncrate->period;
2296
2297                         /*
2298                          * At some speeds, we only support
2299                          * ST transfers.
2300                          */
2301                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
2302                                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2303                         break;
2304                 }
2305         }
2306
2307         if ((*period == 0)
2308          || (syncrate->rate == NULL)
2309          || ((ahc->features & AHC_ULTRA2) != 0
2310           && (syncrate->sxfr_u2 == 0))) {
2311                 /* Use asynchronous transfers. */
2312                 *period = 0;
2313                 syncrate = NULL;
2314                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2315         }
2316         return (syncrate);
2317 }
2318
2319 /*
2320  * Convert from an entry in our syncrate table to the SCSI equivalent
2321  * sync "period" factor.
2322  */
2323 u_int
2324 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
2325 {
2326         const struct ahc_syncrate *syncrate;
2327
2328         if ((ahc->features & AHC_ULTRA2) != 0)
2329                 scsirate &= SXFR_ULTRA2;
2330         else
2331                 scsirate &= SXFR;
2332
2333         /* now set maxsync based on card capabilities */
2334         if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
2335                 maxsync = AHC_SYNCRATE_ULTRA2;
2336         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2337             && maxsync < AHC_SYNCRATE_ULTRA)
2338                 maxsync = AHC_SYNCRATE_ULTRA;
2339         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2340             && maxsync < AHC_SYNCRATE_FAST)
2341                 maxsync = AHC_SYNCRATE_FAST;
2342
2343
2344         syncrate = &ahc_syncrates[maxsync];
2345         while (syncrate->rate != NULL) {
2346
2347                 if ((ahc->features & AHC_ULTRA2) != 0) {
2348                         if (syncrate->sxfr_u2 == 0)
2349                                 break;
2350                         else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
2351                                 return (syncrate->period);
2352                 } else if (scsirate == (syncrate->sxfr & SXFR)) {
2353                                 return (syncrate->period);
2354                 }
2355                 syncrate++;
2356         }
2357         return (0); /* async */
2358 }
2359
2360 /*
2361  * Truncate the given synchronous offset to a value the
2362  * current adapter type and syncrate are capable of.
2363  */
2364 static void
2365 ahc_validate_offset(struct ahc_softc *ahc,
2366                     struct ahc_initiator_tinfo *tinfo,
2367                     const struct ahc_syncrate *syncrate,
2368                     u_int *offset, int wide, role_t role)
2369 {
2370         u_int maxoffset;
2371
2372         /* Limit offset to what we can do */
2373         if (syncrate == NULL) {
2374                 maxoffset = 0;
2375         } else if ((ahc->features & AHC_ULTRA2) != 0) {
2376                 maxoffset = MAX_OFFSET_ULTRA2;
2377         } else {
2378                 if (wide)
2379                         maxoffset = MAX_OFFSET_16BIT;
2380                 else
2381                         maxoffset = MAX_OFFSET_8BIT;
2382         }
2383         *offset = min(*offset, maxoffset);
2384         if (tinfo != NULL) {
2385                 if (role == ROLE_TARGET)
2386                         *offset = min(*offset, (u_int)tinfo->user.offset);
2387                 else
2388                         *offset = min(*offset, (u_int)tinfo->goal.offset);
2389         }
2390 }
2391
2392 /*
2393  * Truncate the given transfer width parameter to a value the
2394  * current adapter type is capable of.
2395  */
2396 static void
2397 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
2398                    u_int *bus_width, role_t role)
2399 {
2400         switch (*bus_width) {
2401         default:
2402                 if (ahc->features & AHC_WIDE) {
2403                         /* Respond Wide */
2404                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2405                         break;
2406                 }
2407                 fallthrough;
2408         case MSG_EXT_WDTR_BUS_8_BIT:
2409                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2410                 break;
2411         }
2412         if (tinfo != NULL) {
2413                 if (role == ROLE_TARGET)
2414                         *bus_width = min((u_int)tinfo->user.width, *bus_width);
2415                 else
2416                         *bus_width = min((u_int)tinfo->goal.width, *bus_width);
2417         }
2418 }
2419
2420 /*
2421  * Update the bitmask of targets for which the controller should
2422  * negotiate with at the next convenient opportunity.  This currently
2423  * means the next time we send the initial identify messages for
2424  * a new transaction.
2425  */
2426 int
2427 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2428                        struct ahc_tmode_tstate *tstate,
2429                        struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
2430 {
2431         u_int auto_negotiate_orig;
2432
2433         auto_negotiate_orig = tstate->auto_negotiate;
2434         if (neg_type == AHC_NEG_ALWAYS) {
2435                 /*
2436                  * Force our "current" settings to be
2437                  * unknown so that unless a bus reset
2438                  * occurs the need to renegotiate is
2439                  * recorded persistently.
2440                  */
2441                 if ((ahc->features & AHC_WIDE) != 0)
2442                         tinfo->curr.width = AHC_WIDTH_UNKNOWN;
2443                 tinfo->curr.period = AHC_PERIOD_UNKNOWN;
2444                 tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
2445         }
2446         if (tinfo->curr.period != tinfo->goal.period
2447          || tinfo->curr.width != tinfo->goal.width
2448          || tinfo->curr.offset != tinfo->goal.offset
2449          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2450          || (neg_type == AHC_NEG_IF_NON_ASYNC
2451           && (tinfo->goal.offset != 0
2452            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2453            || tinfo->goal.ppr_options != 0)))
2454                 tstate->auto_negotiate |= devinfo->target_mask;
2455         else
2456                 tstate->auto_negotiate &= ~devinfo->target_mask;
2457
2458         return (auto_negotiate_orig != tstate->auto_negotiate);
2459 }
2460
2461 /*
2462  * Update the user/goal/curr tables of synchronous negotiation
2463  * parameters as well as, in the case of a current or active update,
2464  * any data structures on the host controller.  In the case of an
2465  * active update, the specified target is currently talking to us on
2466  * the bus, so the transfer parameter update must take effect
2467  * immediately.
2468  */
2469 void
2470 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2471                  const struct ahc_syncrate *syncrate, u_int period,
2472                  u_int offset, u_int ppr_options, u_int type, int paused)
2473 {
2474         struct  ahc_initiator_tinfo *tinfo;
2475         struct  ahc_tmode_tstate *tstate;
2476         u_int   old_period;
2477         u_int   old_offset;
2478         u_int   old_ppr;
2479         int     active;
2480         int     update_needed;
2481
2482         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2483         update_needed = 0;
2484
2485         if (syncrate == NULL) {
2486                 period = 0;
2487                 offset = 0;
2488         }
2489
2490         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2491                                     devinfo->target, &tstate);
2492
2493         if ((type & AHC_TRANS_USER) != 0) {
2494                 tinfo->user.period = period;
2495                 tinfo->user.offset = offset;
2496                 tinfo->user.ppr_options = ppr_options;
2497         }
2498
2499         if ((type & AHC_TRANS_GOAL) != 0) {
2500                 tinfo->goal.period = period;
2501                 tinfo->goal.offset = offset;
2502                 tinfo->goal.ppr_options = ppr_options;
2503         }
2504
2505         old_period = tinfo->curr.period;
2506         old_offset = tinfo->curr.offset;
2507         old_ppr    = tinfo->curr.ppr_options;
2508
2509         if ((type & AHC_TRANS_CUR) != 0
2510          && (old_period != period
2511           || old_offset != offset
2512           || old_ppr != ppr_options)) {
2513                 u_int   scsirate;
2514
2515                 update_needed++;
2516                 scsirate = tinfo->scsirate;
2517                 if ((ahc->features & AHC_ULTRA2) != 0) {
2518
2519                         scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
2520                         if (syncrate != NULL) {
2521                                 scsirate |= syncrate->sxfr_u2;
2522                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
2523                                         scsirate |= ENABLE_CRC;
2524                                 else
2525                                         scsirate |= SINGLE_EDGE;
2526                         }
2527                 } else {
2528
2529                         scsirate &= ~(SXFR|SOFS);
2530                         /*
2531                          * Ensure Ultra mode is set properly for
2532                          * this target.
2533                          */
2534                         tstate->ultraenb &= ~devinfo->target_mask;
2535                         if (syncrate != NULL) {
2536                                 if (syncrate->sxfr & ULTRA_SXFR) {
2537                                         tstate->ultraenb |=
2538                                                 devinfo->target_mask;
2539                                 }
2540                                 scsirate |= syncrate->sxfr & SXFR;
2541                                 scsirate |= offset & SOFS;
2542                         }
2543                         if (active) {
2544                                 u_int sxfrctl0;
2545
2546                                 sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
2547                                 sxfrctl0 &= ~FAST20;
2548                                 if (tstate->ultraenb & devinfo->target_mask)
2549                                         sxfrctl0 |= FAST20;
2550                                 ahc_outb(ahc, SXFRCTL0, sxfrctl0);
2551                         }
2552                 }
2553                 if (active) {
2554                         ahc_outb(ahc, SCSIRATE, scsirate);
2555                         if ((ahc->features & AHC_ULTRA2) != 0)
2556                                 ahc_outb(ahc, SCSIOFFSET, offset);
2557                 }
2558
2559                 tinfo->scsirate = scsirate;
2560                 tinfo->curr.period = period;
2561                 tinfo->curr.offset = offset;
2562                 tinfo->curr.ppr_options = ppr_options;
2563
2564                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2565                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2566                 if (bootverbose) {
2567                         if (offset != 0) {
2568                                 printk("%s: target %d synchronous at %sMHz%s, "
2569                                        "offset = 0x%x\n", ahc_name(ahc),
2570                                        devinfo->target, syncrate->rate,
2571                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
2572                                        ? " DT" : "", offset);
2573                         } else {
2574                                 printk("%s: target %d using "
2575                                        "asynchronous transfers\n",
2576                                        ahc_name(ahc), devinfo->target);
2577                         }
2578                 }
2579         }
2580
2581         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2582                                                 tinfo, AHC_NEG_TO_GOAL);
2583
2584         if (update_needed)
2585                 ahc_update_pending_scbs(ahc);
2586 }
2587
2588 /*
2589  * Update the user/goal/curr tables of wide negotiation
2590  * parameters as well as, in the case of a current or active update,
2591  * any data structures on the host controller.  In the case of an
2592  * active update, the specified target is currently talking to us on
2593  * the bus, so the transfer parameter update must take effect
2594  * immediately.
2595  */
2596 void
2597 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2598               u_int width, u_int type, int paused)
2599 {
2600         struct  ahc_initiator_tinfo *tinfo;
2601         struct  ahc_tmode_tstate *tstate;
2602         u_int   oldwidth;
2603         int     active;
2604         int     update_needed;
2605
2606         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2607         update_needed = 0;
2608         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2609                                     devinfo->target, &tstate);
2610
2611         if ((type & AHC_TRANS_USER) != 0)
2612                 tinfo->user.width = width;
2613
2614         if ((type & AHC_TRANS_GOAL) != 0)
2615                 tinfo->goal.width = width;
2616
2617         oldwidth = tinfo->curr.width;
2618         if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2619                 u_int   scsirate;
2620
2621                 update_needed++;
2622                 scsirate =  tinfo->scsirate;
2623                 scsirate &= ~WIDEXFER;
2624                 if (width == MSG_EXT_WDTR_BUS_16_BIT)
2625                         scsirate |= WIDEXFER;
2626
2627                 tinfo->scsirate = scsirate;
2628
2629                 if (active)
2630                         ahc_outb(ahc, SCSIRATE, scsirate);
2631
2632                 tinfo->curr.width = width;
2633
2634                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2635                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2636                 if (bootverbose) {
2637                         printk("%s: target %d using %dbit transfers\n",
2638                                ahc_name(ahc), devinfo->target,
2639                                8 * (0x01 << width));
2640                 }
2641         }
2642
2643         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2644                                                 tinfo, AHC_NEG_TO_GOAL);
2645         if (update_needed)
2646                 ahc_update_pending_scbs(ahc);
2647 }
2648
2649 /*
2650  * Update the current state of tagged queuing for a given target.
2651  */
2652 static void
2653 ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2654              struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2655 {
2656         struct scsi_device *sdev = cmd->device;
2657
2658         ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2659         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2660                        devinfo->lun, AC_TRANSFER_NEG);
2661 }
2662
2663 /*
2664  * When the transfer settings for a connection change, update any
2665  * in-transit SCBs to contain the new data so the hardware will
2666  * be set correctly during future (re)selections.
2667  */
2668 static void
2669 ahc_update_pending_scbs(struct ahc_softc *ahc)
2670 {
2671         struct  scb *pending_scb;
2672         int     pending_scb_count;
2673         int     i;
2674         int     paused;
2675         u_int   saved_scbptr;
2676
2677         /*
2678          * Traverse the pending SCB list and ensure that all of the
2679          * SCBs there have the proper settings.
2680          */
2681         pending_scb_count = 0;
2682         LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2683                 struct ahc_devinfo devinfo;
2684                 struct hardware_scb *pending_hscb;
2685                 struct ahc_initiator_tinfo *tinfo;
2686                 struct ahc_tmode_tstate *tstate;
2687
2688                 ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2689                 tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2690                                             devinfo.our_scsiid,
2691                                             devinfo.target, &tstate);
2692                 pending_hscb = pending_scb->hscb;
2693                 pending_hscb->control &= ~ULTRAENB;
2694                 if ((tstate->ultraenb & devinfo.target_mask) != 0)
2695                         pending_hscb->control |= ULTRAENB;
2696                 pending_hscb->scsirate = tinfo->scsirate;
2697                 pending_hscb->scsioffset = tinfo->curr.offset;
2698                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2699                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2700                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2701                         pending_hscb->control &= ~MK_MESSAGE;
2702                 }
2703                 ahc_sync_scb(ahc, pending_scb,
2704                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2705                 pending_scb_count++;
2706         }
2707
2708         if (pending_scb_count == 0)
2709                 return;
2710
2711         if (ahc_is_paused(ahc)) {
2712                 paused = 1;
2713         } else {
2714                 paused = 0;
2715                 ahc_pause(ahc);
2716         }
2717
2718         saved_scbptr = ahc_inb(ahc, SCBPTR);
2719         /* Ensure that the hscbs down on the card match the new information */
2720         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2721                 struct  hardware_scb *pending_hscb;
2722                 u_int   control;
2723                 u_int   scb_tag;
2724
2725                 ahc_outb(ahc, SCBPTR, i);
2726                 scb_tag = ahc_inb(ahc, SCB_TAG);
2727                 pending_scb = ahc_lookup_scb(ahc, scb_tag);
2728                 if (pending_scb == NULL)
2729                         continue;
2730
2731                 pending_hscb = pending_scb->hscb;
2732                 control = ahc_inb(ahc, SCB_CONTROL);
2733                 control &= ~(ULTRAENB|MK_MESSAGE);
2734                 control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2735                 ahc_outb(ahc, SCB_CONTROL, control);
2736                 ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2737                 ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2738         }
2739         ahc_outb(ahc, SCBPTR, saved_scbptr);
2740
2741         if (paused == 0)
2742                 ahc_unpause(ahc);
2743 }
2744
2745 /**************************** Pathing Information *****************************/
2746 static void
2747 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2748 {
2749         u_int   saved_scsiid;
2750         role_t  role;
2751         int     our_id;
2752
2753         if (ahc_inb(ahc, SSTAT0) & TARGET)
2754                 role = ROLE_TARGET;
2755         else
2756                 role = ROLE_INITIATOR;
2757
2758         if (role == ROLE_TARGET
2759          && (ahc->features & AHC_MULTI_TID) != 0
2760          && (ahc_inb(ahc, SEQ_FLAGS)
2761            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2762                 /* We were selected, so pull our id from TARGIDIN */
2763                 our_id = ahc_inb(ahc, TARGIDIN) & OID;
2764         } else if ((ahc->features & AHC_ULTRA2) != 0)
2765                 our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2766         else
2767                 our_id = ahc_inb(ahc, SCSIID) & OID;
2768
2769         saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2770         ahc_compile_devinfo(devinfo,
2771                             our_id,
2772                             SCSIID_TARGET(ahc, saved_scsiid),
2773                             ahc_inb(ahc, SAVED_LUN),
2774                             SCSIID_CHANNEL(ahc, saved_scsiid),
2775                             role);
2776 }
2777
2778 static const struct ahc_phase_table_entry*
2779 ahc_lookup_phase_entry(int phase)
2780 {
2781         const struct ahc_phase_table_entry *entry;
2782         const struct ahc_phase_table_entry *last_entry;
2783
2784         /*
2785          * num_phases doesn't include the default entry which
2786          * will be returned if the phase doesn't match.
2787          */
2788         last_entry = &ahc_phase_table[num_phases];
2789         for (entry = ahc_phase_table; entry < last_entry; entry++) {
2790                 if (phase == entry->phase)
2791                         break;
2792         }
2793         return (entry);
2794 }
2795
2796 void
2797 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2798                     u_int lun, char channel, role_t role)
2799 {
2800         devinfo->our_scsiid = our_id;
2801         devinfo->target = target;
2802         devinfo->lun = lun;
2803         devinfo->target_offset = target;
2804         devinfo->channel = channel;
2805         devinfo->role = role;
2806         if (channel == 'B')
2807                 devinfo->target_offset += 8;
2808         devinfo->target_mask = (0x01 << devinfo->target_offset);
2809 }
2810
2811 void
2812 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2813 {
2814         printk("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2815                devinfo->target, devinfo->lun);
2816 }
2817
2818 static void
2819 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2820                 struct scb *scb)
2821 {
2822         role_t  role;
2823         int     our_id;
2824
2825         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2826         role = ROLE_INITIATOR;
2827         if ((scb->flags & SCB_TARGET_SCB) != 0)
2828                 role = ROLE_TARGET;
2829         ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2830                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2831 }
2832
2833
2834 /************************ Message Phase Processing ****************************/
2835 static void
2836 ahc_assert_atn(struct ahc_softc *ahc)
2837 {
2838         u_int scsisigo;
2839
2840         scsisigo = ATNO;
2841         if ((ahc->features & AHC_DT) == 0)
2842                 scsisigo |= ahc_inb(ahc, SCSISIGI);
2843         ahc_outb(ahc, SCSISIGO, scsisigo);
2844 }
2845
2846 /*
2847  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2848  * or enters the initial message out phase, we are interrupted.  Fill our
2849  * outgoing message buffer with the appropriate message and beging handing
2850  * the message phase(s) manually.
2851  */
2852 static void
2853 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2854                            struct scb *scb)
2855 {
2856         /*
2857          * To facilitate adding multiple messages together,
2858          * each routine should increment the index and len
2859          * variables instead of setting them explicitly.
2860          */
2861         ahc->msgout_index = 0;
2862         ahc->msgout_len = 0;
2863
2864         if ((scb->flags & SCB_DEVICE_RESET) == 0
2865          && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2866                 u_int identify_msg;
2867
2868                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2869                 if ((scb->hscb->control & DISCENB) != 0)
2870                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2871                 ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2872                 ahc->msgout_len++;
2873
2874                 if ((scb->hscb->control & TAG_ENB) != 0) {
2875                         ahc->msgout_buf[ahc->msgout_index++] =
2876                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2877                         ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2878                         ahc->msgout_len += 2;
2879                 }
2880         }
2881
2882         if (scb->flags & SCB_DEVICE_RESET) {
2883                 ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2884                 ahc->msgout_len++;
2885                 ahc_print_path(ahc, scb);
2886                 printk("Bus Device Reset Message Sent\n");
2887                 /*
2888                  * Clear our selection hardware in advance of
2889                  * the busfree.  We may have an entry in the waiting
2890                  * Q for this target, and we don't want to go about
2891                  * selecting while we handle the busfree and blow it
2892                  * away.
2893                  */
2894                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2895         } else if ((scb->flags & SCB_ABORT) != 0) {
2896                 if ((scb->hscb->control & TAG_ENB) != 0)
2897                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2898                 else
2899                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2900                 ahc->msgout_len++;
2901                 ahc_print_path(ahc, scb);
2902                 printk("Abort%s Message Sent\n",
2903                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2904                 /*
2905                  * Clear our selection hardware in advance of
2906                  * the busfree.  We may have an entry in the waiting
2907                  * Q for this target, and we don't want to go about
2908                  * selecting while we handle the busfree and blow it
2909                  * away.
2910                  */
2911                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2912         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2913                 ahc_build_transfer_msg(ahc, devinfo);
2914         } else {
2915                 printk("ahc_intr: AWAITING_MSG for an SCB that "
2916                        "does not have a waiting message\n");
2917                 printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2918                        devinfo->target_mask);
2919                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2920                       "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2921                       ahc_inb(ahc, MSG_OUT), scb->flags);
2922         }
2923
2924         /*
2925          * Clear the MK_MESSAGE flag from the SCB so we aren't
2926          * asked to send this message again.
2927          */
2928         ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2929         scb->hscb->control &= ~MK_MESSAGE;
2930         ahc->msgout_index = 0;
2931         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2932 }
2933
2934 /*
2935  * Build an appropriate transfer negotiation message for the
2936  * currently active target.
2937  */
2938 static void
2939 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2940 {
2941         /*
2942          * We need to initiate transfer negotiations.
2943          * If our current and goal settings are identical,
2944          * we want to renegotiate due to a check condition.
2945          */
2946         struct  ahc_initiator_tinfo *tinfo;
2947         struct  ahc_tmode_tstate *tstate;
2948         const struct ahc_syncrate *rate;
2949         int     dowide;
2950         int     dosync;
2951         int     doppr;
2952         u_int   period;
2953         u_int   ppr_options;
2954         u_int   offset;
2955
2956         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2957                                     devinfo->target, &tstate);
2958         /*
2959          * Filter our period based on the current connection.
2960          * If we can't perform DT transfers on this segment (not in LVD
2961          * mode for instance), then our decision to issue a PPR message
2962          * may change.
2963          */
2964         period = tinfo->goal.period;
2965         offset = tinfo->goal.offset;
2966         ppr_options = tinfo->goal.ppr_options;
2967         /* Target initiated PPR is not allowed in the SCSI spec */
2968         if (devinfo->role == ROLE_TARGET)
2969                 ppr_options = 0;
2970         rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2971                                        &ppr_options, devinfo->role);
2972         dowide = tinfo->curr.width != tinfo->goal.width;
2973         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2974         /*
2975          * Only use PPR if we have options that need it, even if the device
2976          * claims to support it.  There might be an expander in the way
2977          * that doesn't.
2978          */
2979         doppr = ppr_options != 0;
2980
2981         if (!dowide && !dosync && !doppr) {
2982                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2983                 dosync = tinfo->goal.offset != 0;
2984         }
2985
2986         if (!dowide && !dosync && !doppr) {
2987                 /*
2988                  * Force async with a WDTR message if we have a wide bus,
2989                  * or just issue an SDTR with a 0 offset.
2990                  */
2991                 if ((ahc->features & AHC_WIDE) != 0)
2992                         dowide = 1;
2993                 else
2994                         dosync = 1;
2995
2996                 if (bootverbose) {
2997                         ahc_print_devinfo(ahc, devinfo);
2998                         printk("Ensuring async\n");
2999                 }
3000         }
3001
3002         /* Target initiated PPR is not allowed in the SCSI spec */
3003         if (devinfo->role == ROLE_TARGET)
3004                 doppr = 0;
3005
3006         /*
3007          * Both the PPR message and SDTR message require the
3008          * goal syncrate to be limited to what the target device
3009          * is capable of handling (based on whether an LVD->SE
3010          * expander is on the bus), so combine these two cases.
3011          * Regardless, guarantee that if we are using WDTR and SDTR
3012          * messages that WDTR comes first.
3013          */
3014         if (doppr || (dosync && !dowide)) {
3015
3016                 offset = tinfo->goal.offset;
3017                 ahc_validate_offset(ahc, tinfo, rate, &offset,
3018                                     doppr ? tinfo->goal.width
3019                                           : tinfo->curr.width,
3020                                     devinfo->role);
3021                 if (doppr) {
3022                         ahc_construct_ppr(ahc, devinfo, period, offset,
3023                                           tinfo->goal.width, ppr_options);
3024                 } else {
3025                         ahc_construct_sdtr(ahc, devinfo, period, offset);
3026                 }
3027         } else {
3028                 ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
3029         }
3030 }
3031
3032 /*
3033  * Build a synchronous negotiation message in our message
3034  * buffer based on the input parameters.
3035  */
3036 static void
3037 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3038                    u_int period, u_int offset)
3039 {
3040         if (offset == 0)
3041                 period = AHC_ASYNC_XFER_PERIOD;
3042         ahc->msgout_index += spi_populate_sync_msg(
3043                         ahc->msgout_buf + ahc->msgout_index, period, offset);
3044         ahc->msgout_len += 5;
3045         if (bootverbose) {
3046                 printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3047                        ahc_name(ahc), devinfo->channel, devinfo->target,
3048                        devinfo->lun, period, offset);
3049         }
3050 }
3051
3052 /*
3053  * Build a wide negotiation message in our message
3054  * buffer based on the input parameters.
3055  */
3056 static void
3057 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3058                    u_int bus_width)
3059 {
3060         ahc->msgout_index += spi_populate_width_msg(
3061                         ahc->msgout_buf + ahc->msgout_index, bus_width);
3062         ahc->msgout_len += 4;
3063         if (bootverbose) {
3064                 printk("(%s:%c:%d:%d): Sending WDTR %x\n",
3065                        ahc_name(ahc), devinfo->channel, devinfo->target,
3066                        devinfo->lun, bus_width);
3067         }
3068 }
3069
3070 /*
3071  * Build a parallel protocol request message in our message
3072  * buffer based on the input parameters.
3073  */
3074 static void
3075 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3076                   u_int period, u_int offset, u_int bus_width,
3077                   u_int ppr_options)
3078 {
3079         if (offset == 0)
3080                 period = AHC_ASYNC_XFER_PERIOD;
3081         ahc->msgout_index += spi_populate_ppr_msg(
3082                         ahc->msgout_buf + ahc->msgout_index, period, offset,
3083                         bus_width, ppr_options);
3084         ahc->msgout_len += 8;
3085         if (bootverbose) {
3086                 printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3087                        "offset %x, ppr_options %x\n", ahc_name(ahc),
3088                        devinfo->channel, devinfo->target, devinfo->lun,
3089                        bus_width, period, offset, ppr_options);
3090         }
3091 }
3092
3093 /*
3094  * Clear any active message state.
3095  */
3096 static void
3097 ahc_clear_msg_state(struct ahc_softc *ahc)
3098 {
3099         ahc->msgout_len = 0;
3100         ahc->msgin_index = 0;
3101         ahc->msg_type = MSG_TYPE_NONE;
3102         if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
3103                 /*
3104                  * The target didn't care to respond to our
3105                  * message request, so clear ATN.
3106                  */
3107                 ahc_outb(ahc, CLRSINT1, CLRATNO);
3108         }
3109         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
3110         ahc_outb(ahc, SEQ_FLAGS2,
3111                  ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3112 }
3113
3114 static void
3115 ahc_handle_proto_violation(struct ahc_softc *ahc)
3116 {
3117         struct  ahc_devinfo devinfo;
3118         struct  scb *scb;
3119         u_int   scbid;
3120         u_int   seq_flags;
3121         u_int   curphase;
3122         u_int   lastphase;
3123         int     found;
3124
3125         ahc_fetch_devinfo(ahc, &devinfo);
3126         scbid = ahc_inb(ahc, SCB_TAG);
3127         scb = ahc_lookup_scb(ahc, scbid);
3128         seq_flags = ahc_inb(ahc, SEQ_FLAGS);
3129         curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3130         lastphase = ahc_inb(ahc, LASTPHASE);
3131         if ((seq_flags & NOT_IDENTIFIED) != 0) {
3132
3133                 /*
3134                  * The reconnecting target either did not send an
3135                  * identify message, or did, but we didn't find an SCB
3136                  * to match.
3137                  */
3138                 ahc_print_devinfo(ahc, &devinfo);
3139                 printk("Target did not send an IDENTIFY message. "
3140                        "LASTPHASE = 0x%x.\n", lastphase);
3141                 scb = NULL;
3142         } else if (scb == NULL) {
3143                 /*
3144                  * We don't seem to have an SCB active for this
3145                  * transaction.  Print an error and reset the bus.
3146                  */
3147                 ahc_print_devinfo(ahc, &devinfo);
3148                 printk("No SCB found during protocol violation\n");
3149                 goto proto_violation_reset;
3150         } else {
3151                 ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3152                 if ((seq_flags & NO_CDB_SENT) != 0) {
3153                         ahc_print_path(ahc, scb);
3154                         printk("No or incomplete CDB sent to device.\n");
3155                 } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
3156                         /*
3157                          * The target never bothered to provide status to
3158                          * us prior to completing the command.  Since we don't
3159                          * know the disposition of this command, we must attempt
3160                          * to abort it.  Assert ATN and prepare to send an abort
3161                          * message.
3162                          */
3163                         ahc_print_path(ahc, scb);
3164                         printk("Completed command without status.\n");
3165                 } else {
3166                         ahc_print_path(ahc, scb);
3167                         printk("Unknown protocol violation.\n");
3168                         ahc_dump_card_state(ahc);
3169                 }
3170         }
3171         if ((lastphase & ~P_DATAIN_DT) == 0
3172          || lastphase == P_COMMAND) {
3173 proto_violation_reset:
3174                 /*
3175                  * Target either went directly to data/command
3176                  * phase or didn't respond to our ATN.
3177                  * The only safe thing to do is to blow
3178                  * it away with a bus reset.
3179                  */
3180                 found = ahc_reset_channel(ahc, 'A', TRUE);
3181                 printk("%s: Issued Channel %c Bus Reset. "
3182                        "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
3183         } else {
3184                 /*
3185                  * Leave the selection hardware off in case
3186                  * this abort attempt will affect yet to
3187                  * be sent commands.
3188                  */
3189                 ahc_outb(ahc, SCSISEQ,
3190                          ahc_inb(ahc, SCSISEQ) & ~ENSELO);
3191                 ahc_assert_atn(ahc);
3192                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
3193                 if (scb == NULL) {
3194                         ahc_print_devinfo(ahc, &devinfo);
3195                         ahc->msgout_buf[0] = MSG_ABORT_TASK;
3196                         ahc->msgout_len = 1;
3197                         ahc->msgout_index = 0;
3198                         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3199                 } else {
3200                         ahc_print_path(ahc, scb);
3201                         scb->flags |= SCB_ABORT;
3202                 }
3203                 printk("Protocol violation %s.  Attempting to abort.\n",
3204                        ahc_lookup_phase_entry(curphase)->phasemsg);
3205         }
3206 }
3207
3208 /*
3209  * Manual message loop handler.
3210  */
3211 static void
3212 ahc_handle_message_phase(struct ahc_softc *ahc)
3213 {
3214         struct  ahc_devinfo devinfo;
3215         u_int   bus_phase;
3216         int     end_session;
3217
3218         ahc_fetch_devinfo(ahc, &devinfo);
3219         end_session = FALSE;
3220         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3221
3222 reswitch:
3223         switch (ahc->msg_type) {
3224         case MSG_TYPE_INITIATOR_MSGOUT:
3225         {
3226                 int lastbyte;
3227                 int phasemis;
3228                 int msgdone;
3229
3230                 if (ahc->msgout_len == 0)
3231                         panic("HOST_MSG_LOOP interrupt with no active message");
3232
3233 #ifdef AHC_DEBUG
3234                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3235                         ahc_print_devinfo(ahc, &devinfo);
3236                         printk("INITIATOR_MSG_OUT");
3237                 }
3238 #endif
3239                 phasemis = bus_phase != P_MESGOUT;
3240                 if (phasemis) {
3241 #ifdef AHC_DEBUG
3242                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3243                                 printk(" PHASEMIS %s\n",
3244                                        ahc_lookup_phase_entry(bus_phase)
3245                                                              ->phasemsg);
3246                         }
3247 #endif
3248                         if (bus_phase == P_MESGIN) {
3249                                 /*
3250                                  * Change gears and see if
3251                                  * this messages is of interest to
3252                                  * us or should be passed back to
3253                                  * the sequencer.
3254                                  */
3255                                 ahc_outb(ahc, CLRSINT1, CLRATNO);
3256                                 ahc->send_msg_perror = FALSE;
3257                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3258                                 ahc->msgin_index = 0;
3259                                 goto reswitch;
3260                         }
3261                         end_session = TRUE;
3262                         break;
3263                 }
3264
3265                 if (ahc->send_msg_perror) {
3266                         ahc_outb(ahc, CLRSINT1, CLRATNO);
3267                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3268 #ifdef AHC_DEBUG
3269                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3270                                 printk(" byte 0x%x\n", ahc->send_msg_perror);
3271 #endif
3272                         ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
3273                         break;
3274                 }
3275
3276                 msgdone = ahc->msgout_index == ahc->msgout_len;
3277                 if (msgdone) {
3278                         /*
3279                          * The target has requested a retry.
3280                          * Re-assert ATN, reset our message index to
3281                          * 0, and try again.
3282                          */
3283                         ahc->msgout_index = 0;
3284                         ahc_assert_atn(ahc);
3285                 }
3286
3287                 lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
3288                 if (lastbyte) {
3289                         /* Last byte is signified by dropping ATN */
3290                         ahc_outb(ahc, CLRSINT1, CLRATNO);
3291                 }
3292
3293                 /*
3294                  * Clear our interrupt status and present
3295                  * the next byte on the bus.
3296                  */
3297                 ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3298 #ifdef AHC_DEBUG
3299                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3300                         printk(" byte 0x%x\n",
3301                                ahc->msgout_buf[ahc->msgout_index]);
3302 #endif
3303                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3304                 break;
3305         }
3306         case MSG_TYPE_INITIATOR_MSGIN:
3307         {
3308                 int phasemis;
3309                 int message_done;
3310
3311 #ifdef AHC_DEBUG
3312                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3313                         ahc_print_devinfo(ahc, &devinfo);
3314                         printk("INITIATOR_MSG_IN");
3315                 }
3316 #endif
3317                 phasemis = bus_phase != P_MESGIN;
3318                 if (phasemis) {
3319 #ifdef AHC_DEBUG
3320                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3321                                 printk(" PHASEMIS %s\n",
3322                                        ahc_lookup_phase_entry(bus_phase)
3323                                                              ->phasemsg);
3324                         }
3325 #endif
3326                         ahc->msgin_index = 0;
3327                         if (bus_phase == P_MESGOUT
3328                          && (ahc->send_msg_perror == TRUE
3329                           || (ahc->msgout_len != 0
3330                            && ahc->msgout_index == 0))) {
3331                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3332                                 goto reswitch;
3333                         }
3334                         end_session = TRUE;
3335                         break;
3336                 }
3337
3338                 /* Pull the byte in without acking it */
3339                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
3340 #ifdef AHC_DEBUG
3341                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3342                         printk(" byte 0x%x\n",
3343                                ahc->msgin_buf[ahc->msgin_index]);
3344 #endif
3345
3346                 message_done = ahc_parse_msg(ahc, &devinfo);
3347
3348                 if (message_done) {
3349                         /*
3350                          * Clear our incoming message buffer in case there
3351                          * is another message following this one.
3352                          */
3353                         ahc->msgin_index = 0;
3354
3355                         /*
3356                          * If this message illicited a response,
3357                          * assert ATN so the target takes us to the
3358                          * message out phase.
3359                          */
3360                         if (ahc->msgout_len != 0) {
3361 #ifdef AHC_DEBUG
3362                                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3363                                         ahc_print_devinfo(ahc, &devinfo);
3364                                         printk("Asserting ATN for response\n");
3365                                 }
3366 #endif
3367                                 ahc_assert_atn(ahc);
3368                         }
3369                 } else
3370                         ahc->msgin_index++;
3371
3372                 if (message_done == MSGLOOP_TERMINATED) {
3373                         end_session = TRUE;
3374                 } else {
3375                         /* Ack the byte */
3376                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3377                         ahc_inb(ahc, SCSIDATL);
3378                 }
3379                 break;
3380         }
3381         case MSG_TYPE_TARGET_MSGIN:
3382         {
3383                 int msgdone;
3384                 int msgout_request;
3385
3386                 if (ahc->msgout_len == 0)
3387                         panic("Target MSGIN with no active message");
3388
3389                 /*
3390                  * If we interrupted a mesgout session, the initiator
3391                  * will not know this until our first REQ.  So, we
3392                  * only honor mesgout requests after we've sent our
3393                  * first byte.
3394                  */
3395                 if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
3396                  && ahc->msgout_index > 0)
3397                         msgout_request = TRUE;
3398                 else
3399                         msgout_request = FALSE;
3400
3401                 if (msgout_request) {
3402
3403                         /*
3404                          * Change gears and see if
3405                          * this messages is of interest to
3406                          * us or should be passed back to
3407                          * the sequencer.
3408                          */
3409                         ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
3410                         ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
3411                         ahc->msgin_index = 0;
3412                         /* Dummy read to REQ for first byte */
3413                         ahc_inb(ahc, SCSIDATL);
3414                         ahc_outb(ahc, SXFRCTL0,
3415                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3416                         break;
3417                 }
3418
3419                 msgdone = ahc->msgout_index == ahc->msgout_len;
3420                 if (msgdone) {
3421                         ahc_outb(ahc, SXFRCTL0,
3422                                  ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3423                         end_session = TRUE;
3424                         break;
3425                 }
3426
3427                 /*
3428                  * Present the next byte on the bus.
3429                  */
3430                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3431                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3432                 break;
3433         }
3434         case MSG_TYPE_TARGET_MSGOUT:
3435         {
3436                 int lastbyte;
3437                 int msgdone;
3438
3439                 /*
3440                  * The initiator signals that this is
3441                  * the last byte by dropping ATN.
3442                  */
3443                 lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
3444
3445                 /*
3446                  * Read the latched byte, but turn off SPIOEN first
3447                  * so that we don't inadvertently cause a REQ for the
3448                  * next byte.
3449                  */
3450                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3451                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
3452                 msgdone = ahc_parse_msg(ahc, &devinfo);
3453                 if (msgdone == MSGLOOP_TERMINATED) {
3454                         /*
3455                          * The message is *really* done in that it caused
3456                          * us to go to bus free.  The sequencer has already
3457                          * been reset at this point, so pull the ejection
3458                          * handle.
3459                          */
3460                         return;
3461                 }
3462
3463                 ahc->msgin_index++;
3464
3465                 /*
3466                  * XXX Read spec about initiator dropping ATN too soon
3467                  *     and use msgdone to detect it.
3468                  */
3469                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
3470                         ahc->msgin_index = 0;
3471
3472                         /*
3473                          * If this message illicited a response, transition
3474                          * to the Message in phase and send it.
3475                          */
3476                         if (ahc->msgout_len != 0) {
3477                                 ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
3478                                 ahc_outb(ahc, SXFRCTL0,
3479                                          ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3480                                 ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3481                                 ahc->msgin_index = 0;
3482                                 break;
3483                         }
3484                 }
3485
3486                 if (lastbyte)
3487                         end_session = TRUE;
3488                 else {
3489                         /* Ask for the next byte. */
3490                         ahc_outb(ahc, SXFRCTL0,
3491                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3492                 }
3493
3494                 break;
3495         }
3496         default:
3497                 panic("Unknown REQINIT message type");
3498         }
3499
3500         if (end_session) {
3501                 ahc_clear_msg_state(ahc);
3502                 ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
3503         } else
3504                 ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
3505 }
3506
3507 /*
3508  * See if we sent a particular extended message to the target.
3509  * If "full" is true, return true only if the target saw the full
3510  * message.  If "full" is false, return true if the target saw at
3511  * least the first byte of the message.
3512  */
3513 static int
3514 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
3515 {
3516         int found;
3517         u_int index;
3518
3519         found = FALSE;
3520         index = 0;
3521
3522         while (index < ahc->msgout_len) {
3523                 if (ahc->msgout_buf[index] == MSG_EXTENDED) {
3524                         u_int end_index;
3525
3526                         end_index = index + 1 + ahc->msgout_buf[index + 1];
3527                         if (ahc->msgout_buf[index+2] == msgval
3528                          && type == AHCMSG_EXT) {
3529
3530                                 if (full) {
3531                                         if (ahc->msgout_index > end_index)
3532                                                 found = TRUE;
3533                                 } else if (ahc->msgout_index > index)
3534                                         found = TRUE;
3535                         }
3536                         index = end_index;
3537                 } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
3538                         && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
3539
3540                         /* Skip tag type and tag id or residue param*/
3541                         index += 2;
3542                 } else {
3543                         /* Single byte message */
3544                         if (type == AHCMSG_1B
3545                          && ahc->msgout_buf[index] == msgval
3546                          && ahc->msgout_index > index)
3547                                 found = TRUE;
3548                         index++;
3549                 }
3550
3551                 if (found)
3552                         break;
3553         }
3554         return (found);
3555 }
3556
3557 /*
3558  * Wait for a complete incoming message, parse it, and respond accordingly.
3559  */
3560 static int
3561 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3562 {
3563         struct  ahc_initiator_tinfo *tinfo;
3564         struct  ahc_tmode_tstate *tstate;
3565         int     reject;
3566         int     done;
3567         int     response;
3568         u_int   targ_scsirate;
3569
3570         done = MSGLOOP_IN_PROG;
3571         response = FALSE;
3572         reject = FALSE;
3573         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3574                                     devinfo->target, &tstate);
3575         targ_scsirate = tinfo->scsirate;
3576
3577         /*
3578          * Parse as much of the message as is available,
3579          * rejecting it if we don't support it.  When
3580          * the entire message is available and has been
3581          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3582          * that we have parsed an entire message.
3583          *
3584          * In the case of extended messages, we accept the length
3585          * byte outright and perform more checking once we know the
3586          * extended message type.
3587          */
3588         switch (ahc->msgin_buf[0]) {
3589         case MSG_DISCONNECT:
3590         case MSG_SAVEDATAPOINTER:
3591         case MSG_CMDCOMPLETE:
3592         case MSG_RESTOREPOINTERS:
3593         case MSG_IGN_WIDE_RESIDUE:
3594                 /*
3595                  * End our message loop as these are messages
3596                  * the sequencer handles on its own.
3597                  */
3598                 done = MSGLOOP_TERMINATED;
3599                 break;
3600         case MSG_MESSAGE_REJECT:
3601                 response = ahc_handle_msg_reject(ahc, devinfo);
3602                 fallthrough;
3603         case MSG_NOOP:
3604                 done = MSGLOOP_MSGCOMPLETE;
3605                 break;
3606         case MSG_EXTENDED:
3607         {
3608                 /* Wait for enough of the message to begin validation */
3609                 if (ahc->msgin_index < 2)
3610                         break;
3611                 switch (ahc->msgin_buf[2]) {
3612                 case MSG_EXT_SDTR:
3613                 {
3614                         const struct ahc_syncrate *syncrate;
3615                         u_int    period;
3616                         u_int    ppr_options;
3617                         u_int    offset;
3618                         u_int    saved_offset;
3619
3620                         if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3621                                 reject = TRUE;
3622                                 break;
3623                         }
3624
3625                         /*
3626                          * Wait until we have both args before validating
3627                          * and acting on this message.
3628                          *
3629                          * Add one to MSG_EXT_SDTR_LEN to account for
3630                          * the extended message preamble.
3631                          */
3632                         if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3633                                 break;
3634
3635                         period = ahc->msgin_buf[3];
3636                         ppr_options = 0;
3637                         saved_offset = offset = ahc->msgin_buf[4];
3638                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3639                                                            &ppr_options,
3640                                                            devinfo->role);
3641                         ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3642                                             targ_scsirate & WIDEXFER,
3643                                             devinfo->role);
3644                         if (bootverbose) {
3645                                 printk("(%s:%c:%d:%d): Received "
3646                                        "SDTR period %x, offset %x\n\t"
3647                                        "Filtered to period %x, offset %x\n",
3648                                        ahc_name(ahc), devinfo->channel,
3649                                        devinfo->target, devinfo->lun,
3650                                        ahc->msgin_buf[3], saved_offset,
3651                                        period, offset);
3652                         }
3653                         ahc_set_syncrate(ahc, devinfo,
3654                                          syncrate, period,
3655                                          offset, ppr_options,
3656                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3657                                          /*paused*/TRUE);
3658
3659                         /*
3660                          * See if we initiated Sync Negotiation
3661                          * and didn't have to fall down to async
3662                          * transfers.
3663                          */
3664                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3665                                 /* We started it */
3666                                 if (saved_offset != offset) {
3667                                         /* Went too low - force async */
3668                                         reject = TRUE;
3669                                 }
3670                         } else {
3671                                 /*
3672                                  * Send our own SDTR in reply
3673                                  */
3674                                 if (bootverbose
3675                                  && devinfo->role == ROLE_INITIATOR) {
3676                                         printk("(%s:%c:%d:%d): Target "
3677                                                "Initiated SDTR\n",
3678                                                ahc_name(ahc), devinfo->channel,
3679                                                devinfo->target, devinfo->lun);
3680                                 }
3681                                 ahc->msgout_index = 0;
3682                                 ahc->msgout_len = 0;
3683                                 ahc_construct_sdtr(ahc, devinfo,
3684                                                    period, offset);
3685                                 ahc->msgout_index = 0;
3686                                 response = TRUE;
3687                         }
3688                         done = MSGLOOP_MSGCOMPLETE;
3689                         break;
3690                 }
3691                 case MSG_EXT_WDTR:
3692                 {
3693                         u_int bus_width;
3694                         u_int saved_width;
3695                         u_int sending_reply;
3696
3697                         sending_reply = FALSE;
3698                         if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3699                                 reject = TRUE;
3700                                 break;
3701                         }
3702
3703                         /*
3704                          * Wait until we have our arg before validating
3705                          * and acting on this message.
3706                          *
3707                          * Add one to MSG_EXT_WDTR_LEN to account for
3708                          * the extended message preamble.
3709                          */
3710                         if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3711                                 break;
3712
3713                         bus_width = ahc->msgin_buf[3];
3714                         saved_width = bus_width;
3715                         ahc_validate_width(ahc, tinfo, &bus_width,
3716                                            devinfo->role);
3717                         if (bootverbose) {
3718                                 printk("(%s:%c:%d:%d): Received WDTR "
3719                                        "%x filtered to %x\n",
3720                                        ahc_name(ahc), devinfo->channel,
3721                                        devinfo->target, devinfo->lun,
3722                                        saved_width, bus_width);
3723                         }
3724
3725                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3726                                 /*
3727                                  * Don't send a WDTR back to the
3728                                  * target, since we asked first.
3729                                  * If the width went higher than our
3730                                  * request, reject it.
3731                                  */
3732                                 if (saved_width > bus_width) {
3733                                         reject = TRUE;
3734                                         printk("(%s:%c:%d:%d): requested %dBit "
3735                                                "transfers.  Rejecting...\n",
3736                                                ahc_name(ahc), devinfo->channel,
3737                                                devinfo->target, devinfo->lun,
3738                                                8 * (0x01 << bus_width));
3739                                         bus_width = 0;
3740                                 }
3741                         } else {
3742                                 /*
3743                                  * Send our own WDTR in reply
3744                                  */
3745                                 if (bootverbose
3746                                  && devinfo->role == ROLE_INITIATOR) {
3747                                         printk("(%s:%c:%d:%d): Target "
3748                                                "Initiated WDTR\n",
3749                                                ahc_name(ahc), devinfo->channel,
3750                                                devinfo->target, devinfo->lun);
3751                                 }
3752                                 ahc->msgout_index = 0;
3753                                 ahc->msgout_len = 0;
3754                                 ahc_construct_wdtr(ahc, devinfo, bus_width);
3755                                 ahc->msgout_index = 0;
3756                                 response = TRUE;
3757                                 sending_reply = TRUE;
3758                         }
3759                         /*
3760                          * After a wide message, we are async, but
3761                          * some devices don't seem to honor this portion
3762                          * of the spec.  Force a renegotiation of the
3763                          * sync component of our transfer agreement even
3764                          * if our goal is async.  By updating our width
3765                          * after forcing the negotiation, we avoid
3766                          * renegotiating for width.
3767                          */
3768                         ahc_update_neg_request(ahc, devinfo, tstate,
3769                                                tinfo, AHC_NEG_ALWAYS);
3770                         ahc_set_width(ahc, devinfo, bus_width,
3771                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3772                                       /*paused*/TRUE);
3773                         if (sending_reply == FALSE && reject == FALSE) {
3774
3775                                 /*
3776                                  * We will always have an SDTR to send.
3777                                  */
3778                                 ahc->msgout_index = 0;
3779                                 ahc->msgout_len = 0;
3780                                 ahc_build_transfer_msg(ahc, devinfo);
3781                                 ahc->msgout_index = 0;
3782                                 response = TRUE;
3783                         }
3784                         done = MSGLOOP_MSGCOMPLETE;
3785                         break;
3786                 }
3787                 case MSG_EXT_PPR:
3788                 {
3789                         const struct ahc_syncrate *syncrate;
3790                         u_int   period;
3791                         u_int   offset;
3792                         u_int   bus_width;
3793                         u_int   ppr_options;
3794                         u_int   saved_width;
3795                         u_int   saved_offset;
3796                         u_int   saved_ppr_options;
3797
3798                         if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3799                                 reject = TRUE;
3800                                 break;
3801                         }
3802
3803                         /*
3804                          * Wait until we have all args before validating
3805                          * and acting on this message.
3806                          *
3807                          * Add one to MSG_EXT_PPR_LEN to account for
3808                          * the extended message preamble.
3809                          */
3810                         if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3811                                 break;
3812
3813                         period = ahc->msgin_buf[3];
3814                         offset = ahc->msgin_buf[5];
3815                         bus_width = ahc->msgin_buf[6];
3816                         saved_width = bus_width;
3817                         ppr_options = ahc->msgin_buf[7];
3818                         /*
3819                          * According to the spec, a DT only
3820                          * period factor with no DT option
3821                          * set implies async.
3822                          */
3823                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3824                          && period == 9)
3825                                 offset = 0;
3826                         saved_ppr_options = ppr_options;
3827                         saved_offset = offset;
3828
3829                         /*
3830                          * Mask out any options we don't support
3831                          * on any controller.  Transfer options are
3832                          * only available if we are negotiating wide.
3833                          */
3834                         ppr_options &= MSG_EXT_PPR_DT_REQ;
3835                         if (bus_width == 0)
3836                                 ppr_options = 0;
3837
3838                         ahc_validate_width(ahc, tinfo, &bus_width,
3839                                            devinfo->role);
3840                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3841                                                            &ppr_options,
3842                                                            devinfo->role);
3843                         ahc_validate_offset(ahc, tinfo, syncrate,
3844                                             &offset, bus_width,
3845                                             devinfo->role);
3846
3847                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3848                                 /*
3849                                  * If we are unable to do any of the
3850                                  * requested options (we went too low),
3851                                  * then we'll have to reject the message.
3852                                  */
3853                                 if (saved_width > bus_width
3854                                  || saved_offset != offset
3855                                  || saved_ppr_options != ppr_options) {
3856                                         reject = TRUE;
3857                                         period = 0;
3858                                         offset = 0;
3859                                         bus_width = 0;
3860                                         ppr_options = 0;
3861                                         syncrate = NULL;
3862                                 }
3863                         } else {
3864                                 if (devinfo->role != ROLE_TARGET)
3865                                         printk("(%s:%c:%d:%d): Target "
3866                                                "Initiated PPR\n",
3867                                                ahc_name(ahc), devinfo->channel,
3868                                                devinfo->target, devinfo->lun);
3869                                 else
3870                                         printk("(%s:%c:%d:%d): Initiator "
3871                                                "Initiated PPR\n",
3872                                                ahc_name(ahc), devinfo->channel,
3873                                                devinfo->target, devinfo->lun);
3874                                 ahc->msgout_index = 0;
3875                                 ahc->msgout_len = 0;
3876                                 ahc_construct_ppr(ahc, devinfo, period, offset,
3877                                                   bus_width, ppr_options);
3878                                 ahc->msgout_index = 0;
3879                                 response = TRUE;
3880                         }
3881                         if (bootverbose) {
3882                                 printk("(%s:%c:%d:%d): Received PPR width %x, "
3883                                        "period %x, offset %x,options %x\n"
3884                                        "\tFiltered to width %x, period %x, "
3885                                        "offset %x, options %x\n",
3886                                        ahc_name(ahc), devinfo->channel,
3887                                        devinfo->target, devinfo->lun,
3888                                        saved_width, ahc->msgin_buf[3],
3889                                        saved_offset, saved_ppr_options,
3890                                        bus_width, period, offset, ppr_options);
3891                         }
3892                         ahc_set_width(ahc, devinfo, bus_width,
3893                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3894                                       /*paused*/TRUE);
3895                         ahc_set_syncrate(ahc, devinfo,
3896                                          syncrate, period,
3897                                          offset, ppr_options,
3898                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3899                                          /*paused*/TRUE);
3900                         done = MSGLOOP_MSGCOMPLETE;
3901                         break;
3902                 }
3903                 default:
3904                         /* Unknown extended message.  Reject it. */
3905                         reject = TRUE;
3906                         break;
3907                 }
3908                 break;
3909         }
3910 #ifdef AHC_TARGET_MODE
3911         case MSG_BUS_DEV_RESET:
3912                 ahc_handle_devreset(ahc, devinfo,
3913                                     CAM_BDR_SENT,
3914                                     "Bus Device Reset Received",
3915                                     /*verbose_level*/0);
3916                 ahc_restart(ahc);
3917                 done = MSGLOOP_TERMINATED;
3918                 break;
3919         case MSG_ABORT_TAG:
3920         case MSG_ABORT:
3921         case MSG_CLEAR_QUEUE:
3922         {
3923                 int tag;
3924
3925                 /* Target mode messages */
3926                 if (devinfo->role != ROLE_TARGET) {
3927                         reject = TRUE;
3928                         break;
3929                 }
3930                 tag = SCB_LIST_NULL;
3931                 if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3932                         tag = ahc_inb(ahc, INITIATOR_TAG);
3933                 ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3934                                devinfo->lun, tag, ROLE_TARGET,
3935                                CAM_REQ_ABORTED);
3936
3937                 tstate = ahc->enabled_targets[devinfo->our_scsiid];
3938                 if (tstate != NULL) {
3939                         struct ahc_tmode_lstate* lstate;
3940
3941                         lstate = tstate->enabled_luns[devinfo->lun];
3942                         if (lstate != NULL) {
3943                                 ahc_queue_lstate_event(ahc, lstate,
3944                                                        devinfo->our_scsiid,
3945                                                        ahc->msgin_buf[0],
3946                                                        /*arg*/tag);
3947                                 ahc_send_lstate_events(ahc, lstate);
3948                         }
3949                 }
3950                 ahc_restart(ahc);
3951                 done = MSGLOOP_TERMINATED;
3952                 break;
3953         }
3954 #endif
3955         case MSG_TERM_IO_PROC:
3956         default:
3957                 reject = TRUE;
3958                 break;
3959         }
3960
3961         if (reject) {
3962                 /*
3963                  * Setup to reject the message.
3964                  */
3965                 ahc->msgout_index = 0;
3966                 ahc->msgout_len = 1;
3967                 ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3968                 done = MSGLOOP_MSGCOMPLETE;
3969                 response = TRUE;
3970         }
3971
3972         if (done != MSGLOOP_IN_PROG && !response)
3973                 /* Clear the outgoing message buffer */
3974                 ahc->msgout_len = 0;
3975
3976         return (done);
3977 }
3978
3979 /*
3980  * Process a message reject message.
3981  */
3982 static int
3983 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3984 {
3985         /*
3986          * What we care about here is if we had an
3987          * outstanding SDTR or WDTR message for this
3988          * target.  If we did, this is a signal that
3989          * the target is refusing negotiation.
3990          */
3991         struct scb *scb;
3992         struct ahc_initiator_tinfo *tinfo;
3993         struct ahc_tmode_tstate *tstate;
3994         u_int scb_index;
3995         u_int last_msg;
3996         int   response = 0;
3997
3998         scb_index = ahc_inb(ahc, SCB_TAG);
3999         scb = ahc_lookup_scb(ahc, scb_index);
4000         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
4001                                     devinfo->our_scsiid,
4002                                     devinfo->target, &tstate);
4003         /* Might be necessary */
4004         last_msg = ahc_inb(ahc, LAST_MSG);
4005
4006         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4007                 /*
4008                  * Target does not support the PPR message.
4009                  * Attempt to negotiate SPI-2 style.
4010                  */
4011                 if (bootverbose) {
4012                         printk("(%s:%c:%d:%d): PPR Rejected. "
4013                                "Trying WDTR/SDTR\n",
4014                                ahc_name(ahc), devinfo->channel,
4015                                devinfo->target, devinfo->lun);
4016                 }
4017                 tinfo->goal.ppr_options = 0;
4018                 tinfo->curr.transport_version = 2;
4019                 tinfo->goal.transport_version = 2;
4020                 ahc->msgout_index = 0;
4021                 ahc->msgout_len = 0;
4022                 ahc_build_transfer_msg(ahc, devinfo);
4023                 ahc->msgout_index = 0;
4024                 response = 1;
4025         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4026
4027                 /* note 8bit xfers */
4028                 printk("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4029                        "8bit transfers\n", ahc_name(ahc),
4030                        devinfo->channel, devinfo->target, devinfo->lun);
4031                 ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4032                               AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4033                               /*paused*/TRUE);
4034                 /*
4035                  * No need to clear the sync rate.  If the target
4036                  * did not accept the command, our syncrate is
4037                  * unaffected.  If the target started the negotiation,
4038                  * but rejected our response, we already cleared the
4039                  * sync rate before sending our WDTR.
4040                  */
4041                 if (tinfo->goal.offset != tinfo->curr.offset) {
4042
4043                         /* Start the sync negotiation */
4044                         ahc->msgout_index = 0;
4045                         ahc->msgout_len = 0;
4046                         ahc_build_transfer_msg(ahc, devinfo);
4047                         ahc->msgout_index = 0;
4048                         response = 1;
4049                 }
4050         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4051                 /* note asynch xfers and clear flag */
4052                 ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
4053                                  /*offset*/0, /*ppr_options*/0,
4054                                  AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4055                                  /*paused*/TRUE);
4056                 printk("(%s:%c:%d:%d): refuses synchronous negotiation. "
4057                        "Using asynchronous transfers\n",
4058                        ahc_name(ahc), devinfo->channel,
4059                        devinfo->target, devinfo->lun);
4060         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4061                 int tag_type;
4062                 int mask;
4063
4064                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4065
4066                 if (tag_type == MSG_SIMPLE_TASK) {
4067                         printk("(%s:%c:%d:%d): refuses tagged commands.  "
4068                                "Performing non-tagged I/O\n", ahc_name(ahc),
4069                                devinfo->channel, devinfo->target, devinfo->lun);
4070                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_NONE);
4071                         mask = ~0x23;
4072                 } else {
4073                         printk("(%s:%c:%d:%d): refuses %s tagged commands.  "
4074                                "Performing simple queue tagged I/O only\n",
4075                                ahc_name(ahc), devinfo->channel, devinfo->target,
4076                                devinfo->lun, tag_type == MSG_ORDERED_TASK
4077                                ? "ordered" : "head of queue");
4078                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_BASIC);
4079                         mask = ~0x03;
4080                 }
4081
4082                 /*
4083                  * Resend the identify for this CCB as the target
4084                  * may believe that the selection is invalid otherwise.
4085                  */
4086                 ahc_outb(ahc, SCB_CONTROL,
4087                          ahc_inb(ahc, SCB_CONTROL) & mask);
4088                 scb->hscb->control &= mask;
4089                 ahc_set_transaction_tag(scb, /*enabled*/FALSE,
4090                                         /*type*/MSG_SIMPLE_TASK);
4091                 ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
4092                 ahc_assert_atn(ahc);
4093
4094                 /*
4095                  * This transaction is now at the head of
4096                  * the untagged queue for this target.
4097                  */
4098                 if ((ahc->flags & AHC_SCB_BTT) == 0) {
4099                         struct scb_tailq *untagged_q;
4100
4101                         untagged_q =
4102                             &(ahc->untagged_queues[devinfo->target_offset]);
4103                         TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
4104                         scb->flags |= SCB_UNTAGGEDQ;
4105                 }
4106                 ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4107                              scb->hscb->tag);
4108
4109                 /*
4110                  * Requeue all tagged commands for this target
4111                  * currently in our possession so they can be
4112                  * converted to untagged commands.
4113                  */
4114                 ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
4115                                    SCB_GET_CHANNEL(ahc, scb),
4116                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4117                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
4118                                    SEARCH_COMPLETE);
4119         } else {
4120                 /*
4121                  * Otherwise, we ignore it.
4122                  */
4123                 printk("%s:%c:%d: Message reject for %x -- ignored\n",
4124                        ahc_name(ahc), devinfo->channel, devinfo->target,
4125                        last_msg);
4126         }
4127         return (response);
4128 }
4129
4130 /*
4131  * Process an ingnore wide residue message.
4132  */
4133 static void
4134 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4135 {
4136         u_int scb_index;
4137         struct scb *scb;
4138
4139         scb_index = ahc_inb(ahc, SCB_TAG);
4140         scb = ahc_lookup_scb(ahc, scb_index);
4141         /*
4142          * XXX Actually check data direction in the sequencer?
4143          * Perhaps add datadir to some spare bits in the hscb?
4144          */
4145         if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
4146          || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
4147                 /*
4148                  * Ignore the message if we haven't
4149                  * seen an appropriate data phase yet.
4150                  */
4151         } else {
4152                 /*
4153                  * If the residual occurred on the last
4154                  * transfer and the transfer request was
4155                  * expected to end on an odd count, do
4156                  * nothing.  Otherwise, subtract a byte
4157                  * and update the residual count accordingly.
4158                  */
4159                 uint32_t sgptr;
4160
4161                 sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4162                 if ((sgptr & SG_LIST_NULL) != 0
4163                  && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
4164                         /*
4165                          * If the residual occurred on the last
4166                          * transfer and the transfer request was
4167                          * expected to end on an odd count, do
4168                          * nothing.
4169                          */
4170                 } else {
4171                         struct ahc_dma_seg *sg;
4172                         uint32_t data_cnt;
4173                         uint32_t data_addr;
4174                         uint32_t sglen;
4175
4176                         /* Pull in all of the sgptr */
4177                         sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
4178                         data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
4179
4180                         if ((sgptr & SG_LIST_NULL) != 0) {
4181                                 /*
4182                                  * The residual data count is not updated
4183                                  * for the command run to completion case.
4184                                  * Explicitly zero the count.
4185                                  */
4186                                 data_cnt &= ~AHC_SG_LEN_MASK;
4187                         }
4188
4189                         data_addr = ahc_inl(ahc, SHADDR);
4190
4191                         data_cnt += 1;
4192                         data_addr -= 1;
4193                         sgptr &= SG_PTR_MASK;
4194
4195                         sg = ahc_sg_bus_to_virt(scb, sgptr);
4196
4197                         /*
4198                          * The residual sg ptr points to the next S/G
4199                          * to load so we must go back one.
4200                          */
4201                         sg--;
4202                         sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
4203                         if (sg != scb->sg_list
4204                          && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
4205
4206                                 sg--;
4207                                 sglen = ahc_le32toh(sg->len);
4208                                 /*
4209                                  * Preserve High Address and SG_LIST bits
4210                                  * while setting the count to 1.
4211                                  */
4212                                 data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
4213                                 data_addr = ahc_le32toh(sg->addr)
4214                                           + (sglen & AHC_SG_LEN_MASK) - 1;
4215
4216                                 /*
4217                                  * Increment sg so it points to the
4218                                  * "next" sg.
4219                                  */
4220                                 sg++;
4221                                 sgptr = ahc_sg_virt_to_bus(scb, sg);
4222                         }
4223                         ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
4224                         ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
4225                         /*
4226                          * Toggle the "oddness" of the transfer length
4227                          * to handle this mid-transfer ignore wide
4228                          * residue.  This ensures that the oddness is
4229                          * correct for subsequent data transfers.
4230                          */
4231                         ahc_outb(ahc, SCB_LUN,
4232                                  ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
4233                 }
4234         }
4235 }
4236
4237
4238 /*
4239  * Reinitialize the data pointers for the active transfer
4240  * based on its current residual.
4241  */
4242 static void
4243 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
4244 {
4245         struct   scb *scb;
4246         struct   ahc_dma_seg *sg;
4247         u_int    scb_index;
4248         uint32_t sgptr;
4249         uint32_t resid;
4250         uint32_t dataptr;
4251
4252         scb_index = ahc_inb(ahc, SCB_TAG);
4253         scb = ahc_lookup_scb(ahc, scb_index);
4254         sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
4255               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
4256               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
4257               | ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4258
4259         sgptr &= SG_PTR_MASK;
4260         sg = ahc_sg_bus_to_virt(scb, sgptr);
4261
4262         /* The residual sg_ptr always points to the next sg */
4263         sg--;
4264
4265         resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
4266               | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
4267               | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
4268
4269         dataptr = ahc_le32toh(sg->addr)
4270                 + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
4271                 - resid;
4272         if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
4273                 u_int dscommand1;
4274
4275                 dscommand1 = ahc_inb(ahc, DSCOMMAND1);
4276                 ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
4277                 ahc_outb(ahc, HADDR,
4278                          (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
4279                 ahc_outb(ahc, DSCOMMAND1, dscommand1);
4280         }
4281         ahc_outb(ahc, HADDR + 3, dataptr >> 24);
4282         ahc_outb(ahc, HADDR + 2, dataptr >> 16);
4283         ahc_outb(ahc, HADDR + 1, dataptr >> 8);
4284         ahc_outb(ahc, HADDR, dataptr);
4285         ahc_outb(ahc, HCNT + 2, resid >> 16);
4286         ahc_outb(ahc, HCNT + 1, resid >> 8);
4287         ahc_outb(ahc, HCNT, resid);
4288         if ((ahc->features & AHC_ULTRA2) == 0) {
4289                 ahc_outb(ahc, STCNT + 2, resid >> 16);
4290                 ahc_outb(ahc, STCNT + 1, resid >> 8);
4291                 ahc_outb(ahc, STCNT, resid);
4292         }
4293 }
4294
4295 /*
4296  * Handle the effects of issuing a bus device reset message.
4297  */
4298 static void
4299 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4300                     cam_status status, char *message, int verbose_level)
4301 {
4302 #ifdef AHC_TARGET_MODE
4303         struct ahc_tmode_tstate* tstate;
4304         u_int lun;
4305 #endif
4306         int found;
4307
4308         found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
4309                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
4310                                status);
4311
4312 #ifdef AHC_TARGET_MODE
4313         /*
4314          * Send an immediate notify ccb to all target mord peripheral
4315          * drivers affected by this action.
4316          */
4317         tstate = ahc->enabled_targets[devinfo->our_scsiid];
4318         if (tstate != NULL) {
4319                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
4320                         struct ahc_tmode_lstate* lstate;
4321
4322                         lstate = tstate->enabled_luns[lun];
4323                         if (lstate == NULL)
4324                                 continue;
4325
4326                         ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
4327                                                MSG_BUS_DEV_RESET, /*arg*/0);
4328                         ahc_send_lstate_events(ahc, lstate);
4329                 }
4330         }
4331 #endif
4332
4333         /*
4334          * Go back to async/narrow transfers and renegotiate.
4335          */
4336         ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4337                       AHC_TRANS_CUR, /*paused*/TRUE);
4338         ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
4339                          /*period*/0, /*offset*/0, /*ppr_options*/0,
4340                          AHC_TRANS_CUR, /*paused*/TRUE);
4341
4342         if (status != CAM_SEL_TIMEOUT)
4343                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
4344                                CAM_LUN_WILDCARD, AC_SENT_BDR);
4345
4346         if (message != NULL
4347          && (verbose_level <= bootverbose))
4348                 printk("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
4349                        message, devinfo->channel, devinfo->target, found);
4350 }
4351
4352 #ifdef AHC_TARGET_MODE
4353 static void
4354 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4355                        struct scb *scb)
4356 {
4357
4358         /*
4359          * To facilitate adding multiple messages together,
4360          * each routine should increment the index and len
4361          * variables instead of setting them explicitly.
4362          */
4363         ahc->msgout_index = 0;
4364         ahc->msgout_len = 0;
4365
4366         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
4367                 ahc_build_transfer_msg(ahc, devinfo);
4368         else
4369                 panic("ahc_intr: AWAITING target message with no message");
4370
4371         ahc->msgout_index = 0;
4372         ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
4373 }
4374 #endif
4375 /**************************** Initialization **********************************/
4376 /*
4377  * Allocate a controller structure for a new device
4378  * and perform initial initializion.
4379  */
4380 struct ahc_softc *
4381 ahc_alloc(void *platform_arg, char *name)
4382 {
4383         struct  ahc_softc *ahc;
4384         int     i;
4385
4386         ahc = kzalloc(sizeof(*ahc), GFP_ATOMIC);
4387         if (!ahc) {
4388                 printk("aic7xxx: cannot malloc softc!\n");
4389                 kfree(name);
4390                 return NULL;
4391         }
4392
4393         ahc->seep_config = kmalloc(sizeof(*ahc->seep_config), GFP_ATOMIC);
4394         if (ahc->seep_config == NULL) {
4395                 kfree(ahc);
4396                 kfree(name);
4397                 return (NULL);
4398         }
4399         LIST_INIT(&ahc->pending_scbs);
4400         /* We don't know our unit number until the OSM sets it */
4401         ahc->name = name;
4402         ahc->unit = -1;
4403         ahc->description = NULL;
4404         ahc->channel = 'A';
4405         ahc->channel_b = 'B';
4406         ahc->chip = AHC_NONE;
4407         ahc->features = AHC_FENONE;
4408         ahc->bugs = AHC_BUGNONE;
4409         ahc->flags = AHC_FNONE;
4410         /*
4411          * Default to all error reporting enabled with the
4412          * sequencer operating at its fastest speed.
4413          * The bus attach code may modify this.
4414          */
4415         ahc->seqctl = FASTMODE;
4416
4417         for (i = 0; i < AHC_NUM_TARGETS; i++)
4418                 TAILQ_INIT(&ahc->untagged_queues[i]);
4419         if (ahc_platform_alloc(ahc, platform_arg) != 0) {
4420                 ahc_free(ahc);
4421                 ahc = NULL;
4422         }
4423         return (ahc);
4424 }
4425
4426 int
4427 ahc_softc_init(struct ahc_softc *ahc)
4428 {
4429
4430         /* The IRQMS bit is only valid on VL and EISA chips */
4431         if ((ahc->chip & AHC_PCI) == 0)
4432                 ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
4433         else
4434                 ahc->unpause = 0;
4435         ahc->pause = ahc->unpause | PAUSE;
4436         /* XXX The shared scb data stuff should be deprecated */
4437         if (ahc->scb_data == NULL) {
4438                 ahc->scb_data = kzalloc(sizeof(*ahc->scb_data), GFP_ATOMIC);
4439                 if (ahc->scb_data == NULL)
4440                         return (ENOMEM);
4441         }
4442
4443         return (0);
4444 }
4445
4446 void
4447 ahc_set_unit(struct ahc_softc *ahc, int unit)
4448 {
4449         ahc->unit = unit;
4450 }
4451
4452 void
4453 ahc_set_name(struct ahc_softc *ahc, char *name)
4454 {
4455         kfree(ahc->name);
4456         ahc->name = name;
4457 }
4458
4459 void
4460 ahc_free(struct ahc_softc *ahc)
4461 {
4462         int i;
4463
4464         switch (ahc->init_level) {
4465         default:
4466         case 5:
4467                 ahc_shutdown(ahc);
4468                 fallthrough;
4469         case 4:
4470                 ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
4471                                   ahc->shared_data_dmamap);
4472                 fallthrough;
4473         case 3:
4474                 ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
4475                                 ahc->shared_data_dmamap);
4476                 ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
4477                                    ahc->shared_data_dmamap);
4478                 fallthrough;
4479         case 2:
4480                 ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
4481         case 1:
4482                 break;
4483         case 0:
4484                 break;
4485         }
4486
4487         ahc_platform_free(ahc);
4488         ahc_fini_scbdata(ahc);
4489         for (i = 0; i < AHC_NUM_TARGETS; i++) {
4490                 struct ahc_tmode_tstate *tstate;
4491
4492                 tstate = ahc->enabled_targets[i];
4493                 if (tstate != NULL) {
4494 #ifdef AHC_TARGET_MODE
4495                         int j;
4496
4497                         for (j = 0; j < AHC_NUM_LUNS; j++) {
4498                                 struct ahc_tmode_lstate *lstate;
4499
4500                                 lstate = tstate->enabled_luns[j];
4501                                 if (lstate != NULL) {
4502                                         xpt_free_path(lstate->path);
4503                                         kfree(lstate);
4504                                 }
4505                         }
4506 #endif
4507                         kfree(tstate);
4508                 }
4509         }
4510 #ifdef AHC_TARGET_MODE
4511         if (ahc->black_hole != NULL) {
4512                 xpt_free_path(ahc->black_hole->path);
4513                 kfree(ahc->black_hole);
4514         }
4515 #endif
4516         kfree(ahc->name);
4517         kfree(ahc->seep_config);
4518         kfree(ahc);
4519         return;
4520 }
4521
4522 static void
4523 ahc_shutdown(void *arg)
4524 {
4525         struct  ahc_softc *ahc;
4526         int     i;
4527
4528         ahc = (struct ahc_softc *)arg;
4529
4530         /* This will reset most registers to 0, but not all */
4531         ahc_reset(ahc, /*reinit*/FALSE);
4532         ahc_outb(ahc, SCSISEQ, 0);
4533         ahc_outb(ahc, SXFRCTL0, 0);
4534         ahc_outb(ahc, DSPCISTATUS, 0);
4535
4536         for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4537                 ahc_outb(ahc, i, 0);
4538 }
4539
4540 /*
4541  * Reset the controller and record some information about it
4542  * that is only available just after a reset.  If "reinit" is
4543  * non-zero, this reset occurred after initial configuration
4544  * and the caller requests that the chip be fully reinitialized
4545  * to a runable state.  Chip interrupts are *not* enabled after
4546  * a reinitialization.  The caller must enable interrupts via
4547  * ahc_intr_enable().
4548  */
4549 int
4550 ahc_reset(struct ahc_softc *ahc, int reinit)
4551 {
4552         u_int   sblkctl;
4553         u_int   sxfrctl1_a, sxfrctl1_b;
4554         int     error;
4555         int     wait;
4556
4557         /*
4558          * Preserve the value of the SXFRCTL1 register for all channels.
4559          * It contains settings that affect termination and we don't want
4560          * to disturb the integrity of the bus.
4561          */
4562         ahc_pause(ahc);
4563         sxfrctl1_b = 0;
4564         if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4565                 u_int sblkctl;
4566
4567                 /*
4568                  * Save channel B's settings in case this chip
4569                  * is setup for TWIN channel operation.
4570                  */
4571                 sblkctl = ahc_inb(ahc, SBLKCTL);
4572                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4573                 sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4574                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4575         }
4576         sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4577
4578         ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4579
4580         /*
4581          * Ensure that the reset has finished.  We delay 1000us
4582          * prior to reading the register to make sure the chip
4583          * has sufficiently completed its reset to handle register
4584          * accesses.
4585          */
4586         wait = 1000;
4587         do {
4588                 ahc_delay(1000);
4589         } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4590
4591         if (wait == 0) {
4592                 printk("%s: WARNING - Failed chip reset!  "
4593                        "Trying to initialize anyway.\n", ahc_name(ahc));
4594         }
4595         ahc_outb(ahc, HCNTRL, ahc->pause);
4596
4597         /* Determine channel configuration */
4598         sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4599         /* No Twin Channel PCI cards */
4600         if ((ahc->chip & AHC_PCI) != 0)
4601                 sblkctl &= ~SELBUSB;
4602         switch (sblkctl) {
4603         case 0:
4604                 /* Single Narrow Channel */
4605                 break;
4606         case 2:
4607                 /* Wide Channel */
4608                 ahc->features |= AHC_WIDE;
4609                 break;
4610         case 8:
4611                 /* Twin Channel */
4612                 ahc->features |= AHC_TWIN;
4613                 break;
4614         default:
4615                 printk(" Unsupported adapter type.  Ignoring\n");
4616                 return(-1);
4617         }
4618
4619         /*
4620          * Reload sxfrctl1.
4621          *
4622          * We must always initialize STPWEN to 1 before we
4623          * restore the saved values.  STPWEN is initialized
4624          * to a tri-state condition which can only be cleared
4625          * by turning it on.
4626          */
4627         if ((ahc->features & AHC_TWIN) != 0) {
4628                 u_int sblkctl;
4629
4630                 sblkctl = ahc_inb(ahc, SBLKCTL);
4631                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4632                 ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4633                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4634         }
4635         ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4636
4637         error = 0;
4638         if (reinit != 0)
4639                 /*
4640                  * If a recovery action has forced a chip reset,
4641                  * re-initialize the chip to our liking.
4642                  */
4643                 error = ahc->bus_chip_init(ahc);
4644 #ifdef AHC_DUMP_SEQ
4645         else
4646                 ahc_dumpseq(ahc);
4647 #endif
4648
4649         return (error);
4650 }
4651
4652 /*
4653  * Determine the number of SCBs available on the controller
4654  */
4655 int
4656 ahc_probe_scbs(struct ahc_softc *ahc) {
4657         int i;
4658
4659         for (i = 0; i < AHC_SCB_MAX; i++) {
4660
4661                 ahc_outb(ahc, SCBPTR, i);
4662                 ahc_outb(ahc, SCB_BASE, i);
4663                 if (ahc_inb(ahc, SCB_BASE) != i)
4664                         break;
4665                 ahc_outb(ahc, SCBPTR, 0);
4666                 if (ahc_inb(ahc, SCB_BASE) != 0)
4667                         break;
4668         }
4669         return (i);
4670 }
4671
4672 static void
4673 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4674 {
4675         dma_addr_t *baddr;
4676
4677         baddr = (dma_addr_t *)arg;
4678         *baddr = segs->ds_addr;
4679 }
4680
4681 static void
4682 ahc_build_free_scb_list(struct ahc_softc *ahc)
4683 {
4684         int scbsize;
4685         int i;
4686
4687         scbsize = 32;
4688         if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4689                 scbsize = 64;
4690
4691         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4692                 int j;
4693
4694                 ahc_outb(ahc, SCBPTR, i);
4695
4696                 /*
4697                  * Touch all SCB bytes to avoid parity errors
4698                  * should one of our debugging routines read
4699                  * an otherwise uninitiatlized byte.
4700                  */
4701                 for (j = 0; j < scbsize; j++)
4702                         ahc_outb(ahc, SCB_BASE+j, 0xFF);
4703
4704                 /* Clear the control byte. */
4705                 ahc_outb(ahc, SCB_CONTROL, 0);
4706
4707                 /* Set the next pointer */
4708                 if ((ahc->flags & AHC_PAGESCBS) != 0)
4709                         ahc_outb(ahc, SCB_NEXT, i+1);
4710                 else
4711                         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4712
4713                 /* Make the tag number, SCSIID, and lun invalid */
4714                 ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4715                 ahc_outb(ahc, SCB_SCSIID, 0xFF);
4716                 ahc_outb(ahc, SCB_LUN, 0xFF);
4717         }
4718
4719         if ((ahc->flags & AHC_PAGESCBS) != 0) {
4720                 /* SCB 0 heads the free list. */
4721                 ahc_outb(ahc, FREE_SCBH, 0);
4722         } else {
4723                 /* No free list. */
4724                 ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4725         }
4726
4727         /* Make sure that the last SCB terminates the free list */
4728         ahc_outb(ahc, SCBPTR, i-1);
4729         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4730 }
4731
4732 static int
4733 ahc_init_scbdata(struct ahc_softc *ahc)
4734 {
4735         struct scb_data *scb_data;
4736
4737         scb_data = ahc->scb_data;
4738         SLIST_INIT(&scb_data->free_scbs);
4739         SLIST_INIT(&scb_data->sg_maps);
4740
4741         /* Allocate SCB resources */
4742         scb_data->scbarray = kcalloc(AHC_SCB_MAX_ALLOC, sizeof(struct scb),
4743                                      GFP_ATOMIC);
4744         if (scb_data->scbarray == NULL)
4745                 return (ENOMEM);
4746
4747         /* Determine the number of hardware SCBs and initialize them */
4748
4749         scb_data->maxhscbs = ahc_probe_scbs(ahc);
4750         if (ahc->scb_data->maxhscbs == 0) {
4751                 printk("%s: No SCB space found\n", ahc_name(ahc));
4752                 return (ENXIO);
4753         }
4754
4755         /*
4756          * Create our DMA tags.  These tags define the kinds of device
4757          * accessible memory allocations and memory mappings we will
4758          * need to perform during normal operation.
4759          *
4760          * Unless we need to further restrict the allocation, we rely
4761          * on the restrictions of the parent dmat, hence the common
4762          * use of MAXADDR and MAXSIZE.
4763          */
4764
4765         /* DMA tag for our hardware scb structures */
4766         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4767                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4768                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4769                                /*highaddr*/BUS_SPACE_MAXADDR,
4770                                /*filter*/NULL, /*filterarg*/NULL,
4771                                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4772                                /*nsegments*/1,
4773                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4774                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4775                 goto error_exit;
4776         }
4777
4778         scb_data->init_level++;
4779
4780         /* Allocation for our hscbs */
4781         if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4782                              (void **)&scb_data->hscbs,
4783                              BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4784                 goto error_exit;
4785         }
4786
4787         scb_data->init_level++;
4788
4789         /* And permanently map them */
4790         ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4791                         scb_data->hscbs,
4792                         AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4793                         ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4794
4795         scb_data->init_level++;
4796
4797         /* DMA tag for our sense buffers */
4798         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4799                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4800                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4801                                /*highaddr*/BUS_SPACE_MAXADDR,
4802                                /*filter*/NULL, /*filterarg*/NULL,
4803                                AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4804                                /*nsegments*/1,
4805                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4806                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4807                 goto error_exit;
4808         }
4809
4810         scb_data->init_level++;
4811
4812         /* Allocate them */
4813         if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4814                              (void **)&scb_data->sense,
4815                              BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4816                 goto error_exit;
4817         }
4818
4819         scb_data->init_level++;
4820
4821         /* And permanently map them */
4822         ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4823                         scb_data->sense,
4824                         AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4825                         ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4826
4827         scb_data->init_level++;
4828
4829         /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4830         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4831                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4832                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4833                                /*highaddr*/BUS_SPACE_MAXADDR,
4834                                /*filter*/NULL, /*filterarg*/NULL,
4835                                PAGE_SIZE, /*nsegments*/1,
4836                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4837                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4838                 goto error_exit;
4839         }
4840
4841         scb_data->init_level++;
4842
4843         /* Perform initial CCB allocation */
4844         memset(scb_data->hscbs, 0,
4845                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4846         ahc_alloc_scbs(ahc);
4847
4848         if (scb_data->numscbs == 0) {
4849                 printk("%s: ahc_init_scbdata - "
4850                        "Unable to allocate initial scbs\n",
4851                        ahc_name(ahc));
4852                 goto error_exit;
4853         }
4854
4855         /*
4856          * Reserve the next queued SCB.
4857          */
4858         ahc->next_queued_scb = ahc_get_scb(ahc);
4859
4860         /*
4861          * Note that we were successful
4862          */
4863         return (0);
4864
4865 error_exit:
4866
4867         return (ENOMEM);
4868 }
4869
4870 static void
4871 ahc_fini_scbdata(struct ahc_softc *ahc)
4872 {
4873         struct scb_data *scb_data;
4874
4875         scb_data = ahc->scb_data;
4876         if (scb_data == NULL)
4877                 return;
4878
4879         switch (scb_data->init_level) {
4880         default:
4881         case 7:
4882         {
4883                 struct sg_map_node *sg_map;
4884
4885                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4886                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4887                         ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4888                                           sg_map->sg_dmamap);
4889                         ahc_dmamem_free(ahc, scb_data->sg_dmat,
4890                                         sg_map->sg_vaddr,
4891                                         sg_map->sg_dmamap);
4892                         kfree(sg_map);
4893                 }
4894                 ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4895         }
4896                 fallthrough;
4897         case 6:
4898                 ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4899                                   scb_data->sense_dmamap);
4900                 fallthrough;
4901         case 5:
4902                 ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4903                                 scb_data->sense_dmamap);
4904                 ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4905                                    scb_data->sense_dmamap);
4906                 fallthrough;
4907         case 4:
4908                 ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4909                 fallthrough;
4910         case 3:
4911                 ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4912                                   scb_data->hscb_dmamap);
4913                 fallthrough;
4914         case 2:
4915                 ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4916                                 scb_data->hscb_dmamap);
4917                 ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4918                                    scb_data->hscb_dmamap);
4919                 fallthrough;
4920         case 1:
4921                 ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4922                 break;
4923         case 0:
4924                 break;
4925         }
4926         kfree(scb_data->scbarray);
4927 }
4928
4929 static void
4930 ahc_alloc_scbs(struct ahc_softc *ahc)
4931 {
4932         struct scb_data *scb_data;
4933         struct scb *next_scb;
4934         struct sg_map_node *sg_map;
4935         dma_addr_t physaddr;
4936         struct ahc_dma_seg *segs;
4937         int newcount;
4938         int i;
4939
4940         scb_data = ahc->scb_data;
4941         if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4942                 /* Can't allocate any more */
4943                 return;
4944
4945         next_scb = &scb_data->scbarray[scb_data->numscbs];
4946
4947         sg_map = kmalloc(sizeof(*sg_map), GFP_ATOMIC);
4948
4949         if (sg_map == NULL)
4950                 return;
4951
4952         /* Allocate S/G space for the next batch of SCBS */
4953         if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4954                              (void **)&sg_map->sg_vaddr,
4955                              BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4956                 kfree(sg_map);
4957                 return;
4958         }
4959
4960         SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4961
4962         ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4963                         sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4964                         &sg_map->sg_physaddr, /*flags*/0);
4965
4966         segs = sg_map->sg_vaddr;
4967         physaddr = sg_map->sg_physaddr;
4968
4969         newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
4970         newcount = min(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
4971         for (i = 0; i < newcount; i++) {
4972                 struct scb_platform_data *pdata;
4973
4974                 pdata = kmalloc(sizeof(*pdata), GFP_ATOMIC);
4975                 if (pdata == NULL)
4976                         break;
4977                 next_scb->platform_data = pdata;
4978                 next_scb->sg_map = sg_map;
4979                 next_scb->sg_list = segs;
4980                 /*
4981                  * The sequencer always starts with the second entry.
4982                  * The first entry is embedded in the scb.
4983                  */
4984                 next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
4985                 next_scb->ahc_softc = ahc;
4986                 next_scb->flags = SCB_FREE;
4987                 next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
4988                 next_scb->hscb->tag = ahc->scb_data->numscbs;
4989                 SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
4990                                   next_scb, links.sle);
4991                 segs += AHC_NSEG;
4992                 physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
4993                 next_scb++;
4994                 ahc->scb_data->numscbs++;
4995         }
4996 }
4997
4998 void
4999 ahc_controller_info(struct ahc_softc *ahc, char *buf)
5000 {
5001         int len;
5002
5003         len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
5004         buf += len;
5005         if ((ahc->features & AHC_TWIN) != 0)
5006                 len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
5007                               "B SCSI Id=%d, primary %c, ",
5008                               ahc->our_id, ahc->our_id_b,
5009                               (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
5010         else {
5011                 const char *speed;
5012                 const char *type;
5013
5014                 speed = "";
5015                 if ((ahc->features & AHC_ULTRA) != 0) {
5016                         speed = "Ultra ";
5017                 } else if ((ahc->features & AHC_DT) != 0) {
5018                         speed = "Ultra160 ";
5019                 } else if ((ahc->features & AHC_ULTRA2) != 0) {
5020                         speed = "Ultra2 ";
5021                 }
5022                 if ((ahc->features & AHC_WIDE) != 0) {
5023                         type = "Wide";
5024                 } else {
5025                         type = "Single";
5026                 }
5027                 len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
5028                               speed, type, ahc->channel, ahc->our_id);
5029         }
5030         buf += len;
5031
5032         if ((ahc->flags & AHC_PAGESCBS) != 0)
5033                 sprintf(buf, "%d/%d SCBs",
5034                         ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
5035         else
5036                 sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
5037 }
5038
5039 int
5040 ahc_chip_init(struct ahc_softc *ahc)
5041 {
5042         int      term;
5043         int      error;
5044         u_int    i;
5045         u_int    scsi_conf;
5046         u_int    scsiseq_template;
5047         uint32_t physaddr;
5048
5049         ahc_outb(ahc, SEQ_FLAGS, 0);
5050         ahc_outb(ahc, SEQ_FLAGS2, 0);
5051
5052         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
5053         if (ahc->features & AHC_TWIN) {
5054
5055                 /*
5056                  * Setup Channel B first.
5057                  */
5058                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
5059                 term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
5060                 ahc_outb(ahc, SCSIID, ahc->our_id_b);
5061                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5062                 ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5063                                         |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
5064                 if ((ahc->features & AHC_ULTRA2) != 0)
5065                         ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5066                 ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5067                 ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5068
5069                 /* Select Channel A */
5070                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
5071         }
5072         term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
5073         if ((ahc->features & AHC_ULTRA2) != 0)
5074                 ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
5075         else
5076                 ahc_outb(ahc, SCSIID, ahc->our_id);
5077         scsi_conf = ahc_inb(ahc, SCSICONF);
5078         ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5079                                 |term|ahc->seltime
5080                                 |ENSTIMER|ACTNEGEN);
5081         if ((ahc->features & AHC_ULTRA2) != 0)
5082                 ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5083         ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5084         ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5085
5086         /* There are no untagged SCBs active yet. */
5087         for (i = 0; i < 16; i++) {
5088                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
5089                 if ((ahc->flags & AHC_SCB_BTT) != 0) {
5090                         int lun;
5091
5092                         /*
5093                          * The SCB based BTT allows an entry per
5094                          * target and lun pair.
5095                          */
5096                         for (lun = 1; lun < AHC_NUM_LUNS; lun++)
5097                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
5098                 }
5099         }
5100
5101         /* All of our queues are empty */
5102         for (i = 0; i < 256; i++)
5103                 ahc->qoutfifo[i] = SCB_LIST_NULL;
5104         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
5105
5106         for (i = 0; i < 256; i++)
5107                 ahc->qinfifo[i] = SCB_LIST_NULL;
5108
5109         if ((ahc->features & AHC_MULTI_TID) != 0) {
5110                 ahc_outb(ahc, TARGID, 0);
5111                 ahc_outb(ahc, TARGID + 1, 0);
5112         }
5113
5114         /*
5115          * Tell the sequencer where it can find our arrays in memory.
5116          */
5117         physaddr = ahc->scb_data->hscb_busaddr;
5118         ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
5119         ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
5120         ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
5121         ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
5122
5123         physaddr = ahc->shared_data_busaddr;
5124         ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
5125         ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
5126         ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
5127         ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
5128
5129         /*
5130          * Initialize the group code to command length table.
5131          * This overrides the values in TARG_SCSIRATE, so only
5132          * setup the table after we have processed that information.
5133          */
5134         ahc_outb(ahc, CMDSIZE_TABLE, 5);
5135         ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
5136         ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
5137         ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
5138         ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
5139         ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
5140         ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
5141         ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
5142
5143         if ((ahc->features & AHC_HS_MAILBOX) != 0)
5144                 ahc_outb(ahc, HS_MAILBOX, 0);
5145
5146         /* Tell the sequencer of our initial queue positions */
5147         if ((ahc->features & AHC_TARGETMODE) != 0) {
5148                 ahc->tqinfifonext = 1;
5149                 ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
5150                 ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
5151         }
5152         ahc->qinfifonext = 0;
5153         ahc->qoutfifonext = 0;
5154         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5155                 ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
5156                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5157                 ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
5158                 ahc_outb(ahc, SDSCB_QOFF, 0);
5159         } else {
5160                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5161                 ahc_outb(ahc, QINPOS, ahc->qinfifonext);
5162                 ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
5163         }
5164
5165         /* We don't have any waiting selections */
5166         ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
5167
5168         /* Our disconnection list is empty too */
5169         ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
5170
5171         /* Message out buffer starts empty */
5172         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
5173
5174         /*
5175          * Setup the allowed SCSI Sequences based on operational mode.
5176          * If we are a target, we'll enable select in operations once
5177          * we've had a lun enabled.
5178          */
5179         scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
5180         if ((ahc->flags & AHC_INITIATORROLE) != 0)
5181                 scsiseq_template |= ENRSELI;
5182         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
5183
5184         /* Initialize our list of free SCBs. */
5185         ahc_build_free_scb_list(ahc);
5186
5187         /*
5188          * Tell the sequencer which SCB will be the next one it receives.
5189          */
5190         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5191
5192         /*
5193          * Load the Sequencer program and Enable the adapter
5194          * in "fast" mode.
5195          */
5196         if (bootverbose)
5197                 printk("%s: Downloading Sequencer Program...",
5198                        ahc_name(ahc));
5199
5200         error = ahc_loadseq(ahc);
5201         if (error != 0)
5202                 return (error);
5203
5204         if ((ahc->features & AHC_ULTRA2) != 0) {
5205                 int wait;
5206
5207                 /*
5208                  * Wait for up to 500ms for our transceivers
5209                  * to settle.  If the adapter does not have
5210                  * a cable attached, the transceivers may
5211                  * never settle, so don't complain if we
5212                  * fail here.
5213                  */
5214                 for (wait = 5000;
5215                      (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
5216                      wait--)
5217                         ahc_delay(100);
5218         }
5219         ahc_restart(ahc);
5220         return (0);
5221 }
5222
5223 /*
5224  * Start the board, ready for normal operation
5225  */
5226 int
5227 ahc_init(struct ahc_softc *ahc)
5228 {
5229         int      max_targ;
5230         u_int    i;
5231         u_int    scsi_conf;
5232         u_int    ultraenb;
5233         u_int    discenable;
5234         u_int    tagenable;
5235         size_t   driver_data_size;
5236
5237 #ifdef AHC_DEBUG
5238         if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
5239                 ahc->flags |= AHC_SEQUENCER_DEBUG;
5240 #endif
5241
5242 #ifdef AHC_PRINT_SRAM
5243         printk("Scratch Ram:");
5244         for (i = 0x20; i < 0x5f; i++) {
5245                 if (((i % 8) == 0) && (i != 0)) {
5246                         printk ("\n              ");
5247                 }
5248                 printk (" 0x%x", ahc_inb(ahc, i));
5249         }
5250         if ((ahc->features & AHC_MORE_SRAM) != 0) {
5251                 for (i = 0x70; i < 0x7f; i++) {
5252                         if (((i % 8) == 0) && (i != 0)) {
5253                                 printk ("\n              ");
5254                         }
5255                         printk (" 0x%x", ahc_inb(ahc, i));
5256                 }
5257         }
5258         printk ("\n");
5259         /*
5260          * Reading uninitialized scratch ram may
5261          * generate parity errors.
5262          */
5263         ahc_outb(ahc, CLRINT, CLRPARERR);
5264         ahc_outb(ahc, CLRINT, CLRBRKADRINT);
5265 #endif
5266         max_targ = 15;
5267
5268         /*
5269          * Assume we have a board at this stage and it has been reset.
5270          */
5271         if ((ahc->flags & AHC_USEDEFAULTS) != 0)
5272                 ahc->our_id = ahc->our_id_b = 7;
5273
5274         /*
5275          * Default to allowing initiator operations.
5276          */
5277         ahc->flags |= AHC_INITIATORROLE;
5278
5279         /*
5280          * Only allow target mode features if this unit has them enabled.
5281          */
5282         if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
5283                 ahc->features &= ~AHC_TARGETMODE;
5284
5285         ahc->init_level++;
5286
5287         /*
5288          * DMA tag for our command fifos and other data in system memory
5289          * the card's sequencer must be able to access.  For initiator
5290          * roles, we need to allocate space for the qinfifo and qoutfifo.
5291          * The qinfifo and qoutfifo are composed of 256 1 byte elements.
5292          * When providing for the target mode role, we must additionally
5293          * provide space for the incoming target command fifo and an extra
5294          * byte to deal with a dma bug in some chip versions.
5295          */
5296         driver_data_size = 2 * 256 * sizeof(uint8_t);
5297         if ((ahc->features & AHC_TARGETMODE) != 0)
5298                 driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
5299                                  + /*DMA WideOdd Bug Buffer*/1;
5300         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5301                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5302                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5303                                /*highaddr*/BUS_SPACE_MAXADDR,
5304                                /*filter*/NULL, /*filterarg*/NULL,
5305                                driver_data_size,
5306                                /*nsegments*/1,
5307                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5308                                /*flags*/0, &ahc->shared_data_dmat) != 0) {
5309                 return (ENOMEM);
5310         }
5311
5312         ahc->init_level++;
5313
5314         /* Allocation of driver data */
5315         if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
5316                              (void **)&ahc->qoutfifo,
5317                              BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
5318                 return (ENOMEM);
5319         }
5320
5321         ahc->init_level++;
5322
5323         /* And permanently map it in */
5324         ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
5325                         ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
5326                         &ahc->shared_data_busaddr, /*flags*/0);
5327
5328         if ((ahc->features & AHC_TARGETMODE) != 0) {
5329                 ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
5330                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
5331                 ahc->dma_bug_buf = ahc->shared_data_busaddr
5332                                  + driver_data_size - 1;
5333                 /* All target command blocks start out invalid. */
5334                 for (i = 0; i < AHC_TMODE_CMDS; i++)
5335                         ahc->targetcmds[i].cmd_valid = 0;
5336                 ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
5337                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
5338         }
5339         ahc->qinfifo = &ahc->qoutfifo[256];
5340
5341         ahc->init_level++;
5342
5343         /* Allocate SCB data now that buffer_dmat is initialized */
5344         if (ahc->scb_data->maxhscbs == 0)
5345                 if (ahc_init_scbdata(ahc) != 0)
5346                         return (ENOMEM);
5347
5348         /*
5349          * Allocate a tstate to house information for our
5350          * initiator presence on the bus as well as the user
5351          * data for any target mode initiator.
5352          */
5353         if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
5354                 printk("%s: unable to allocate ahc_tmode_tstate.  "
5355                        "Failing attach\n", ahc_name(ahc));
5356                 return (ENOMEM);
5357         }
5358
5359         if ((ahc->features & AHC_TWIN) != 0) {
5360                 if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
5361                         printk("%s: unable to allocate ahc_tmode_tstate.  "
5362                                "Failing attach\n", ahc_name(ahc));
5363                         return (ENOMEM);
5364                 }
5365         }
5366
5367         if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
5368                 ahc->flags |= AHC_PAGESCBS;
5369         } else {
5370                 ahc->flags &= ~AHC_PAGESCBS;
5371         }
5372
5373 #ifdef AHC_DEBUG
5374         if (ahc_debug & AHC_SHOW_MISC) {
5375                 printk("%s: hardware scb %u bytes; kernel scb %u bytes; "
5376                        "ahc_dma %u bytes\n",
5377                         ahc_name(ahc),
5378                         (u_int)sizeof(struct hardware_scb),
5379                         (u_int)sizeof(struct scb),
5380                         (u_int)sizeof(struct ahc_dma_seg));
5381         }
5382 #endif /* AHC_DEBUG */
5383
5384         /*
5385          * Look at the information that board initialization or
5386          * the board bios has left us.
5387          */
5388         if (ahc->features & AHC_TWIN) {
5389                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5390                 if ((scsi_conf & RESET_SCSI) != 0
5391                  && (ahc->flags & AHC_INITIATORROLE) != 0)
5392                         ahc->flags |= AHC_RESET_BUS_B;
5393         }
5394
5395         scsi_conf = ahc_inb(ahc, SCSICONF);
5396         if ((scsi_conf & RESET_SCSI) != 0
5397          && (ahc->flags & AHC_INITIATORROLE) != 0)
5398                 ahc->flags |= AHC_RESET_BUS_A;
5399
5400         ultraenb = 0;
5401         tagenable = ALL_TARGETS_MASK;
5402
5403         /* Grab the disconnection disable table and invert it for our needs */
5404         if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
5405                 printk("%s: Host Adapter Bios disabled.  Using default SCSI "
5406                         "device parameters\n", ahc_name(ahc));
5407                 ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
5408                               AHC_TERM_ENB_A|AHC_TERM_ENB_B;
5409                 discenable = ALL_TARGETS_MASK;
5410                 if ((ahc->features & AHC_ULTRA) != 0)
5411                         ultraenb = ALL_TARGETS_MASK;
5412         } else {
5413                 discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
5414                            | ahc_inb(ahc, DISC_DSB));
5415                 if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
5416                         ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
5417                                       | ahc_inb(ahc, ULTRA_ENB);
5418         }
5419
5420         if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
5421                 max_targ = 7;
5422
5423         for (i = 0; i <= max_targ; i++) {
5424                 struct ahc_initiator_tinfo *tinfo;
5425                 struct ahc_tmode_tstate *tstate;
5426                 u_int our_id;
5427                 u_int target_id;
5428                 char channel;
5429
5430                 channel = 'A';
5431                 our_id = ahc->our_id;
5432                 target_id = i;
5433                 if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
5434                         channel = 'B';
5435                         our_id = ahc->our_id_b;
5436                         target_id = i % 8;
5437                 }
5438                 tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
5439                                             target_id, &tstate);
5440                 /* Default to async narrow across the board */
5441                 memset(tinfo, 0, sizeof(*tinfo));
5442                 if (ahc->flags & AHC_USEDEFAULTS) {
5443                         if ((ahc->features & AHC_WIDE) != 0)
5444                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5445
5446                         /*
5447                          * These will be truncated when we determine the
5448                          * connection type we have with the target.
5449                          */
5450                         tinfo->user.period = ahc_syncrates->period;
5451                         tinfo->user.offset = MAX_OFFSET;
5452                 } else {
5453                         u_int scsirate;
5454                         uint16_t mask;
5455
5456                         /* Take the settings leftover in scratch RAM. */
5457                         scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
5458                         mask = (0x01 << i);
5459                         if ((ahc->features & AHC_ULTRA2) != 0) {
5460                                 u_int offset;
5461                                 u_int maxsync;
5462
5463                                 if ((scsirate & SOFS) == 0x0F) {
5464                                         /*
5465                                          * Haven't negotiated yet,
5466                                          * so the format is different.
5467                                          */
5468                                         scsirate = (scsirate & SXFR) >> 4
5469                                                  | (ultraenb & mask)
5470                                                   ? 0x08 : 0x0
5471                                                  | (scsirate & WIDEXFER);
5472                                         offset = MAX_OFFSET_ULTRA2;
5473                                 } else
5474                                         offset = ahc_inb(ahc, TARG_OFFSET + i);
5475                                 if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
5476                                         /* Set to the lowest sync rate, 5MHz */
5477                                         scsirate |= 0x1c;
5478                                 maxsync = AHC_SYNCRATE_ULTRA2;
5479                                 if ((ahc->features & AHC_DT) != 0)
5480                                         maxsync = AHC_SYNCRATE_DT;
5481                                 tinfo->user.period =
5482                                     ahc_find_period(ahc, scsirate, maxsync);
5483                                 if (offset == 0)
5484                                         tinfo->user.period = 0;
5485                                 else
5486                                         tinfo->user.offset = MAX_OFFSET;
5487                                 if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5488                                  && (ahc->features & AHC_DT) != 0)
5489                                         tinfo->user.ppr_options =
5490                                             MSG_EXT_PPR_DT_REQ;
5491                         } else if ((scsirate & SOFS) != 0) {
5492                                 if ((scsirate & SXFR) == 0x40
5493                                  && (ultraenb & mask) != 0) {
5494                                         /* Treat 10MHz as a non-ultra speed */
5495                                         scsirate &= ~SXFR;
5496                                         ultraenb &= ~mask;
5497                                 }
5498                                 tinfo->user.period =
5499                                     ahc_find_period(ahc, scsirate,
5500                                                     (ultraenb & mask)
5501                                                    ? AHC_SYNCRATE_ULTRA
5502                                                    : AHC_SYNCRATE_FAST);
5503                                 if (tinfo->user.period != 0)
5504                                         tinfo->user.offset = MAX_OFFSET;
5505                         }
5506                         if (tinfo->user.period == 0)
5507                                 tinfo->user.offset = 0;
5508                         if ((scsirate & WIDEXFER) != 0
5509                          && (ahc->features & AHC_WIDE) != 0)
5510                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5511                         tinfo->user.protocol_version = 4;
5512                         if ((ahc->features & AHC_DT) != 0)
5513                                 tinfo->user.transport_version = 3;
5514                         else
5515                                 tinfo->user.transport_version = 2;
5516                         tinfo->goal.protocol_version = 2;
5517                         tinfo->goal.transport_version = 2;
5518                         tinfo->curr.protocol_version = 2;
5519                         tinfo->curr.transport_version = 2;
5520                 }
5521                 tstate->ultraenb = 0;
5522         }
5523         ahc->user_discenable = discenable;
5524         ahc->user_tagenable = tagenable;
5525
5526         return (ahc->bus_chip_init(ahc));
5527 }
5528
5529 void
5530 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5531 {
5532         u_int hcntrl;
5533
5534         hcntrl = ahc_inb(ahc, HCNTRL);
5535         hcntrl &= ~INTEN;
5536         ahc->pause &= ~INTEN;
5537         ahc->unpause &= ~INTEN;
5538         if (enable) {
5539                 hcntrl |= INTEN;
5540                 ahc->pause |= INTEN;
5541                 ahc->unpause |= INTEN;
5542         }
5543         ahc_outb(ahc, HCNTRL, hcntrl);
5544 }
5545
5546 /*
5547  * Ensure that the card is paused in a location
5548  * outside of all critical sections and that all
5549  * pending work is completed prior to returning.
5550  * This routine should only be called from outside
5551  * an interrupt context.
5552  */
5553 void
5554 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5555 {
5556         int intstat;
5557         int maxloops;
5558         int paused;
5559
5560         maxloops = 1000;
5561         ahc->flags |= AHC_ALL_INTERRUPTS;
5562         paused = FALSE;
5563         do {
5564                 if (paused) {
5565                         ahc_unpause(ahc);
5566                         /*
5567                          * Give the sequencer some time to service
5568                          * any active selections.
5569                          */
5570                         ahc_delay(500);
5571                 }
5572                 ahc_intr(ahc);
5573                 ahc_pause(ahc);
5574                 paused = TRUE;
5575                 ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5576                 intstat = ahc_inb(ahc, INTSTAT);
5577                 if ((intstat & INT_PEND) == 0) {
5578                         ahc_clear_critical_section(ahc);
5579                         intstat = ahc_inb(ahc, INTSTAT);
5580                 }
5581         } while (--maxloops
5582               && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5583               && ((intstat & INT_PEND) != 0
5584                || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5585         if (maxloops == 0) {
5586                 printk("Infinite interrupt loop, INTSTAT = %x",
5587                        ahc_inb(ahc, INTSTAT));
5588         }
5589         ahc_platform_flushwork(ahc);
5590         ahc->flags &= ~AHC_ALL_INTERRUPTS;
5591 }
5592
5593 #ifdef CONFIG_PM
5594 int
5595 ahc_suspend(struct ahc_softc *ahc)
5596 {
5597
5598         ahc_pause_and_flushwork(ahc);
5599
5600         if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5601                 ahc_unpause(ahc);
5602                 return (EBUSY);
5603         }
5604
5605 #ifdef AHC_TARGET_MODE
5606         /*
5607          * XXX What about ATIOs that have not yet been serviced?
5608          * Perhaps we should just refuse to be suspended if we
5609          * are acting in a target role.
5610          */
5611         if (ahc->pending_device != NULL) {
5612                 ahc_unpause(ahc);
5613                 return (EBUSY);
5614         }
5615 #endif
5616         ahc_shutdown(ahc);
5617         return (0);
5618 }
5619
5620 int
5621 ahc_resume(struct ahc_softc *ahc)
5622 {
5623
5624         ahc_reset(ahc, /*reinit*/TRUE);
5625         ahc_intr_enable(ahc, TRUE);
5626         ahc_restart(ahc);
5627         return (0);
5628 }
5629 #endif
5630 /************************** Busy Target Table *********************************/
5631 /*
5632  * Return the untagged transaction id for a given target/channel lun.
5633  * Optionally, clear the entry.
5634  */
5635 static u_int
5636 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5637 {
5638         u_int scbid;
5639         u_int target_offset;
5640
5641         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5642                 u_int saved_scbptr;
5643
5644                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5645                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5646                 scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5647                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5648         } else {
5649                 target_offset = TCL_TARGET_OFFSET(tcl);
5650                 scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5651         }
5652
5653         return (scbid);
5654 }
5655
5656 static void
5657 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5658 {
5659         u_int target_offset;
5660
5661         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5662                 u_int saved_scbptr;
5663
5664                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5665                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5666                 ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5667                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5668         } else {
5669                 target_offset = TCL_TARGET_OFFSET(tcl);
5670                 ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5671         }
5672 }
5673
5674 static void
5675 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5676 {
5677         u_int target_offset;
5678
5679         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5680                 u_int saved_scbptr;
5681
5682                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5683                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5684                 ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5685                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5686         } else {
5687                 target_offset = TCL_TARGET_OFFSET(tcl);
5688                 ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5689         }
5690 }
5691
5692 /************************** SCB and SCB queue management **********************/
5693 int
5694 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5695               char channel, int lun, u_int tag, role_t role)
5696 {
5697         int targ = SCB_GET_TARGET(ahc, scb);
5698         char chan = SCB_GET_CHANNEL(ahc, scb);
5699         int slun = SCB_GET_LUN(scb);
5700         int match;
5701
5702         match = ((chan == channel) || (channel == ALL_CHANNELS));
5703         if (match != 0)
5704                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5705         if (match != 0)
5706                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5707         if (match != 0) {
5708 #ifdef AHC_TARGET_MODE
5709                 int group;
5710
5711                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5712                 if (role == ROLE_INITIATOR) {
5713                         match = (group != XPT_FC_GROUP_TMODE)
5714                               && ((tag == scb->hscb->tag)
5715                                || (tag == SCB_LIST_NULL));
5716                 } else if (role == ROLE_TARGET) {
5717                         match = (group == XPT_FC_GROUP_TMODE)
5718                               && ((tag == scb->io_ctx->csio.tag_id)
5719                                || (tag == SCB_LIST_NULL));
5720                 }
5721 #else /* !AHC_TARGET_MODE */
5722                 match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5723 #endif /* AHC_TARGET_MODE */
5724         }
5725
5726         return match;
5727 }
5728
5729 static void
5730 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5731 {
5732         int     target;
5733         char    channel;
5734         int     lun;
5735
5736         target = SCB_GET_TARGET(ahc, scb);
5737         lun = SCB_GET_LUN(scb);
5738         channel = SCB_GET_CHANNEL(ahc, scb);
5739
5740         ahc_search_qinfifo(ahc, target, channel, lun,
5741                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5742                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5743
5744         ahc_platform_freeze_devq(ahc, scb);
5745 }
5746
5747 void
5748 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5749 {
5750         struct scb *prev_scb;
5751
5752         prev_scb = NULL;
5753         if (ahc_qinfifo_count(ahc) != 0) {
5754                 u_int prev_tag;
5755                 uint8_t prev_pos;
5756
5757                 prev_pos = ahc->qinfifonext - 1;
5758                 prev_tag = ahc->qinfifo[prev_pos];
5759                 prev_scb = ahc_lookup_scb(ahc, prev_tag);
5760         }
5761         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5762         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5763                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5764         } else {
5765                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5766         }
5767 }
5768
5769 static void
5770 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5771                     struct scb *scb)
5772 {
5773         if (prev_scb == NULL) {
5774                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5775         } else {
5776                 prev_scb->hscb->next = scb->hscb->tag;
5777                 ahc_sync_scb(ahc, prev_scb,
5778                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5779         }
5780         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5781         scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5782         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5783 }
5784
5785 static int
5786 ahc_qinfifo_count(struct ahc_softc *ahc)
5787 {
5788         uint8_t qinpos;
5789         uint8_t diff;
5790
5791         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5792                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
5793                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
5794         } else
5795                 qinpos = ahc_inb(ahc, QINPOS);
5796         diff = ahc->qinfifonext - qinpos;
5797         return (diff);
5798 }
5799
5800 int
5801 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5802                    int lun, u_int tag, role_t role, uint32_t status,
5803                    ahc_search_action action)
5804 {
5805         struct  scb *scb;
5806         struct  scb *prev_scb;
5807         uint8_t qinstart;
5808         uint8_t qinpos;
5809         uint8_t qintail;
5810         uint8_t next;
5811         uint8_t prev;
5812         uint8_t curscbptr;
5813         int     found;
5814         int     have_qregs;
5815
5816         qintail = ahc->qinfifonext;
5817         have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5818         if (have_qregs) {
5819                 qinstart = ahc_inb(ahc, SNSCB_QOFF);
5820                 ahc_outb(ahc, SNSCB_QOFF, qinstart);
5821         } else
5822                 qinstart = ahc_inb(ahc, QINPOS);
5823         qinpos = qinstart;
5824         found = 0;
5825         prev_scb = NULL;
5826
5827         if (action == SEARCH_COMPLETE) {
5828                 /*
5829                  * Don't attempt to run any queued untagged transactions
5830                  * until we are done with the abort process.
5831                  */
5832                 ahc_freeze_untagged_queues(ahc);
5833         }
5834
5835         /*
5836          * Start with an empty queue.  Entries that are not chosen
5837          * for removal will be re-added to the queue as we go.
5838          */
5839         ahc->qinfifonext = qinpos;
5840         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5841
5842         while (qinpos != qintail) {
5843                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5844                 if (scb == NULL) {
5845                         printk("qinpos = %d, SCB index = %d\n",
5846                                 qinpos, ahc->qinfifo[qinpos]);
5847                         panic("Loop 1\n");
5848                 }
5849
5850                 if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5851                         /*
5852                          * We found an scb that needs to be acted on.
5853                          */
5854                         found++;
5855                         switch (action) {
5856                         case SEARCH_COMPLETE:
5857                         {
5858                                 cam_status ostat;
5859                                 cam_status cstat;
5860
5861                                 ostat = ahc_get_transaction_status(scb);
5862                                 if (ostat == CAM_REQ_INPROG)
5863                                         ahc_set_transaction_status(scb, status);
5864                                 cstat = ahc_get_transaction_status(scb);
5865                                 if (cstat != CAM_REQ_CMP)
5866                                         ahc_freeze_scb(scb);
5867                                 if ((scb->flags & SCB_ACTIVE) == 0)
5868                                         printk("Inactive SCB in qinfifo\n");
5869                                 ahc_done(ahc, scb);
5870
5871                                 /* FALLTHROUGH */
5872                         }
5873                         case SEARCH_REMOVE:
5874                                 break;
5875                         case SEARCH_COUNT:
5876                                 ahc_qinfifo_requeue(ahc, prev_scb, scb);
5877                                 prev_scb = scb;
5878                                 break;
5879                         }
5880                 } else {
5881                         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5882                         prev_scb = scb;
5883                 }
5884                 qinpos++;
5885         }
5886
5887         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5888                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5889         } else {
5890                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5891         }
5892
5893         if (action != SEARCH_COUNT
5894          && (found != 0)
5895          && (qinstart != ahc->qinfifonext)) {
5896                 /*
5897                  * The sequencer may be in the process of dmaing
5898                  * down the SCB at the beginning of the queue.
5899                  * This could be problematic if either the first,
5900                  * or the second SCB is removed from the queue
5901                  * (the first SCB includes a pointer to the "next"
5902                  * SCB to dma). If we have removed any entries, swap
5903                  * the first element in the queue with the next HSCB
5904                  * so the sequencer will notice that NEXT_QUEUED_SCB
5905                  * has changed during its dma attempt and will retry
5906                  * the DMA.
5907                  */
5908                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5909
5910                 if (scb == NULL) {
5911                         printk("found = %d, qinstart = %d, qinfifionext = %d\n",
5912                                 found, qinstart, ahc->qinfifonext);
5913                         panic("First/Second Qinfifo fixup\n");
5914                 }
5915                 /*
5916                  * ahc_swap_with_next_hscb forces our next pointer to
5917                  * point to the reserved SCB for future commands.  Save
5918                  * and restore our original next pointer to maintain
5919                  * queue integrity.
5920                  */
5921                 next = scb->hscb->next;
5922                 ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5923                 ahc_swap_with_next_hscb(ahc, scb);
5924                 scb->hscb->next = next;
5925                 ahc->qinfifo[qinstart] = scb->hscb->tag;
5926
5927                 /* Tell the card about the new head of the qinfifo. */
5928                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5929
5930                 /* Fixup the tail "next" pointer. */
5931                 qintail = ahc->qinfifonext - 1;
5932                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5933                 scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5934         }
5935
5936         /*
5937          * Search waiting for selection list.
5938          */
5939         curscbptr = ahc_inb(ahc, SCBPTR);
5940         next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
5941         prev = SCB_LIST_NULL;
5942
5943         while (next != SCB_LIST_NULL) {
5944                 uint8_t scb_index;
5945
5946                 ahc_outb(ahc, SCBPTR, next);
5947                 scb_index = ahc_inb(ahc, SCB_TAG);
5948                 if (scb_index >= ahc->scb_data->numscbs) {
5949                         printk("Waiting List inconsistency. "
5950                                "SCB index == %d, yet numscbs == %d.",
5951                                scb_index, ahc->scb_data->numscbs);
5952                         ahc_dump_card_state(ahc);
5953                         panic("for safety");
5954                 }
5955                 scb = ahc_lookup_scb(ahc, scb_index);
5956                 if (scb == NULL) {
5957                         printk("scb_index = %d, next = %d\n",
5958                                 scb_index, next);
5959                         panic("Waiting List traversal\n");
5960                 }
5961                 if (ahc_match_scb(ahc, scb, target, channel,
5962                                   lun, SCB_LIST_NULL, role)) {
5963                         /*
5964                          * We found an scb that needs to be acted on.
5965                          */
5966                         found++;
5967                         switch (action) {
5968                         case SEARCH_COMPLETE:
5969                         {
5970                                 cam_status ostat;
5971                                 cam_status cstat;
5972
5973                                 ostat = ahc_get_transaction_status(scb);
5974                                 if (ostat == CAM_REQ_INPROG)
5975                                         ahc_set_transaction_status(scb,
5976                                                                    status);
5977                                 cstat = ahc_get_transaction_status(scb);
5978                                 if (cstat != CAM_REQ_CMP)
5979                                         ahc_freeze_scb(scb);
5980                                 if ((scb->flags & SCB_ACTIVE) == 0)
5981                                         printk("Inactive SCB in Waiting List\n");
5982                                 ahc_done(ahc, scb);
5983                         }
5984                                 fallthrough;
5985                         case SEARCH_REMOVE:
5986                                 next = ahc_rem_wscb(ahc, next, prev);
5987                                 break;
5988                         case SEARCH_COUNT:
5989                                 prev = next;
5990                                 next = ahc_inb(ahc, SCB_NEXT);
5991                                 break;
5992                         }
5993                 } else {
5994
5995                         prev = next;
5996                         next = ahc_inb(ahc, SCB_NEXT);
5997                 }
5998         }
5999         ahc_outb(ahc, SCBPTR, curscbptr);
6000
6001         found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
6002                                             channel, lun, status, action);
6003
6004         if (action == SEARCH_COMPLETE)
6005                 ahc_release_untagged_queues(ahc);
6006         return (found);
6007 }
6008
6009 int
6010 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
6011                            int target, char channel, int lun, uint32_t status,
6012                            ahc_search_action action)
6013 {
6014         struct  scb *scb;
6015         int     maxtarget;
6016         int     found;
6017         int     i;
6018
6019         if (action == SEARCH_COMPLETE) {
6020                 /*
6021                  * Don't attempt to run any queued untagged transactions
6022                  * until we are done with the abort process.
6023                  */
6024                 ahc_freeze_untagged_queues(ahc);
6025         }
6026
6027         found = 0;
6028         i = 0;
6029         if ((ahc->flags & AHC_SCB_BTT) == 0) {
6030
6031                 maxtarget = 16;
6032                 if (target != CAM_TARGET_WILDCARD) {
6033
6034                         i = target;
6035                         if (channel == 'B')
6036                                 i += 8;
6037                         maxtarget = i + 1;
6038                 }
6039         } else {
6040                 maxtarget = 0;
6041         }
6042
6043         for (; i < maxtarget; i++) {
6044                 struct scb_tailq *untagged_q;
6045                 struct scb *next_scb;
6046
6047                 untagged_q = &(ahc->untagged_queues[i]);
6048                 next_scb = TAILQ_FIRST(untagged_q);
6049                 while (next_scb != NULL) {
6050
6051                         scb = next_scb;
6052                         next_scb = TAILQ_NEXT(scb, links.tqe);
6053
6054                         /*
6055                          * The head of the list may be the currently
6056                          * active untagged command for a device.
6057                          * We're only searching for commands that
6058                          * have not been started.  A transaction
6059                          * marked active but still in the qinfifo
6060                          * is removed by the qinfifo scanning code
6061                          * above.
6062                          */
6063                         if ((scb->flags & SCB_ACTIVE) != 0)
6064                                 continue;
6065
6066                         if (ahc_match_scb(ahc, scb, target, channel, lun,
6067                                           SCB_LIST_NULL, ROLE_INITIATOR) == 0
6068                          || (ctx != NULL && ctx != scb->io_ctx))
6069                                 continue;
6070
6071                         /*
6072                          * We found an scb that needs to be acted on.
6073                          */
6074                         found++;
6075                         switch (action) {
6076                         case SEARCH_COMPLETE:
6077                         {
6078                                 cam_status ostat;
6079                                 cam_status cstat;
6080
6081                                 ostat = ahc_get_transaction_status(scb);
6082                                 if (ostat == CAM_REQ_INPROG)
6083                                         ahc_set_transaction_status(scb, status);
6084                                 cstat = ahc_get_transaction_status(scb);
6085                                 if (cstat != CAM_REQ_CMP)
6086                                         ahc_freeze_scb(scb);
6087                                 if ((scb->flags & SCB_ACTIVE) == 0)
6088                                         printk("Inactive SCB in untaggedQ\n");
6089                                 ahc_done(ahc, scb);
6090                                 break;
6091                         }
6092                         case SEARCH_REMOVE:
6093                                 scb->flags &= ~SCB_UNTAGGEDQ;
6094                                 TAILQ_REMOVE(untagged_q, scb, links.tqe);
6095                                 break;
6096                         case SEARCH_COUNT:
6097                                 break;
6098                         }
6099                 }
6100         }
6101
6102         if (action == SEARCH_COMPLETE)
6103                 ahc_release_untagged_queues(ahc);
6104         return (found);
6105 }
6106
6107 int
6108 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
6109                      int lun, u_int tag, int stop_on_first, int remove,
6110                      int save_state)
6111 {
6112         struct  scb *scbp;
6113         u_int   next;
6114         u_int   prev;
6115         u_int   count;
6116         u_int   active_scb;
6117
6118         count = 0;
6119         next = ahc_inb(ahc, DISCONNECTED_SCBH);
6120         prev = SCB_LIST_NULL;
6121
6122         if (save_state) {
6123                 /* restore this when we're done */
6124                 active_scb = ahc_inb(ahc, SCBPTR);
6125         } else
6126                 /* Silence compiler */
6127                 active_scb = SCB_LIST_NULL;
6128
6129         while (next != SCB_LIST_NULL) {
6130                 u_int scb_index;
6131
6132                 ahc_outb(ahc, SCBPTR, next);
6133                 scb_index = ahc_inb(ahc, SCB_TAG);
6134                 if (scb_index >= ahc->scb_data->numscbs) {
6135                         printk("Disconnected List inconsistency. "
6136                                "SCB index == %d, yet numscbs == %d.",
6137                                scb_index, ahc->scb_data->numscbs);
6138                         ahc_dump_card_state(ahc);
6139                         panic("for safety");
6140                 }
6141
6142                 if (next == prev) {
6143                         panic("Disconnected List Loop. "
6144                               "cur SCBPTR == %x, prev SCBPTR == %x.",
6145                               next, prev);
6146                 }
6147                 scbp = ahc_lookup_scb(ahc, scb_index);
6148                 if (ahc_match_scb(ahc, scbp, target, channel, lun,
6149                                   tag, ROLE_INITIATOR)) {
6150                         count++;
6151                         if (remove) {
6152                                 next =
6153                                     ahc_rem_scb_from_disc_list(ahc, prev, next);
6154                         } else {
6155                                 prev = next;
6156                                 next = ahc_inb(ahc, SCB_NEXT);
6157                         }
6158                         if (stop_on_first)
6159                                 break;
6160                 } else {
6161                         prev = next;
6162                         next = ahc_inb(ahc, SCB_NEXT);
6163                 }
6164         }
6165         if (save_state)
6166                 ahc_outb(ahc, SCBPTR, active_scb);
6167         return (count);
6168 }
6169
6170 /*
6171  * Remove an SCB from the on chip list of disconnected transactions.
6172  * This is empty/unused if we are not performing SCB paging.
6173  */
6174 static u_int
6175 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
6176 {
6177         u_int next;
6178
6179         ahc_outb(ahc, SCBPTR, scbptr);
6180         next = ahc_inb(ahc, SCB_NEXT);
6181
6182         ahc_outb(ahc, SCB_CONTROL, 0);
6183
6184         ahc_add_curscb_to_free_list(ahc);
6185
6186         if (prev != SCB_LIST_NULL) {
6187                 ahc_outb(ahc, SCBPTR, prev);
6188                 ahc_outb(ahc, SCB_NEXT, next);
6189         } else
6190                 ahc_outb(ahc, DISCONNECTED_SCBH, next);
6191
6192         return (next);
6193 }
6194
6195 /*
6196  * Add the SCB as selected by SCBPTR onto the on chip list of
6197  * free hardware SCBs.  This list is empty/unused if we are not
6198  * performing SCB paging.
6199  */
6200 static void
6201 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
6202 {
6203         /*
6204          * Invalidate the tag so that our abort
6205          * routines don't think it's active.
6206          */
6207         ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
6208
6209         if ((ahc->flags & AHC_PAGESCBS) != 0) {
6210                 ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
6211                 ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
6212         }
6213 }
6214
6215 /*
6216  * Manipulate the waiting for selection list and return the
6217  * scb that follows the one that we remove.
6218  */
6219 static u_int
6220 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
6221 {
6222         u_int curscb, next;
6223
6224         /*
6225          * Select the SCB we want to abort and
6226          * pull the next pointer out of it.
6227          */
6228         curscb = ahc_inb(ahc, SCBPTR);
6229         ahc_outb(ahc, SCBPTR, scbpos);
6230         next = ahc_inb(ahc, SCB_NEXT);
6231
6232         /* Clear the necessary fields */
6233         ahc_outb(ahc, SCB_CONTROL, 0);
6234
6235         ahc_add_curscb_to_free_list(ahc);
6236
6237         /* update the waiting list */
6238         if (prev == SCB_LIST_NULL) {
6239                 /* First in the list */
6240                 ahc_outb(ahc, WAITING_SCBH, next);
6241
6242                 /*
6243                  * Ensure we aren't attempting to perform
6244                  * selection for this entry.
6245                  */
6246                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
6247         } else {
6248                 /*
6249                  * Select the scb that pointed to us
6250                  * and update its next pointer.
6251                  */
6252                 ahc_outb(ahc, SCBPTR, prev);
6253                 ahc_outb(ahc, SCB_NEXT, next);
6254         }
6255
6256         /*
6257          * Point us back at the original scb position.
6258          */
6259         ahc_outb(ahc, SCBPTR, curscb);
6260         return next;
6261 }
6262
6263 /******************************** Error Handling ******************************/
6264 /*
6265  * Abort all SCBs that match the given description (target/channel/lun/tag),
6266  * setting their status to the passed in status if the status has not already
6267  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
6268  * is paused before it is called.
6269  */
6270 static int
6271 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
6272                int lun, u_int tag, role_t role, uint32_t status)
6273 {
6274         struct  scb *scbp;
6275         struct  scb *scbp_next;
6276         u_int   active_scb;
6277         int     i, j;
6278         int     maxtarget;
6279         int     minlun;
6280         int     maxlun;
6281
6282         int     found;
6283
6284         /*
6285          * Don't attempt to run any queued untagged transactions
6286          * until we are done with the abort process.
6287          */
6288         ahc_freeze_untagged_queues(ahc);
6289
6290         /* restore this when we're done */
6291         active_scb = ahc_inb(ahc, SCBPTR);
6292
6293         found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
6294                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6295
6296         /*
6297          * Clean out the busy target table for any untagged commands.
6298          */
6299         i = 0;
6300         maxtarget = 16;
6301         if (target != CAM_TARGET_WILDCARD) {
6302                 i = target;
6303                 if (channel == 'B')
6304                         i += 8;
6305                 maxtarget = i + 1;
6306         }
6307
6308         if (lun == CAM_LUN_WILDCARD) {
6309
6310                 /*
6311                  * Unless we are using an SCB based
6312                  * busy targets table, there is only
6313                  * one table entry for all luns of
6314                  * a target.
6315                  */
6316                 minlun = 0;
6317                 maxlun = 1;
6318                 if ((ahc->flags & AHC_SCB_BTT) != 0)
6319                         maxlun = AHC_NUM_LUNS;
6320         } else {
6321                 minlun = lun;
6322                 maxlun = lun + 1;
6323         }
6324
6325         if (role != ROLE_TARGET) {
6326                 for (;i < maxtarget; i++) {
6327                         for (j = minlun;j < maxlun; j++) {
6328                                 u_int scbid;
6329                                 u_int tcl;
6330
6331                                 tcl = BUILD_TCL(i << 4, j);
6332                                 scbid = ahc_index_busy_tcl(ahc, tcl);
6333                                 scbp = ahc_lookup_scb(ahc, scbid);
6334                                 if (scbp == NULL
6335                                  || ahc_match_scb(ahc, scbp, target, channel,
6336                                                   lun, tag, role) == 0)
6337                                         continue;
6338                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
6339                         }
6340                 }
6341
6342                 /*
6343                  * Go through the disconnected list and remove any entries we
6344                  * have queued for completion, 0'ing their control byte too.
6345                  * We save the active SCB and restore it ourselves, so there
6346                  * is no reason for this search to restore it too.
6347                  */
6348                 ahc_search_disc_list(ahc, target, channel, lun, tag,
6349                                      /*stop_on_first*/FALSE, /*remove*/TRUE,
6350                                      /*save_state*/FALSE);
6351         }
6352
6353         /*
6354          * Go through the hardware SCB array looking for commands that
6355          * were active but not on any list.  In some cases, these remnants
6356          * might not still have mappings in the scbindex array (e.g. unexpected
6357          * bus free with the same scb queued for an abort).  Don't hold this
6358          * against them.
6359          */
6360         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6361                 u_int scbid;
6362
6363                 ahc_outb(ahc, SCBPTR, i);
6364                 scbid = ahc_inb(ahc, SCB_TAG);
6365                 scbp = ahc_lookup_scb(ahc, scbid);
6366                 if ((scbp == NULL && scbid != SCB_LIST_NULL)
6367                  || (scbp != NULL
6368                   && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
6369                         ahc_add_curscb_to_free_list(ahc);
6370         }
6371
6372         /*
6373          * Go through the pending CCB list and look for
6374          * commands for this target that are still active.
6375          * These are other tagged commands that were
6376          * disconnected when the reset occurred.
6377          */
6378         scbp_next = LIST_FIRST(&ahc->pending_scbs);
6379         while (scbp_next != NULL) {
6380                 scbp = scbp_next;
6381                 scbp_next = LIST_NEXT(scbp, pending_links);
6382                 if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
6383                         cam_status ostat;
6384
6385                         ostat = ahc_get_transaction_status(scbp);
6386                         if (ostat == CAM_REQ_INPROG)
6387                                 ahc_set_transaction_status(scbp, status);
6388                         if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
6389                                 ahc_freeze_scb(scbp);
6390                         if ((scbp->flags & SCB_ACTIVE) == 0)
6391                                 printk("Inactive SCB on pending list\n");
6392                         ahc_done(ahc, scbp);
6393                         found++;
6394                 }
6395         }
6396         ahc_outb(ahc, SCBPTR, active_scb);
6397         ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
6398         ahc_release_untagged_queues(ahc);
6399         return found;
6400 }
6401
6402 static void
6403 ahc_reset_current_bus(struct ahc_softc *ahc)
6404 {
6405         uint8_t scsiseq;
6406
6407         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
6408         scsiseq = ahc_inb(ahc, SCSISEQ);
6409         ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
6410         ahc_flush_device_writes(ahc);
6411         ahc_delay(AHC_BUSRESET_DELAY);
6412         /* Turn off the bus reset */
6413         ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
6414
6415         ahc_clear_intstat(ahc);
6416
6417         /* Re-enable reset interrupts */
6418         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
6419 }
6420
6421 int
6422 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
6423 {
6424         struct  ahc_devinfo devinfo;
6425         u_int   initiator, target, max_scsiid;
6426         u_int   sblkctl;
6427         u_int   scsiseq;
6428         u_int   simode1;
6429         int     found;
6430         int     restart_needed;
6431         char    cur_channel;
6432
6433         ahc->pending_device = NULL;
6434
6435         ahc_compile_devinfo(&devinfo,
6436                             CAM_TARGET_WILDCARD,
6437                             CAM_TARGET_WILDCARD,
6438                             CAM_LUN_WILDCARD,
6439                             channel, ROLE_UNKNOWN);
6440         ahc_pause(ahc);
6441
6442         /* Make sure the sequencer is in a safe location. */
6443         ahc_clear_critical_section(ahc);
6444
6445         /*
6446          * Run our command complete fifos to ensure that we perform
6447          * completion processing on any commands that 'completed'
6448          * before the reset occurred.
6449          */
6450         ahc_run_qoutfifo(ahc);
6451 #ifdef AHC_TARGET_MODE
6452         /*
6453          * XXX - In Twin mode, the tqinfifo may have commands
6454          *       for an unaffected channel in it.  However, if
6455          *       we have run out of ATIO resources to drain that
6456          *       queue, we may not get them all out here.  Further,
6457          *       the blocked transactions for the reset channel
6458          *       should just be killed off, irrespecitve of whether
6459          *       we are blocked on ATIO resources.  Write a routine
6460          *       to compact the tqinfifo appropriately.
6461          */
6462         if ((ahc->flags & AHC_TARGETROLE) != 0) {
6463                 ahc_run_tqinfifo(ahc, /*paused*/TRUE);
6464         }
6465 #endif
6466
6467         /*
6468          * Reset the bus if we are initiating this reset
6469          */
6470         sblkctl = ahc_inb(ahc, SBLKCTL);
6471         cur_channel = 'A';
6472         if ((ahc->features & AHC_TWIN) != 0
6473          && ((sblkctl & SELBUSB) != 0))
6474             cur_channel = 'B';
6475         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
6476         if (cur_channel != channel) {
6477                 /* Case 1: Command for another bus is active
6478                  * Stealthily reset the other bus without
6479                  * upsetting the current bus.
6480                  */
6481                 ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
6482                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6483 #ifdef AHC_TARGET_MODE
6484                 /*
6485                  * Bus resets clear ENSELI, so we cannot
6486                  * defer re-enabling bus reset interrupts
6487                  * if we are in target mode.
6488                  */
6489                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6490                         simode1 |= ENSCSIRST;
6491 #endif
6492                 ahc_outb(ahc, SIMODE1, simode1);
6493                 if (initiate_reset)
6494                         ahc_reset_current_bus(ahc);
6495                 ahc_clear_intstat(ahc);
6496                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6497                 ahc_outb(ahc, SBLKCTL, sblkctl);
6498                 restart_needed = FALSE;
6499         } else {
6500                 /* Case 2: A command from this bus is active or we're idle */
6501                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6502 #ifdef AHC_TARGET_MODE
6503                 /*
6504                  * Bus resets clear ENSELI, so we cannot
6505                  * defer re-enabling bus reset interrupts
6506                  * if we are in target mode.
6507                  */
6508                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6509                         simode1 |= ENSCSIRST;
6510 #endif
6511                 ahc_outb(ahc, SIMODE1, simode1);
6512                 if (initiate_reset)
6513                         ahc_reset_current_bus(ahc);
6514                 ahc_clear_intstat(ahc);
6515                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6516                 restart_needed = TRUE;
6517         }
6518
6519         /*
6520          * Clean up all the state information for the
6521          * pending transactions on this bus.
6522          */
6523         found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6524                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6525                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6526
6527         max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6528
6529 #ifdef AHC_TARGET_MODE
6530         /*
6531          * Send an immediate notify ccb to all target more peripheral
6532          * drivers affected by this action.
6533          */
6534         for (target = 0; target <= max_scsiid; target++) {
6535                 struct ahc_tmode_tstate* tstate;
6536                 u_int lun;
6537
6538                 tstate = ahc->enabled_targets[target];
6539                 if (tstate == NULL)
6540                         continue;
6541                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6542                         struct ahc_tmode_lstate* lstate;
6543
6544                         lstate = tstate->enabled_luns[lun];
6545                         if (lstate == NULL)
6546                                 continue;
6547
6548                         ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6549                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6550                         ahc_send_lstate_events(ahc, lstate);
6551                 }
6552         }
6553 #endif
6554         /* Notify the XPT that a bus reset occurred */
6555         ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6556                        CAM_LUN_WILDCARD, AC_BUS_RESET);
6557
6558         /*
6559          * Revert to async/narrow transfers until we renegotiate.
6560          */
6561         for (target = 0; target <= max_scsiid; target++) {
6562
6563                 if (ahc->enabled_targets[target] == NULL)
6564                         continue;
6565                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6566                         struct ahc_devinfo devinfo;
6567
6568                         ahc_compile_devinfo(&devinfo, target, initiator,
6569                                             CAM_LUN_WILDCARD,
6570                                             channel, ROLE_UNKNOWN);
6571                         ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6572                                       AHC_TRANS_CUR, /*paused*/TRUE);
6573                         ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6574                                          /*period*/0, /*offset*/0,
6575                                          /*ppr_options*/0, AHC_TRANS_CUR,
6576                                          /*paused*/TRUE);
6577                 }
6578         }
6579
6580         if (restart_needed)
6581                 ahc_restart(ahc);
6582         else
6583                 ahc_unpause(ahc);
6584         return found;
6585 }
6586
6587
6588 /***************************** Residual Processing ****************************/
6589 /*
6590  * Calculate the residual for a just completed SCB.
6591  */
6592 static void
6593 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6594 {
6595         struct hardware_scb *hscb;
6596         struct status_pkt *spkt;
6597         uint32_t sgptr;
6598         uint32_t resid_sgptr;
6599         uint32_t resid;
6600
6601         /*
6602          * 5 cases.
6603          * 1) No residual.
6604          *    SG_RESID_VALID clear in sgptr.
6605          * 2) Transferless command
6606          * 3) Never performed any transfers.
6607          *    sgptr has SG_FULL_RESID set.
6608          * 4) No residual but target did not
6609          *    save data pointers after the
6610          *    last transfer, so sgptr was
6611          *    never updated.
6612          * 5) We have a partial residual.
6613          *    Use residual_sgptr to determine
6614          *    where we are.
6615          */
6616
6617         hscb = scb->hscb;
6618         sgptr = ahc_le32toh(hscb->sgptr);
6619         if ((sgptr & SG_RESID_VALID) == 0)
6620                 /* Case 1 */
6621                 return;
6622         sgptr &= ~SG_RESID_VALID;
6623
6624         if ((sgptr & SG_LIST_NULL) != 0)
6625                 /* Case 2 */
6626                 return;
6627
6628         spkt = &hscb->shared_data.status;
6629         resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6630         if ((sgptr & SG_FULL_RESID) != 0) {
6631                 /* Case 3 */
6632                 resid = ahc_get_transfer_length(scb);
6633         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6634                 /* Case 4 */
6635                 return;
6636         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6637                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6638         } else {
6639                 struct ahc_dma_seg *sg;
6640
6641                 /*
6642                  * Remainder of the SG where the transfer
6643                  * stopped.
6644                  */
6645                 resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6646                 sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6647
6648                 /* The residual sg_ptr always points to the next sg */
6649                 sg--;
6650
6651                 /*
6652                  * Add up the contents of all residual
6653                  * SG segments that are after the SG where
6654                  * the transfer stopped.
6655                  */
6656                 while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6657                         sg++;
6658                         resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6659                 }
6660         }
6661         if ((scb->flags & SCB_SENSE) == 0)
6662                 ahc_set_residual(scb, resid);
6663         else
6664                 ahc_set_sense_residual(scb, resid);
6665
6666 #ifdef AHC_DEBUG
6667         if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6668                 ahc_print_path(ahc, scb);
6669                 printk("Handled %sResidual of %d bytes\n",
6670                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6671         }
6672 #endif
6673 }
6674
6675 /******************************* Target Mode **********************************/
6676 #ifdef AHC_TARGET_MODE
6677 /*
6678  * Add a target mode event to this lun's queue
6679  */
6680 static void
6681 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6682                        u_int initiator_id, u_int event_type, u_int event_arg)
6683 {
6684         struct ahc_tmode_event *event;
6685         int pending;
6686
6687         xpt_freeze_devq(lstate->path, /*count*/1);
6688         if (lstate->event_w_idx >= lstate->event_r_idx)
6689                 pending = lstate->event_w_idx - lstate->event_r_idx;
6690         else
6691                 pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6692                         - (lstate->event_r_idx - lstate->event_w_idx);
6693
6694         if (event_type == EVENT_TYPE_BUS_RESET
6695          || event_type == MSG_BUS_DEV_RESET) {
6696                 /*
6697                  * Any earlier events are irrelevant, so reset our buffer.
6698                  * This has the effect of allowing us to deal with reset
6699                  * floods (an external device holding down the reset line)
6700                  * without losing the event that is really interesting.
6701                  */
6702                 lstate->event_r_idx = 0;
6703                 lstate->event_w_idx = 0;
6704                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6705         }
6706
6707         if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6708                 xpt_print_path(lstate->path);
6709                 printk("immediate event %x:%x lost\n",
6710                        lstate->event_buffer[lstate->event_r_idx].event_type,
6711                        lstate->event_buffer[lstate->event_r_idx].event_arg);
6712                 lstate->event_r_idx++;
6713                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6714                         lstate->event_r_idx = 0;
6715                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6716         }
6717
6718         event = &lstate->event_buffer[lstate->event_w_idx];
6719         event->initiator_id = initiator_id;
6720         event->event_type = event_type;
6721         event->event_arg = event_arg;
6722         lstate->event_w_idx++;
6723         if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6724                 lstate->event_w_idx = 0;
6725 }
6726
6727 /*
6728  * Send any target mode events queued up waiting
6729  * for immediate notify resources.
6730  */
6731 void
6732 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6733 {
6734         struct ccb_hdr *ccbh;
6735         struct ccb_immed_notify *inot;
6736
6737         while (lstate->event_r_idx != lstate->event_w_idx
6738             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6739                 struct ahc_tmode_event *event;
6740
6741                 event = &lstate->event_buffer[lstate->event_r_idx];
6742                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6743                 inot = (struct ccb_immed_notify *)ccbh;
6744                 switch (event->event_type) {
6745                 case EVENT_TYPE_BUS_RESET:
6746                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6747                         break;
6748                 default:
6749                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6750                         inot->message_args[0] = event->event_type;
6751                         inot->message_args[1] = event->event_arg;
6752                         break;
6753                 }
6754                 inot->initiator_id = event->initiator_id;
6755                 inot->sense_len = 0;
6756                 xpt_done((union ccb *)inot);
6757                 lstate->event_r_idx++;
6758                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6759                         lstate->event_r_idx = 0;
6760         }
6761 }
6762 #endif
6763
6764 /******************** Sequencer Program Patching/Download *********************/
6765
6766 #ifdef AHC_DUMP_SEQ
6767 void
6768 ahc_dumpseq(struct ahc_softc* ahc)
6769 {
6770         int i;
6771
6772         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6773         ahc_outb(ahc, SEQADDR0, 0);
6774         ahc_outb(ahc, SEQADDR1, 0);
6775         for (i = 0; i < ahc->instruction_ram_size; i++) {
6776                 uint8_t ins_bytes[4];
6777
6778                 ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6779                 printk("0x%08x\n", ins_bytes[0] << 24
6780                                  | ins_bytes[1] << 16
6781                                  | ins_bytes[2] << 8
6782                                  | ins_bytes[3]);
6783         }
6784 }
6785 #endif
6786
6787 static int
6788 ahc_loadseq(struct ahc_softc *ahc)
6789 {
6790         struct  cs cs_table[NUM_CRITICAL_SECTIONS];
6791         u_int   begin_set[NUM_CRITICAL_SECTIONS];
6792         u_int   end_set[NUM_CRITICAL_SECTIONS];
6793         const struct patch *cur_patch;
6794         u_int   cs_count;
6795         u_int   cur_cs;
6796         u_int   i;
6797         u_int   skip_addr;
6798         u_int   sg_prefetch_cnt;
6799         int     downloaded;
6800         uint8_t download_consts[7];
6801
6802         /*
6803          * Start out with 0 critical sections
6804          * that apply to this firmware load.
6805          */
6806         cs_count = 0;
6807         cur_cs = 0;
6808         memset(begin_set, 0, sizeof(begin_set));
6809         memset(end_set, 0, sizeof(end_set));
6810
6811         /* Setup downloadable constant table */
6812         download_consts[QOUTFIFO_OFFSET] = 0;
6813         if (ahc->targetcmds != NULL)
6814                 download_consts[QOUTFIFO_OFFSET] += 32;
6815         download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6816         download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6817         download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6818         sg_prefetch_cnt = ahc->pci_cachesize;
6819         if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6820                 sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6821         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6822         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6823         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6824
6825         cur_patch = patches;
6826         downloaded = 0;
6827         skip_addr = 0;
6828         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6829         ahc_outb(ahc, SEQADDR0, 0);
6830         ahc_outb(ahc, SEQADDR1, 0);
6831
6832         for (i = 0; i < sizeof(seqprog)/4; i++) {
6833                 if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6834                         /*
6835                          * Don't download this instruction as it
6836                          * is in a patch that was removed.
6837                          */
6838                         continue;
6839                 }
6840
6841                 if (downloaded == ahc->instruction_ram_size) {
6842                         /*
6843                          * We're about to exceed the instruction
6844                          * storage capacity for this chip.  Fail
6845                          * the load.
6846                          */
6847                         printk("\n%s: Program too large for instruction memory "
6848                                "size of %d!\n", ahc_name(ahc),
6849                                ahc->instruction_ram_size);
6850                         return (ENOMEM);
6851                 }
6852
6853                 /*
6854                  * Move through the CS table until we find a CS
6855                  * that might apply to this instruction.
6856                  */
6857                 for (; cur_cs < NUM_CRITICAL_SECTIONS; cur_cs++) {
6858                         if (critical_sections[cur_cs].end <= i) {
6859                                 if (begin_set[cs_count] == TRUE
6860                                  && end_set[cs_count] == FALSE) {
6861                                         cs_table[cs_count].end = downloaded;
6862                                         end_set[cs_count] = TRUE;
6863                                         cs_count++;
6864                                 }
6865                                 continue;
6866                         }
6867                         if (critical_sections[cur_cs].begin <= i
6868                          && begin_set[cs_count] == FALSE) {
6869                                 cs_table[cs_count].begin = downloaded;
6870                                 begin_set[cs_count] = TRUE;
6871                         }
6872                         break;
6873                 }
6874                 ahc_download_instr(ahc, i, download_consts);
6875                 downloaded++;
6876         }
6877
6878         ahc->num_critical_sections = cs_count;
6879         if (cs_count != 0) {
6880
6881                 cs_count *= sizeof(struct cs);
6882                 ahc->critical_sections = kmalloc(cs_count, GFP_ATOMIC);
6883                 if (ahc->critical_sections == NULL)
6884                         panic("ahc_loadseq: Could not malloc");
6885                 memcpy(ahc->critical_sections, cs_table, cs_count);
6886         }
6887         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6888
6889         if (bootverbose) {
6890                 printk(" %d instructions downloaded\n", downloaded);
6891                 printk("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6892                        ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6893         }
6894         return (0);
6895 }
6896
6897 static int
6898 ahc_check_patch(struct ahc_softc *ahc, const struct patch **start_patch,
6899                 u_int start_instr, u_int *skip_addr)
6900 {
6901         const struct patch *cur_patch;
6902         const struct patch *last_patch;
6903         u_int   num_patches;
6904
6905         num_patches = ARRAY_SIZE(patches);
6906         last_patch = &patches[num_patches];
6907         cur_patch = *start_patch;
6908
6909         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6910
6911                 if (cur_patch->patch_func(ahc) == 0) {
6912
6913                         /* Start rejecting code */
6914                         *skip_addr = start_instr + cur_patch->skip_instr;
6915                         cur_patch += cur_patch->skip_patch;
6916                 } else {
6917                         /* Accepted this patch.  Advance to the next
6918                          * one and wait for our intruction pointer to
6919                          * hit this point.
6920                          */
6921                         cur_patch++;
6922                 }
6923         }
6924
6925         *start_patch = cur_patch;
6926         if (start_instr < *skip_addr)
6927                 /* Still skipping */
6928                 return (0);
6929
6930         return (1);
6931 }
6932
6933 static void
6934 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6935 {
6936         union   ins_formats instr;
6937         struct  ins_format1 *fmt1_ins;
6938         struct  ins_format3 *fmt3_ins;
6939         u_int   opcode;
6940
6941         /*
6942          * The firmware is always compiled into a little endian format.
6943          */
6944         instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
6945
6946         fmt1_ins = &instr.format1;
6947         fmt3_ins = NULL;
6948
6949         /* Pull the opcode */
6950         opcode = instr.format1.opcode;
6951         switch (opcode) {
6952         case AIC_OP_JMP:
6953         case AIC_OP_JC:
6954         case AIC_OP_JNC:
6955         case AIC_OP_CALL:
6956         case AIC_OP_JNE:
6957         case AIC_OP_JNZ:
6958         case AIC_OP_JE:
6959         case AIC_OP_JZ:
6960         {
6961                 const struct patch *cur_patch;
6962                 int address_offset;
6963                 u_int address;
6964                 u_int skip_addr;
6965                 u_int i;
6966
6967                 fmt3_ins = &instr.format3;
6968                 address_offset = 0;
6969                 address = fmt3_ins->address;
6970                 cur_patch = patches;
6971                 skip_addr = 0;
6972
6973                 for (i = 0; i < address;) {
6974
6975                         ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
6976
6977                         if (skip_addr > i) {
6978                                 int end_addr;
6979
6980                                 end_addr = min(address, skip_addr);
6981                                 address_offset += end_addr - i;
6982                                 i = skip_addr;
6983                         } else {
6984                                 i++;
6985                         }
6986                 }
6987                 address -= address_offset;
6988                 fmt3_ins->address = address;
6989         }
6990                 fallthrough;
6991         case AIC_OP_OR:
6992         case AIC_OP_AND:
6993         case AIC_OP_XOR:
6994         case AIC_OP_ADD:
6995         case AIC_OP_ADC:
6996         case AIC_OP_BMOV:
6997                 if (fmt1_ins->parity != 0) {
6998                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
6999                 }
7000                 fmt1_ins->parity = 0;
7001                 if ((ahc->features & AHC_CMD_CHAN) == 0
7002                  && opcode == AIC_OP_BMOV) {
7003                         /*
7004                          * Block move was added at the same time
7005                          * as the command channel.  Verify that
7006                          * this is only a move of a single element
7007                          * and convert the BMOV to a MOV
7008                          * (AND with an immediate of FF).
7009                          */
7010                         if (fmt1_ins->immediate != 1)
7011                                 panic("%s: BMOV not supported\n",
7012                                       ahc_name(ahc));
7013                         fmt1_ins->opcode = AIC_OP_AND;
7014                         fmt1_ins->immediate = 0xff;
7015                 }
7016                 fallthrough;
7017         case AIC_OP_ROL:
7018                 if ((ahc->features & AHC_ULTRA2) != 0) {
7019                         int i, count;
7020
7021                         /* Calculate odd parity for the instruction */
7022                         for (i = 0, count = 0; i < 31; i++) {
7023                                 uint32_t mask;
7024
7025                                 mask = 0x01 << i;
7026                                 if ((instr.integer & mask) != 0)
7027                                         count++;
7028                         }
7029                         if ((count & 0x01) == 0)
7030                                 instr.format1.parity = 1;
7031                 } else {
7032                         /* Compress the instruction for older sequencers */
7033                         if (fmt3_ins != NULL) {
7034                                 instr.integer =
7035                                         fmt3_ins->immediate
7036                                       | (fmt3_ins->source << 8)
7037                                       | (fmt3_ins->address << 16)
7038                                       | (fmt3_ins->opcode << 25);
7039                         } else {
7040                                 instr.integer =
7041                                         fmt1_ins->immediate
7042                                       | (fmt1_ins->source << 8)
7043                                       | (fmt1_ins->destination << 16)
7044                                       | (fmt1_ins->ret << 24)
7045                                       | (fmt1_ins->opcode << 25);
7046                         }
7047                 }
7048                 /* The sequencer is a little endian cpu */
7049                 instr.integer = ahc_htole32(instr.integer);
7050                 ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
7051                 break;
7052         default:
7053                 panic("Unknown opcode encountered in seq program");
7054                 break;
7055         }
7056 }
7057
7058 int
7059 ahc_print_register(const ahc_reg_parse_entry_t *table, u_int num_entries,
7060                    const char *name, u_int address, u_int value,
7061                    u_int *cur_column, u_int wrap_point)
7062 {
7063         int     printed;
7064         u_int   printed_mask;
7065
7066         if (cur_column != NULL && *cur_column >= wrap_point) {
7067                 printk("\n");
7068                 *cur_column = 0;
7069         }
7070         printed  = printk("%s[0x%x]", name, value);
7071         if (table == NULL) {
7072                 printed += printk(" ");
7073                 *cur_column += printed;
7074                 return (printed);
7075         }
7076         printed_mask = 0;
7077         while (printed_mask != 0xFF) {
7078                 int entry;
7079
7080                 for (entry = 0; entry < num_entries; entry++) {
7081                         if (((value & table[entry].mask)
7082                           != table[entry].value)
7083                          || ((printed_mask & table[entry].mask)
7084                           == table[entry].mask))
7085                                 continue;
7086
7087                         printed += printk("%s%s",
7088                                           printed_mask == 0 ? ":(" : "|",
7089                                           table[entry].name);
7090                         printed_mask |= table[entry].mask;
7091
7092                         break;
7093                 }
7094                 if (entry >= num_entries)
7095                         break;
7096         }
7097         if (printed_mask != 0)
7098                 printed += printk(") ");
7099         else
7100                 printed += printk(" ");
7101         if (cur_column != NULL)
7102                 *cur_column += printed;
7103         return (printed);
7104 }
7105
7106 void
7107 ahc_dump_card_state(struct ahc_softc *ahc)
7108 {
7109         struct  scb *scb;
7110         struct  scb_tailq *untagged_q;
7111         u_int   cur_col;
7112         int     paused;
7113         int     target;
7114         int     maxtarget;
7115         int     i;
7116         uint8_t last_phase;
7117         uint8_t qinpos;
7118         uint8_t qintail;
7119         uint8_t qoutpos;
7120         uint8_t scb_index;
7121         uint8_t saved_scbptr;
7122
7123         if (ahc_is_paused(ahc)) {
7124                 paused = 1;
7125         } else {
7126                 paused = 0;
7127                 ahc_pause(ahc);
7128         }
7129
7130         saved_scbptr = ahc_inb(ahc, SCBPTR);
7131         last_phase = ahc_inb(ahc, LASTPHASE);
7132         printk(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
7133                "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
7134                ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
7135                ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
7136         if (paused)
7137                 printk("Card was paused\n");
7138         printk("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
7139                ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
7140                ahc_inb(ahc, ARG_2));
7141         printk("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
7142                ahc_inb(ahc, SCBPTR));
7143         cur_col = 0;
7144         if ((ahc->features & AHC_DT) != 0)
7145                 ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
7146         ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
7147         ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
7148         ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
7149         ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
7150         ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
7151         ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
7152         ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
7153         ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
7154         ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
7155         ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
7156         ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
7157         ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
7158         ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
7159         ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
7160         ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
7161         ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
7162         ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
7163         ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
7164         if (cur_col != 0)
7165                 printk("\n");
7166         printk("STACK:");
7167         for (i = 0; i < STACK_SIZE; i++)
7168                 printk(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
7169         printk("\nSCB count = %d\n", ahc->scb_data->numscbs);
7170         printk("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
7171         printk("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
7172         /* QINFIFO */
7173         printk("QINFIFO entries: ");
7174         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
7175                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
7176                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
7177         } else
7178                 qinpos = ahc_inb(ahc, QINPOS);
7179         qintail = ahc->qinfifonext;
7180         while (qinpos != qintail) {
7181                 printk("%d ", ahc->qinfifo[qinpos]);
7182                 qinpos++;
7183         }
7184         printk("\n");
7185
7186         printk("Waiting Queue entries: ");
7187         scb_index = ahc_inb(ahc, WAITING_SCBH);
7188         i = 0;
7189         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7190                 ahc_outb(ahc, SCBPTR, scb_index);
7191                 printk("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7192                 scb_index = ahc_inb(ahc, SCB_NEXT);
7193         }
7194         printk("\n");
7195
7196         printk("Disconnected Queue entries: ");
7197         scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
7198         i = 0;
7199         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7200                 ahc_outb(ahc, SCBPTR, scb_index);
7201                 printk("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7202                 scb_index = ahc_inb(ahc, SCB_NEXT);
7203         }
7204         printk("\n");
7205
7206         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
7207         printk("QOUTFIFO entries: ");
7208         qoutpos = ahc->qoutfifonext;
7209         i = 0;
7210         while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
7211                 printk("%d ", ahc->qoutfifo[qoutpos]);
7212                 qoutpos++;
7213         }
7214         printk("\n");
7215
7216         printk("Sequencer Free SCB List: ");
7217         scb_index = ahc_inb(ahc, FREE_SCBH);
7218         i = 0;
7219         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7220                 ahc_outb(ahc, SCBPTR, scb_index);
7221                 printk("%d ", scb_index);
7222                 scb_index = ahc_inb(ahc, SCB_NEXT);
7223         }
7224         printk("\n");
7225
7226         printk("Sequencer SCB Info: ");
7227         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
7228                 ahc_outb(ahc, SCBPTR, i);
7229                 cur_col  = printk("\n%3d ", i);
7230
7231                 ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
7232                 ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
7233                 ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
7234                 ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7235         }
7236         printk("\n");
7237
7238         printk("Pending list: ");
7239         i = 0;
7240         LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7241                 if (i++ > 256)
7242                         break;
7243                 cur_col  = printk("\n%3d ", scb->hscb->tag);
7244                 ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
7245                 ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
7246                 ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
7247                 if ((ahc->flags & AHC_PAGESCBS) == 0) {
7248                         ahc_outb(ahc, SCBPTR, scb->hscb->tag);
7249                         printk("(");
7250                         ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
7251                                               &cur_col, 60);
7252                         ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7253                         printk(")");
7254                 }
7255         }
7256         printk("\n");
7257
7258         printk("Kernel Free SCB list: ");
7259         i = 0;
7260         SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
7261                 if (i++ > 256)
7262                         break;
7263                 printk("%d ", scb->hscb->tag);
7264         }
7265         printk("\n");
7266
7267         maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
7268         for (target = 0; target <= maxtarget; target++) {
7269                 untagged_q = &ahc->untagged_queues[target];
7270                 if (TAILQ_FIRST(untagged_q) == NULL)
7271                         continue;
7272                 printk("Untagged Q(%d): ", target);
7273                 i = 0;
7274                 TAILQ_FOREACH(scb, untagged_q, links.tqe) {
7275                         if (i++ > 256)
7276                                 break;
7277                         printk("%d ", scb->hscb->tag);
7278                 }
7279                 printk("\n");
7280         }
7281
7282         printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
7283         ahc_outb(ahc, SCBPTR, saved_scbptr);
7284         if (paused == 0)
7285                 ahc_unpause(ahc);
7286 }
7287
7288 /************************* Target Mode ****************************************/
7289 #ifdef AHC_TARGET_MODE
7290 cam_status
7291 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
7292                     struct ahc_tmode_tstate **tstate,
7293                     struct ahc_tmode_lstate **lstate,
7294                     int notfound_failure)
7295 {
7296
7297         if ((ahc->features & AHC_TARGETMODE) == 0)
7298                 return (CAM_REQ_INVALID);
7299
7300         /*
7301          * Handle the 'black hole' device that sucks up
7302          * requests to unattached luns on enabled targets.
7303          */
7304         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
7305          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
7306                 *tstate = NULL;
7307                 *lstate = ahc->black_hole;
7308         } else {
7309                 u_int max_id;
7310
7311                 max_id = (ahc->features & AHC_WIDE) ? 16 : 8;
7312                 if (ccb->ccb_h.target_id >= max_id)
7313                         return (CAM_TID_INVALID);
7314
7315                 if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
7316                         return (CAM_LUN_INVALID);
7317
7318                 *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
7319                 *lstate = NULL;
7320                 if (*tstate != NULL)
7321                         *lstate =
7322                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
7323         }
7324
7325         if (notfound_failure != 0 && *lstate == NULL)
7326                 return (CAM_PATH_INVALID);
7327
7328         return (CAM_REQ_CMP);
7329 }
7330
7331 void
7332 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
7333 {
7334         struct     ahc_tmode_tstate *tstate;
7335         struct     ahc_tmode_lstate *lstate;
7336         struct     ccb_en_lun *cel;
7337         cam_status status;
7338         u_long     s;
7339         u_int      target;
7340         u_int      lun;
7341         u_int      target_mask;
7342         u_int      our_id;
7343         int        error;
7344         char       channel;
7345
7346         status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
7347                                      /*notfound_failure*/FALSE);
7348
7349         if (status != CAM_REQ_CMP) {
7350                 ccb->ccb_h.status = status;
7351                 return;
7352         }
7353
7354         if (cam_sim_bus(sim) == 0)
7355                 our_id = ahc->our_id;
7356         else
7357                 our_id = ahc->our_id_b;
7358
7359         if (ccb->ccb_h.target_id != our_id) {
7360                 /*
7361                  * our_id represents our initiator ID, or
7362                  * the ID of the first target to have an
7363                  * enabled lun in target mode.  There are
7364                  * two cases that may preclude enabling a
7365                  * target id other than our_id.
7366                  *
7367                  *   o our_id is for an active initiator role.
7368                  *     Since the hardware does not support
7369                  *     reselections to the initiator role at
7370                  *     anything other than our_id, and our_id
7371                  *     is used by the hardware to indicate the
7372                  *     ID to use for both select-out and
7373                  *     reselect-out operations, the only target
7374                  *     ID we can support in this mode is our_id.
7375                  *
7376                  *   o The MULTARGID feature is not available and
7377                  *     a previous target mode ID has been enabled.
7378                  */
7379                 if ((ahc->features & AHC_MULTIROLE) != 0) {
7380
7381                         if ((ahc->features & AHC_MULTI_TID) != 0
7382                          && (ahc->flags & AHC_INITIATORROLE) != 0) {
7383                                 /*
7384                                  * Only allow additional targets if
7385                                  * the initiator role is disabled.
7386                                  * The hardware cannot handle a re-select-in
7387                                  * on the initiator id during a re-select-out
7388                                  * on a different target id.
7389                                  */
7390                                 status = CAM_TID_INVALID;
7391                         } else if ((ahc->flags & AHC_INITIATORROLE) != 0
7392                                 || ahc->enabled_luns > 0) {
7393                                 /*
7394                                  * Only allow our target id to change
7395                                  * if the initiator role is not configured
7396                                  * and there are no enabled luns which
7397                                  * are attached to the currently registered
7398                                  * scsi id.
7399                                  */
7400                                 status = CAM_TID_INVALID;
7401                         }
7402                 } else if ((ahc->features & AHC_MULTI_TID) == 0
7403                         && ahc->enabled_luns > 0) {
7404
7405                         status = CAM_TID_INVALID;
7406                 }
7407         }
7408
7409         if (status != CAM_REQ_CMP) {
7410                 ccb->ccb_h.status = status;
7411                 return;
7412         }
7413
7414         /*
7415          * We now have an id that is valid.
7416          * If we aren't in target mode, switch modes.
7417          */
7418         if ((ahc->flags & AHC_TARGETROLE) == 0
7419          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
7420                 u_long   s;
7421                 ahc_flag saved_flags;
7422
7423                 printk("Configuring Target Mode\n");
7424                 ahc_lock(ahc, &s);
7425                 if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
7426                         ccb->ccb_h.status = CAM_BUSY;
7427                         ahc_unlock(ahc, &s);
7428                         return;
7429                 }
7430                 saved_flags = ahc->flags;
7431                 ahc->flags |= AHC_TARGETROLE;
7432                 if ((ahc->features & AHC_MULTIROLE) == 0)
7433                         ahc->flags &= ~AHC_INITIATORROLE;
7434                 ahc_pause(ahc);
7435                 error = ahc_loadseq(ahc);
7436                 if (error != 0) {
7437                         /*
7438                          * Restore original configuration and notify
7439                          * the caller that we cannot support target mode.
7440                          * Since the adapter started out in this
7441                          * configuration, the firmware load will succeed,
7442                          * so there is no point in checking ahc_loadseq's
7443                          * return value.
7444                          */
7445                         ahc->flags = saved_flags;
7446                         (void)ahc_loadseq(ahc);
7447                         ahc_restart(ahc);
7448                         ahc_unlock(ahc, &s);
7449                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
7450                         return;
7451                 }
7452                 ahc_restart(ahc);
7453                 ahc_unlock(ahc, &s);
7454         }
7455         cel = &ccb->cel;
7456         target = ccb->ccb_h.target_id;
7457         lun = ccb->ccb_h.target_lun;
7458         channel = SIM_CHANNEL(ahc, sim);
7459         target_mask = 0x01 << target;
7460         if (channel == 'B')
7461                 target_mask <<= 8;
7462
7463         if (cel->enable != 0) {
7464                 u_int scsiseq;
7465
7466                 /* Are we already enabled?? */
7467                 if (lstate != NULL) {
7468                         xpt_print_path(ccb->ccb_h.path);
7469                         printk("Lun already enabled\n");
7470                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
7471                         return;
7472                 }
7473
7474                 if (cel->grp6_len != 0
7475                  || cel->grp7_len != 0) {
7476                         /*
7477                          * Don't (yet?) support vendor
7478                          * specific commands.
7479                          */
7480                         ccb->ccb_h.status = CAM_REQ_INVALID;
7481                         printk("Non-zero Group Codes\n");
7482                         return;
7483                 }
7484
7485                 /*
7486                  * Seems to be okay.
7487                  * Setup our data structures.
7488                  */
7489                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
7490                         tstate = ahc_alloc_tstate(ahc, target, channel);
7491                         if (tstate == NULL) {
7492                                 xpt_print_path(ccb->ccb_h.path);
7493                                 printk("Couldn't allocate tstate\n");
7494                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7495                                 return;
7496                         }
7497                 }
7498                 lstate = kzalloc(sizeof(*lstate), GFP_ATOMIC);
7499                 if (lstate == NULL) {
7500                         xpt_print_path(ccb->ccb_h.path);
7501                         printk("Couldn't allocate lstate\n");
7502                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7503                         return;
7504                 }
7505                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
7506                                          xpt_path_path_id(ccb->ccb_h.path),
7507                                          xpt_path_target_id(ccb->ccb_h.path),
7508                                          xpt_path_lun_id(ccb->ccb_h.path));
7509                 if (status != CAM_REQ_CMP) {
7510                         kfree(lstate);
7511                         xpt_print_path(ccb->ccb_h.path);
7512                         printk("Couldn't allocate path\n");
7513                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7514                         return;
7515                 }
7516                 SLIST_INIT(&lstate->accept_tios);
7517                 SLIST_INIT(&lstate->immed_notifies);
7518                 ahc_lock(ahc, &s);
7519                 ahc_pause(ahc);
7520                 if (target != CAM_TARGET_WILDCARD) {
7521                         tstate->enabled_luns[lun] = lstate;
7522                         ahc->enabled_luns++;
7523
7524                         if ((ahc->features & AHC_MULTI_TID) != 0) {
7525                                 u_int targid_mask;
7526
7527                                 targid_mask = ahc_inb(ahc, TARGID)
7528                                             | (ahc_inb(ahc, TARGID + 1) << 8);
7529
7530                                 targid_mask |= target_mask;
7531                                 ahc_outb(ahc, TARGID, targid_mask);
7532                                 ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7533
7534                                 ahc_update_scsiid(ahc, targid_mask);
7535                         } else {
7536                                 u_int our_id;
7537                                 char  channel;
7538
7539                                 channel = SIM_CHANNEL(ahc, sim);
7540                                 our_id = SIM_SCSI_ID(ahc, sim);
7541
7542                                 /*
7543                                  * This can only happen if selections
7544                                  * are not enabled
7545                                  */
7546                                 if (target != our_id) {
7547                                         u_int sblkctl;
7548                                         char  cur_channel;
7549                                         int   swap;
7550
7551                                         sblkctl = ahc_inb(ahc, SBLKCTL);
7552                                         cur_channel = (sblkctl & SELBUSB)
7553                                                     ? 'B' : 'A';
7554                                         if ((ahc->features & AHC_TWIN) == 0)
7555                                                 cur_channel = 'A';
7556                                         swap = cur_channel != channel;
7557                                         if (channel == 'A')
7558                                                 ahc->our_id = target;
7559                                         else
7560                                                 ahc->our_id_b = target;
7561
7562                                         if (swap)
7563                                                 ahc_outb(ahc, SBLKCTL,
7564                                                          sblkctl ^ SELBUSB);
7565
7566                                         ahc_outb(ahc, SCSIID, target);
7567
7568                                         if (swap)
7569                                                 ahc_outb(ahc, SBLKCTL, sblkctl);
7570                                 }
7571                         }
7572                 } else
7573                         ahc->black_hole = lstate;
7574                 /* Allow select-in operations */
7575                 if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7576                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7577                         scsiseq |= ENSELI;
7578                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7579                         scsiseq = ahc_inb(ahc, SCSISEQ);
7580                         scsiseq |= ENSELI;
7581                         ahc_outb(ahc, SCSISEQ, scsiseq);
7582                 }
7583                 ahc_unpause(ahc);
7584                 ahc_unlock(ahc, &s);
7585                 ccb->ccb_h.status = CAM_REQ_CMP;
7586                 xpt_print_path(ccb->ccb_h.path);
7587                 printk("Lun now enabled for target mode\n");
7588         } else {
7589                 struct scb *scb;
7590                 int i, empty;
7591
7592                 if (lstate == NULL) {
7593                         ccb->ccb_h.status = CAM_LUN_INVALID;
7594                         return;
7595                 }
7596
7597                 ahc_lock(ahc, &s);
7598
7599                 ccb->ccb_h.status = CAM_REQ_CMP;
7600                 LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7601                         struct ccb_hdr *ccbh;
7602
7603                         ccbh = &scb->io_ctx->ccb_h;
7604                         if (ccbh->func_code == XPT_CONT_TARGET_IO
7605                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7606                                 printk("CTIO pending\n");
7607                                 ccb->ccb_h.status = CAM_REQ_INVALID;
7608                                 ahc_unlock(ahc, &s);
7609                                 return;
7610                         }
7611                 }
7612
7613                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7614                         printk("ATIOs pending\n");
7615                         ccb->ccb_h.status = CAM_REQ_INVALID;
7616                 }
7617
7618                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7619                         printk("INOTs pending\n");
7620                         ccb->ccb_h.status = CAM_REQ_INVALID;
7621                 }
7622
7623                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
7624                         ahc_unlock(ahc, &s);
7625                         return;
7626                 }
7627
7628                 xpt_print_path(ccb->ccb_h.path);
7629                 printk("Target mode disabled\n");
7630                 xpt_free_path(lstate->path);
7631                 kfree(lstate);
7632
7633                 ahc_pause(ahc);
7634                 /* Can we clean up the target too? */
7635                 if (target != CAM_TARGET_WILDCARD) {
7636                         tstate->enabled_luns[lun] = NULL;
7637                         ahc->enabled_luns--;
7638                         for (empty = 1, i = 0; i < 8; i++)
7639                                 if (tstate->enabled_luns[i] != NULL) {
7640                                         empty = 0;
7641                                         break;
7642                                 }
7643
7644                         if (empty) {
7645                                 ahc_free_tstate(ahc, target, channel,
7646                                                 /*force*/FALSE);
7647                                 if (ahc->features & AHC_MULTI_TID) {
7648                                         u_int targid_mask;
7649
7650                                         targid_mask = ahc_inb(ahc, TARGID)
7651                                                     | (ahc_inb(ahc, TARGID + 1)
7652                                                        << 8);
7653
7654                                         targid_mask &= ~target_mask;
7655                                         ahc_outb(ahc, TARGID, targid_mask);
7656                                         ahc_outb(ahc, TARGID+1,
7657                                                  (targid_mask >> 8));
7658                                         ahc_update_scsiid(ahc, targid_mask);
7659                                 }
7660                         }
7661                 } else {
7662
7663                         ahc->black_hole = NULL;
7664
7665                         /*
7666                          * We can't allow selections without
7667                          * our black hole device.
7668                          */
7669                         empty = TRUE;
7670                 }
7671                 if (ahc->enabled_luns == 0) {
7672                         /* Disallow select-in */
7673                         u_int scsiseq;
7674
7675                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7676                         scsiseq &= ~ENSELI;
7677                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7678                         scsiseq = ahc_inb(ahc, SCSISEQ);
7679                         scsiseq &= ~ENSELI;
7680                         ahc_outb(ahc, SCSISEQ, scsiseq);
7681
7682                         if ((ahc->features & AHC_MULTIROLE) == 0) {
7683                                 printk("Configuring Initiator Mode\n");
7684                                 ahc->flags &= ~AHC_TARGETROLE;
7685                                 ahc->flags |= AHC_INITIATORROLE;
7686                                 /*
7687                                  * Returning to a configuration that
7688                                  * fit previously will always succeed.
7689                                  */
7690                                 (void)ahc_loadseq(ahc);
7691                                 ahc_restart(ahc);
7692                                 /*
7693                                  * Unpaused.  The extra unpause
7694                                  * that follows is harmless.
7695                                  */
7696                         }
7697                 }
7698                 ahc_unpause(ahc);
7699                 ahc_unlock(ahc, &s);
7700         }
7701 }
7702
7703 static void
7704 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7705 {
7706         u_int scsiid_mask;
7707         u_int scsiid;
7708
7709         if ((ahc->features & AHC_MULTI_TID) == 0)
7710                 panic("ahc_update_scsiid called on non-multitid unit\n");
7711
7712         /*
7713          * Since we will rely on the TARGID mask
7714          * for selection enables, ensure that OID
7715          * in SCSIID is not set to some other ID
7716          * that we don't want to allow selections on.
7717          */
7718         if ((ahc->features & AHC_ULTRA2) != 0)
7719                 scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7720         else
7721                 scsiid = ahc_inb(ahc, SCSIID);
7722         scsiid_mask = 0x1 << (scsiid & OID);
7723         if ((targid_mask & scsiid_mask) == 0) {
7724                 u_int our_id;
7725
7726                 /* ffs counts from 1 */
7727                 our_id = ffs(targid_mask);
7728                 if (our_id == 0)
7729                         our_id = ahc->our_id;
7730                 else
7731                         our_id--;
7732                 scsiid &= TID;
7733                 scsiid |= our_id;
7734         }
7735         if ((ahc->features & AHC_ULTRA2) != 0)
7736                 ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7737         else
7738                 ahc_outb(ahc, SCSIID, scsiid);
7739 }
7740
7741 static void
7742 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7743 {
7744         struct target_cmd *cmd;
7745
7746         /*
7747          * If the card supports auto-access pause,
7748          * we can access the card directly regardless
7749          * of whether it is paused or not.
7750          */
7751         if ((ahc->features & AHC_AUTOPAUSE) != 0)
7752                 paused = TRUE;
7753
7754         ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7755         while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7756
7757                 /*
7758                  * Only advance through the queue if we
7759                  * have the resources to process the command.
7760                  */
7761                 if (ahc_handle_target_cmd(ahc, cmd) != 0)
7762                         break;
7763
7764                 cmd->cmd_valid = 0;
7765                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7766                                 ahc->shared_data_dmamap,
7767                                 ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7768                                 sizeof(struct target_cmd),
7769                                 BUS_DMASYNC_PREREAD);
7770                 ahc->tqinfifonext++;
7771
7772                 /*
7773                  * Lazily update our position in the target mode incoming
7774                  * command queue as seen by the sequencer.
7775                  */
7776                 if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7777                         if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7778                                 u_int hs_mailbox;
7779
7780                                 hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7781                                 hs_mailbox &= ~HOST_TQINPOS;
7782                                 hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7783                                 ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7784                         } else {
7785                                 if (!paused)
7786                                         ahc_pause(ahc);
7787                                 ahc_outb(ahc, KERNEL_TQINPOS,
7788                                          ahc->tqinfifonext & HOST_TQINPOS);
7789                                 if (!paused)
7790                                         ahc_unpause(ahc);
7791                         }
7792                 }
7793         }
7794 }
7795
7796 static int
7797 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7798 {
7799         struct    ahc_tmode_tstate *tstate;
7800         struct    ahc_tmode_lstate *lstate;
7801         struct    ccb_accept_tio *atio;
7802         uint8_t *byte;
7803         int       initiator;
7804         int       target;
7805         int       lun;
7806
7807         initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7808         target = SCSIID_OUR_ID(cmd->scsiid);
7809         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7810
7811         byte = cmd->bytes;
7812         tstate = ahc->enabled_targets[target];
7813         lstate = NULL;
7814         if (tstate != NULL)
7815                 lstate = tstate->enabled_luns[lun];
7816
7817         /*
7818          * Commands for disabled luns go to the black hole driver.
7819          */
7820         if (lstate == NULL)
7821                 lstate = ahc->black_hole;
7822
7823         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7824         if (atio == NULL) {
7825                 ahc->flags |= AHC_TQINFIFO_BLOCKED;
7826                 /*
7827                  * Wait for more ATIOs from the peripheral driver for this lun.
7828                  */
7829                 if (bootverbose)
7830                         printk("%s: ATIOs exhausted\n", ahc_name(ahc));
7831                 return (1);
7832         } else
7833                 ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7834 #if 0
7835         printk("Incoming command from %d for %d:%d%s\n",
7836                initiator, target, lun,
7837                lstate == ahc->black_hole ? "(Black Holed)" : "");
7838 #endif
7839         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7840
7841         if (lstate == ahc->black_hole) {
7842                 /* Fill in the wildcards */
7843                 atio->ccb_h.target_id = target;
7844                 atio->ccb_h.target_lun = lun;
7845         }
7846
7847         /*
7848          * Package it up and send it off to
7849          * whomever has this lun enabled.
7850          */
7851         atio->sense_len = 0;
7852         atio->init_id = initiator;
7853         if (byte[0] != 0xFF) {
7854                 /* Tag was included */
7855                 atio->tag_action = *byte++;
7856                 atio->tag_id = *byte++;
7857                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7858         } else {
7859                 atio->ccb_h.flags = 0;
7860         }
7861         byte++;
7862
7863         /* Okay.  Now determine the cdb size based on the command code */
7864         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7865         case 0:
7866                 atio->cdb_len = 6;
7867                 break;
7868         case 1:
7869         case 2:
7870                 atio->cdb_len = 10;
7871                 break;
7872         case 4:
7873                 atio->cdb_len = 16;
7874                 break;
7875         case 5:
7876                 atio->cdb_len = 12;
7877                 break;
7878         case 3:
7879         default:
7880                 /* Only copy the opcode. */
7881                 atio->cdb_len = 1;
7882                 printk("Reserved or VU command code type encountered\n");
7883                 break;
7884         }
7885
7886         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7887
7888         atio->ccb_h.status |= CAM_CDB_RECVD;
7889
7890         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7891                 /*
7892                  * We weren't allowed to disconnect.
7893                  * We're hanging on the bus until a
7894                  * continue target I/O comes in response
7895                  * to this accept tio.
7896                  */
7897 #if 0
7898                 printk("Received Immediate Command %d:%d:%d - %p\n",
7899                        initiator, target, lun, ahc->pending_device);
7900 #endif
7901                 ahc->pending_device = lstate;
7902                 ahc_freeze_ccb((union ccb *)atio);
7903                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7904         }
7905         xpt_done((union ccb*)atio);
7906         return (0);
7907 }
7908
7909 #endif