drivers/atm/zatm.c

   1 /* drivers/atm/zatm.c - ZeitNet ZN122x device driver */
   2
   3 /* Written 1995-2000 by Werner Almesberger, EPFL LRC/ICA */
   4
   5
   6 #include <linux/module.h>
   7 #include <linux/kernel.h>
   8 #include <linux/mm.h>
   9 #include <linux/pci.h>
  10 #include <linux/errno.h>
  11 #include <linux/atm.h>
  12 #include <linux/atmdev.h>
  13 #include <linux/sonet.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/netdevice.h>
  16 #include <linux/delay.h>
  17 #include <linux/uio.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/atm_zatm.h>
  22 #include <linux/capability.h>
  23 #include <linux/bitops.h>
  24 #include <linux/wait.h>
  25 #include <linux/slab.h>
  26 #include <asm/byteorder.h>
  27 #include <asm/string.h>
  28 #include <asm/io.h>
  29 #include <linux/atomic.h>
  30 #include <linux/uaccess.h>
  31 #include <linux/nospec.h>
  32
  33 #include "uPD98401.h"
  34 #include "uPD98402.h"
  35 #include "zeprom.h"
  36 #include "zatm.h"
  37
  38
  39 /*
  40  * TODO:
  41  *
  42  * Minor features
  43  *  - support 64 kB SDUs (will have to use multibuffer batches then :-( )
  44  *  - proper use of CDV, credit = max(1,CDVT*PCR)
  45  *  - AAL0
  46  *  - better receive timestamps
  47  *  - OAM
  48  */
  49
  50 #define ZATM_COPPER     1
  51
  52 #if 0
  53 #define DPRINTK(format,args...) printk(KERN_DEBUG format,##args)
  54 #else
  55 #define DPRINTK(format,args...)
  56 #endif
  57
  58 #ifndef CONFIG_ATM_ZATM_DEBUG
  59
  60
  61 #define NULLCHECK(x)
  62
  63 #define EVENT(s,a,b)
  64
  65
  66 static void event_dump(void)
  67 {
  68 }
  69
  70
  71 #else
  72
  73
  74 /*
  75  * NULL pointer checking
  76  */
  77
  78 #define NULLCHECK(x) \
  79   if ((unsigned long) (x) < 0x30) printk(KERN_CRIT #x "==0x%x\n", (int) (x))
  80
  81 /*
  82  * Very extensive activity logging. Greatly improves bug detection speed but
  83  * costs a few Mbps if enabled.
  84  */
  85
  86 #define EV 64
  87
  88 static const char *ev[EV];
  89 static unsigned long ev_a[EV],ev_b[EV];
  90 static int ec = 0;
  91
  92
  93 static void EVENT(const char *s,unsigned long a,unsigned long b)
  94 {
  95         ev[ec] = s;
  96         ev_a[ec] = a;
  97         ev_b[ec] = b;
  98         ec = (ec+1) % EV;
  99 }
 100
 101
 102 static void event_dump(void)
 103 {
 104         int n,i;
 105
 106         printk(KERN_NOTICE "----- event dump follows -----\n");
 107         for (n = 0; n < EV; n++) {
 108                 i = (ec+n) % EV;
 109                 printk(KERN_NOTICE);
 110                 printk(ev[i] ? ev[i] : "(null)",ev_a[i],ev_b[i]);
 111         }
 112         printk(KERN_NOTICE "----- event dump ends here -----\n");
 113 }
 114
 115
 116 #endif /* CONFIG_ATM_ZATM_DEBUG */
 117
 118
 119 #define RING_BUSY       1       /* indication from do_tx that PDU has to be
 120                                    backlogged */
 121
 122 static struct atm_dev *zatm_boards = NULL;
 123 static unsigned long dummy[2] = {0,0};
 124
 125
 126 #define zin_n(r) inl(zatm_dev->base+r*4)
 127 #define zin(r) inl(zatm_dev->base+uPD98401_##r*4)
 128 #define zout(v,r) outl(v,zatm_dev->base+uPD98401_##r*4)
 129 #define zwait() do {} while (zin(CMR) & uPD98401_BUSY)
 130
 131 /* RX0, RX1, TX0, TX1 */
 132 static const int mbx_entries[NR_MBX] = { 1024,1024,1024,1024 };
 133 static const int mbx_esize[NR_MBX] = { 16,16,4,4 }; /* entry size in bytes */
 134
 135 #define MBX_SIZE(i) (mbx_entries[i]*mbx_esize[i])
 136
 137
 138 /*-------------------------------- utilities --------------------------------*/
 139
 140
 141 static void zpokel(struct zatm_dev *zatm_dev,u32 value,u32 addr)
 142 {
 143         zwait();
 144         zout(value,CER);
 145         zout(uPD98401_IND_ACC | uPD98401_IA_BALL |
 146             (uPD98401_IA_TGT_CM << uPD98401_IA_TGT_SHIFT) | addr,CMR);
 147 }
 148
 149
 150 static u32 zpeekl(struct zatm_dev *zatm_dev,u32 addr)
 151 {
 152         zwait();
 153         zout(uPD98401_IND_ACC | uPD98401_IA_BALL | uPD98401_IA_RW |
 154           (uPD98401_IA_TGT_CM << uPD98401_IA_TGT_SHIFT) | addr,CMR);
 155         zwait();
 156         return zin(CER);
 157 }
 158
 159
 160 /*------------------------------- free lists --------------------------------*/
 161
 162
 163 /*
 164  * Free buffer head structure:
 165  *   [0] pointer to buffer (for SAR)
 166  *   [1] buffer descr link pointer (for SAR)
 167  *   [2] back pointer to skb (for poll_rx)
 168  *   [3] data
 169  *   ...
 170  */
 171
 172 struct rx_buffer_head {
 173         u32             buffer; /* pointer to buffer (for SAR) */
 174         u32             link;   /* buffer descriptor link pointer (for SAR) */
 175         struct sk_buff  *skb;   /* back pointer to skb (for poll_rx) */
 176 };
 177
 178
 179 static void refill_pool(struct atm_dev *dev,int pool)
 180 {
 181         struct zatm_dev *zatm_dev;
 182         struct sk_buff *skb;
 183         struct rx_buffer_head *first;
 184         unsigned long flags;
 185         int align,offset,free,count,size;
 186
 187         EVENT("refill_pool\n",0,0);
 188         zatm_dev = ZATM_DEV(dev);
 189         size = (64 << (pool <= ZATM_AAL5_POOL_BASE ? 0 :
 190             pool-ZATM_AAL5_POOL_BASE))+sizeof(struct rx_buffer_head);
 191         if (size < PAGE_SIZE) {
 192                 align = 32; /* for 32 byte alignment */
 193                 offset = sizeof(struct rx_buffer_head);
 194         }
 195         else {
 196                 align = 4096;
 197                 offset = zatm_dev->pool_info[pool].offset+
 198                     sizeof(struct rx_buffer_head);
 199         }
 200         size += align;
 201         spin_lock_irqsave(&zatm_dev->lock, flags);
 202         free = zpeekl(zatm_dev,zatm_dev->pool_base+2*pool) &
 203             uPD98401_RXFP_REMAIN;
 204         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 205         if (free >= zatm_dev->pool_info[pool].low_water) return;
 206         EVENT("starting ... POOL: 0x%x, 0x%x\n",
 207             zpeekl(zatm_dev,zatm_dev->pool_base+2*pool),
 208             zpeekl(zatm_dev,zatm_dev->pool_base+2*pool+1));
 209         EVENT("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
 210         count = 0;
 211         first = NULL;
 212         while (free < zatm_dev->pool_info[pool].high_water) {
 213                 struct rx_buffer_head *head;
 214
 215                 skb = alloc_skb(size,GFP_ATOMIC);
 216                 if (!skb) {
 217                         printk(KERN_WARNING DEV_LABEL "(Itf %d): got no new "
 218                             "skb (%d) with %d free\n",dev->number,size,free);
 219                         break;
 220                 }
 221                 skb_reserve(skb,(unsigned char *) ((((unsigned long) skb->data+
 222                     align+offset-1) & ~(unsigned long) (align-1))-offset)-
 223                     skb->data);
 224                 head = (struct rx_buffer_head *) skb->data;
 225                 skb_reserve(skb,sizeof(struct rx_buffer_head));
 226                 if (!first) first = head;
 227                 count++;
 228                 head->buffer = virt_to_bus(skb->data);
 229                 head->link = 0;
 230                 head->skb = skb;
 231                 EVENT("enq skb 0x%08lx/0x%08lx\n",(unsigned long) skb,
 232                     (unsigned long) head);
 233                 spin_lock_irqsave(&zatm_dev->lock, flags);
 234                 if (zatm_dev->last_free[pool])
 235                         ((struct rx_buffer_head *) (zatm_dev->last_free[pool]->
 236                             data))[-1].link = virt_to_bus(head);
 237                 zatm_dev->last_free[pool] = skb;
 238                 skb_queue_tail(&zatm_dev->pool[pool],skb);
 239                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
 240                 free++;
 241         }
 242         if (first) {
 243                 spin_lock_irqsave(&zatm_dev->lock, flags);
 244                 zwait();
 245                 zout(virt_to_bus(first),CER);
 246                 zout(uPD98401_ADD_BAT | (pool << uPD98401_POOL_SHIFT) | count,
 247                     CMR);
 248                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
 249                 EVENT ("POOL: 0x%x, 0x%x\n",
 250                     zpeekl(zatm_dev,zatm_dev->pool_base+2*pool),
 251                     zpeekl(zatm_dev,zatm_dev->pool_base+2*pool+1));
 252                 EVENT("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
 253         }
 254 }
 255
 256
 257 static void drain_free(struct atm_dev *dev,int pool)
 258 {
 259         skb_queue_purge(&ZATM_DEV(dev)->pool[pool]);
 260 }
 261
 262
 263 static int pool_index(int max_pdu)
 264 {
 265         int i;
 266
 267         if (max_pdu % ATM_CELL_PAYLOAD)
 268                 printk(KERN_ERR DEV_LABEL ": driver error in pool_index: "
 269                     "max_pdu is %d\n",max_pdu);
 270         if (max_pdu > 65536) return -1;
 271         for (i = 0; (64 << i) < max_pdu; i++);
 272         return i+ZATM_AAL5_POOL_BASE;
 273 }
 274
 275
 276 /* use_pool isn't reentrant */
 277
 278
 279 static void use_pool(struct atm_dev *dev,int pool)
 280 {
 281         struct zatm_dev *zatm_dev;
 282         unsigned long flags;
 283         int size;
 284
 285         zatm_dev = ZATM_DEV(dev);
 286         if (!(zatm_dev->pool_info[pool].ref_count++)) {
 287                 skb_queue_head_init(&zatm_dev->pool[pool]);
 288                 size = pool-ZATM_AAL5_POOL_BASE;
 289                 if (size < 0) size = 0; /* 64B... */
 290                 else if (size > 10) size = 10; /* ... 64kB */
 291                 spin_lock_irqsave(&zatm_dev->lock, flags);
 292                 zpokel(zatm_dev,((zatm_dev->pool_info[pool].low_water/4) <<
 293                     uPD98401_RXFP_ALERT_SHIFT) |
 294                     (1 << uPD98401_RXFP_BTSZ_SHIFT) |
 295                     (size << uPD98401_RXFP_BFSZ_SHIFT),
 296                     zatm_dev->pool_base+pool*2);
 297                 zpokel(zatm_dev,(unsigned long) dummy,zatm_dev->pool_base+
 298                     pool*2+1);
 299                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
 300                 zatm_dev->last_free[pool] = NULL;
 301                 refill_pool(dev,pool);
 302         }
 303         DPRINTK("pool %d: %d\n",pool,zatm_dev->pool_info[pool].ref_count);
 304 }
 305
 306
 307 static void unuse_pool(struct atm_dev *dev,int pool)
 308 {
 309         if (!(--ZATM_DEV(dev)->pool_info[pool].ref_count))
 310                 drain_free(dev,pool);
 311 }
 312
 313 /*----------------------------------- RX ------------------------------------*/
 314
 315
 316 #if 0
 317 static void exception(struct atm_vcc *vcc)
 318 {
 319    static int count = 0;
 320    struct zatm_dev *zatm_dev = ZATM_DEV(vcc->dev);
 321    struct zatm_vcc *zatm_vcc = ZATM_VCC(vcc);
 322    unsigned long *qrp;
 323    int i;
 324
 325    if (count++ > 2) return;
 326    for (i = 0; i < 8; i++)
 327         printk("TX%d: 0x%08lx\n",i,
 328           zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+i));
 329    for (i = 0; i < 5; i++)
 330         printk("SH%d: 0x%08lx\n",i,
 331           zpeekl(zatm_dev,uPD98401_IM(zatm_vcc->shaper)+16*i));
 332    qrp = (unsigned long *) zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+
 333      uPD98401_TXVC_QRP);
 334    printk("qrp=0x%08lx\n",(unsigned long) qrp);
 335    for (i = 0; i < 4; i++) printk("QRP[%d]: 0x%08lx",i,qrp[i]);
 336 }
 337 #endif
 338
 339
 340 static const char *err_txt[] = {
 341         "No error",
 342         "RX buf underflow",
 343         "RX FIFO overrun",
 344         "Maximum len violation",
 345         "CRC error",
 346         "User abort",
 347         "Length violation",
 348         "T1 error",
 349         "Deactivated",
 350         "???",
 351         "???",
 352         "???",
 353         "???",
 354         "???",
 355         "???",
 356         "???"
 357 };
 358
 359
 360 static void poll_rx(struct atm_dev *dev,int mbx)
 361 {
 362         struct zatm_dev *zatm_dev;
 363         unsigned long pos;
 364         u32 x;
 365         int error;
 366
 367         EVENT("poll_rx\n",0,0);
 368         zatm_dev = ZATM_DEV(dev);
 369         pos = (zatm_dev->mbx_start[mbx] & ~0xffffUL) | zin(MTA(mbx));
 370         while (x = zin(MWA(mbx)), (pos & 0xffff) != x) {
 371                 u32 *here;
 372                 struct sk_buff *skb;
 373                 struct atm_vcc *vcc;
 374                 int cells,size,chan;
 375
 376                 EVENT("MBX: host 0x%lx, nic 0x%x\n",pos,x);
 377                 here = (u32 *) pos;
 378                 if (((pos += 16) & 0xffff) == zatm_dev->mbx_end[mbx])
 379                         pos = zatm_dev->mbx_start[mbx];
 380                 cells = here[0] & uPD98401_AAL5_SIZE;
 381 #if 0
 382 printk("RX IND: 0x%x, 0x%x, 0x%x, 0x%x\n",here[0],here[1],here[2],here[3]);
 383 {
 384 unsigned long *x;
 385                 printk("POOL: 0x%08x, 0x%08x\n",zpeekl(zatm_dev,
 386                       zatm_dev->pool_base),
 387                       zpeekl(zatm_dev,zatm_dev->pool_base+1));
 388                 x = (unsigned long *) here[2];
 389                 printk("[0..3] = 0x%08lx, 0x%08lx, 0x%08lx, 0x%08lx\n",
 390                     x[0],x[1],x[2],x[3]);
 391 }
 392 #endif
 393                 error = 0;
 394                 if (here[3] & uPD98401_AAL5_ERR) {
 395                         error = (here[3] & uPD98401_AAL5_ES) >>
 396                             uPD98401_AAL5_ES_SHIFT;
 397                         if (error == uPD98401_AAL5_ES_DEACT ||
 398                             error == uPD98401_AAL5_ES_FREE) continue;
 399                 }
 400 EVENT("error code 0x%x/0x%x\n",(here[3] & uPD98401_AAL5_ES) >>
 401   uPD98401_AAL5_ES_SHIFT,error);
 402                 skb = ((struct rx_buffer_head *) bus_to_virt(here[2]))->skb;
 403                 __net_timestamp(skb);
 404 #if 0
 405 printk("[-3..0] 0x%08lx 0x%08lx 0x%08lx 0x%08lx\n",((unsigned *) skb->data)[-3],
 406   ((unsigned *) skb->data)[-2],((unsigned *) skb->data)[-1],
 407   ((unsigned *) skb->data)[0]);
 408 #endif
 409                 EVENT("skb 0x%lx, here 0x%lx\n",(unsigned long) skb,
 410                     (unsigned long) here);
 411 #if 0
 412 printk("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
 413 #endif
 414                 size = error ? 0 : ntohs(((__be16 *) skb->data)[cells*
 415                     ATM_CELL_PAYLOAD/sizeof(u16)-3]);
 416                 EVENT("got skb 0x%lx, size %d\n",(unsigned long) skb,size);
 417                 chan = (here[3] & uPD98401_AAL5_CHAN) >>
 418                     uPD98401_AAL5_CHAN_SHIFT;
 419                 if (chan < zatm_dev->chans && zatm_dev->rx_map[chan]) {
 420                         int pos;
 421                         vcc = zatm_dev->rx_map[chan];
 422                         pos = ZATM_VCC(vcc)->pool;
 423                         if (skb == zatm_dev->last_free[pos])
 424                                 zatm_dev->last_free[pos] = NULL;
 425                         skb_unlink(skb, zatm_dev->pool + pos);
 426                 }
 427                 else {
 428                         printk(KERN_ERR DEV_LABEL "(itf %d): RX indication "
 429                             "for non-existing channel\n",dev->number);
 430                         size = 0;
 431                         vcc = NULL;
 432                         event_dump();
 433                 }
 434                 if (error) {
 435                         static unsigned long silence = 0;
 436                         static int last_error = 0;
 437
 438                         if (error != last_error ||
 439                             time_after(jiffies, silence)  || silence == 0){
 440                                 printk(KERN_WARNING DEV_LABEL "(itf %d): "
 441                                     "chan %d error %s\n",dev->number,chan,
 442                                     err_txt[error]);
 443                                 last_error = error;
 444                                 silence = (jiffies+2*HZ)|1;
 445                         }
 446                         size = 0;
 447                 }
 448                 if (size && (size > cells*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER ||
 449                     size <= (cells-1)*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER)) {
 450                         printk(KERN_ERR DEV_LABEL "(itf %d): size %d with %d "
 451                             "cells\n",dev->number,size,cells);
 452                         size = 0;
 453                         event_dump();
 454                 }
 455                 if (size > ATM_MAX_AAL5_PDU) {
 456                         printk(KERN_ERR DEV_LABEL "(itf %d): size too big "
 457                             "(%d)\n",dev->number,size);
 458                         size = 0;
 459                         event_dump();
 460                 }
 461                 if (!size) {
 462                         dev_kfree_skb_irq(skb);
 463                         if (vcc) atomic_inc(&vcc->stats->rx_err);
 464                         continue;
 465                 }
 466                 if (!atm_charge(vcc,skb->truesize)) {
 467                         dev_kfree_skb_irq(skb);
 468                         continue;
 469                 }
 470                 skb->len = size;
 471                 ATM_SKB(skb)->vcc = vcc;
 472                 vcc->push(vcc,skb);
 473                 atomic_inc(&vcc->stats->rx);
 474         }
 475         zout(pos & 0xffff,MTA(mbx));
 476 #if 0 /* probably a stupid idea */
 477         refill_pool(dev,zatm_vcc->pool);
 478                 /* maybe this saves us a few interrupts */
 479 #endif
 480 }
 481
 482
 483 static int open_rx_first(struct atm_vcc *vcc)
 484 {
 485         struct zatm_dev *zatm_dev;
 486         struct zatm_vcc *zatm_vcc;
 487         unsigned long flags;
 488         unsigned short chan;
 489         int cells;
 490
 491         DPRINTK("open_rx_first (0x%x)\n",inb_p(0xc053));
 492         zatm_dev = ZATM_DEV(vcc->dev);
 493         zatm_vcc = ZATM_VCC(vcc);
 494         zatm_vcc->rx_chan = 0;
 495         if (vcc->qos.rxtp.traffic_class == ATM_NONE) return 0;
 496         if (vcc->qos.aal == ATM_AAL5) {
 497                 if (vcc->qos.rxtp.max_sdu > 65464)
 498                         vcc->qos.rxtp.max_sdu = 65464;
 499                         /* fix this - we may want to receive 64kB SDUs
 500                            later */
 501                 cells = DIV_ROUND_UP(vcc->qos.rxtp.max_sdu + ATM_AAL5_TRAILER,
 502                                 ATM_CELL_PAYLOAD);
 503                 zatm_vcc->pool = pool_index(cells*ATM_CELL_PAYLOAD);
 504         }
 505         else {
 506                 cells = 1;
 507                 zatm_vcc->pool = ZATM_AAL0_POOL;
 508         }
 509         if (zatm_vcc->pool < 0) return -EMSGSIZE;
 510         spin_lock_irqsave(&zatm_dev->lock, flags);
 511         zwait();
 512         zout(uPD98401_OPEN_CHAN,CMR);
 513         zwait();
 514         DPRINTK("0x%x 0x%x\n",zin(CMR),zin(CER));
 515         chan = (zin(CMR) & uPD98401_CHAN_ADDR) >> uPD98401_CHAN_ADDR_SHIFT;
 516         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 517         DPRINTK("chan is %d\n",chan);
 518         if (!chan) return -EAGAIN;
 519         use_pool(vcc->dev,zatm_vcc->pool);
 520         DPRINTK("pool %d\n",zatm_vcc->pool);
 521         /* set up VC descriptor */
 522         spin_lock_irqsave(&zatm_dev->lock, flags);
 523         zpokel(zatm_dev,zatm_vcc->pool << uPD98401_RXVC_POOL_SHIFT,
 524             chan*VC_SIZE/4);
 525         zpokel(zatm_dev,uPD98401_RXVC_OD | (vcc->qos.aal == ATM_AAL5 ?
 526             uPD98401_RXVC_AR : 0) | cells,chan*VC_SIZE/4+1);
 527         zpokel(zatm_dev,0,chan*VC_SIZE/4+2);
 528         zatm_vcc->rx_chan = chan;
 529         zatm_dev->rx_map[chan] = vcc;
 530         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 531         return 0;
 532 }
 533
 534
 535 static int open_rx_second(struct atm_vcc *vcc)
 536 {
 537         struct zatm_dev *zatm_dev;
 538         struct zatm_vcc *zatm_vcc;
 539         unsigned long flags;
 540         int pos,shift;
 541
 542         DPRINTK("open_rx_second (0x%x)\n",inb_p(0xc053));
 543         zatm_dev = ZATM_DEV(vcc->dev);
 544         zatm_vcc = ZATM_VCC(vcc);
 545         if (!zatm_vcc->rx_chan) return 0;
 546         spin_lock_irqsave(&zatm_dev->lock, flags);
 547         /* should also handle VPI @@@ */
 548         pos = vcc->vci >> 1;
 549         shift = (1-(vcc->vci & 1)) << 4;
 550         zpokel(zatm_dev,(zpeekl(zatm_dev,pos) & ~(0xffff << shift)) |
 551             ((zatm_vcc->rx_chan | uPD98401_RXLT_ENBL) << shift),pos);
 552         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 553         return 0;
 554 }
 555
 556
 557 static void close_rx(struct atm_vcc *vcc)
 558 {
 559         struct zatm_dev *zatm_dev;
 560         struct zatm_vcc *zatm_vcc;
 561         unsigned long flags;
 562         int pos,shift;
 563
 564         zatm_vcc = ZATM_VCC(vcc);
 565         zatm_dev = ZATM_DEV(vcc->dev);
 566         if (!zatm_vcc->rx_chan) return;
 567         DPRINTK("close_rx\n");
 568         /* disable receiver */
 569         if (vcc->vpi != ATM_VPI_UNSPEC && vcc->vci != ATM_VCI_UNSPEC) {
 570                 spin_lock_irqsave(&zatm_dev->lock, flags);
 571                 pos = vcc->vci >> 1;
 572                 shift = (1-(vcc->vci & 1)) << 4;
 573                 zpokel(zatm_dev,zpeekl(zatm_dev,pos) & ~(0xffff << shift),pos);
 574                 zwait();
 575                 zout(uPD98401_NOP,CMR);
 576                 zwait();
 577                 zout(uPD98401_NOP,CMR);
 578                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
 579         }
 580         spin_lock_irqsave(&zatm_dev->lock, flags);
 581         zwait();
 582         zout(uPD98401_DEACT_CHAN | uPD98401_CHAN_RT | (zatm_vcc->rx_chan <<
 583             uPD98401_CHAN_ADDR_SHIFT),CMR);
 584         zwait();
 585         udelay(10); /* why oh why ... ? */
 586         zout(uPD98401_CLOSE_CHAN | uPD98401_CHAN_RT | (zatm_vcc->rx_chan <<
 587             uPD98401_CHAN_ADDR_SHIFT),CMR);
 588         zwait();
 589         if (!(zin(CMR) & uPD98401_CHAN_ADDR))
 590                 printk(KERN_CRIT DEV_LABEL "(itf %d): can't close RX channel "
 591                     "%d\n",vcc->dev->number,zatm_vcc->rx_chan);
 592         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 593         zatm_dev->rx_map[zatm_vcc->rx_chan] = NULL;
 594         zatm_vcc->rx_chan = 0;
 595         unuse_pool(vcc->dev,zatm_vcc->pool);
 596 }
 597
 598
 599 static int start_rx(struct atm_dev *dev)
 600 {
 601         struct zatm_dev *zatm_dev;
 602         int i;
 603
 604         DPRINTK("start_rx\n");
 605         zatm_dev = ZATM_DEV(dev);
 606         zatm_dev->rx_map = kcalloc(zatm_dev->chans,
 607                                    sizeof(*zatm_dev->rx_map),
 608                                    GFP_KERNEL);
 609         if (!zatm_dev->rx_map) return -ENOMEM;
 610         /* set VPI/VCI split (use all VCIs and give what's left to VPIs) */
 611         zpokel(zatm_dev,(1 << dev->ci_range.vci_bits)-1,uPD98401_VRR);
 612         /* prepare free buffer pools */
 613         for (i = 0; i <= ZATM_LAST_POOL; i++) {
 614                 zatm_dev->pool_info[i].ref_count = 0;
 615                 zatm_dev->pool_info[i].rqa_count = 0;
 616                 zatm_dev->pool_info[i].rqu_count = 0;
 617                 zatm_dev->pool_info[i].low_water = LOW_MARK;
 618                 zatm_dev->pool_info[i].high_water = HIGH_MARK;
 619                 zatm_dev->pool_info[i].offset = 0;
 620                 zatm_dev->pool_info[i].next_off = 0;
 621                 zatm_dev->pool_info[i].next_cnt = 0;
 622                 zatm_dev->pool_info[i].next_thres = OFF_CNG_THRES;
 623         }
 624         return 0;
 625 }
 626
 627
 628 /*----------------------------------- TX ------------------------------------*/
 629
 630
 631 static int do_tx(struct sk_buff *skb)
 632 {
 633         struct atm_vcc *vcc;
 634         struct zatm_dev *zatm_dev;
 635         struct zatm_vcc *zatm_vcc;
 636         u32 *dsc;
 637         unsigned long flags;
 638
 639         EVENT("do_tx\n",0,0);
 640         DPRINTK("sending skb %p\n",skb);
 641         vcc = ATM_SKB(skb)->vcc;
 642         zatm_dev = ZATM_DEV(vcc->dev);
 643         zatm_vcc = ZATM_VCC(vcc);
 644         EVENT("iovcnt=%d\n",skb_shinfo(skb)->nr_frags,0);
 645         spin_lock_irqsave(&zatm_dev->lock, flags);
 646         if (!skb_shinfo(skb)->nr_frags) {
 647                 if (zatm_vcc->txing == RING_ENTRIES-1) {
 648                         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 649                         return RING_BUSY;
 650                 }
 651                 zatm_vcc->txing++;
 652                 dsc = zatm_vcc->ring+zatm_vcc->ring_curr;
 653                 zatm_vcc->ring_curr = (zatm_vcc->ring_curr+RING_WORDS) &
 654                     (RING_ENTRIES*RING_WORDS-1);
 655                 dsc[1] = 0;
 656                 dsc[2] = skb->len;
 657                 dsc[3] = virt_to_bus(skb->data);
 658                 mb();
 659                 dsc[0] = uPD98401_TXPD_V | uPD98401_TXPD_DP | uPD98401_TXPD_SM
 660                     | (vcc->qos.aal == ATM_AAL5 ? uPD98401_TXPD_AAL5 : 0 |
 661                     (ATM_SKB(skb)->atm_options & ATM_ATMOPT_CLP ?
 662                     uPD98401_CLPM_1 : uPD98401_CLPM_0));
 663                 EVENT("dsc (0x%lx)\n",(unsigned long) dsc,0);
 664         }
 665         else {
 666 printk("NONONONOO!!!!\n");
 667                 dsc = NULL;
 668 #if 0
 669                 u32 *put;
 670                 int i;
 671
 672                 dsc = kmalloc(uPD98401_TXPD_SIZE * 2 +
 673                         uPD98401_TXBD_SIZE * ATM_SKB(skb)->iovcnt, GFP_ATOMIC);
 674                 if (!dsc) {
 675                         if (vcc->pop)
 676                                 vcc->pop(vcc, skb);
 677                         else
 678                                 dev_kfree_skb_irq(skb);
 679                         return -EAGAIN;
 680                 }
 681                 /* @@@ should check alignment */
 682                 put = dsc+8;
 683                 dsc[0] = uPD98401_TXPD_V | uPD98401_TXPD_DP |
 684                     (vcc->aal == ATM_AAL5 ? uPD98401_TXPD_AAL5 : 0 |
 685                     (ATM_SKB(skb)->atm_options & ATM_ATMOPT_CLP ?
 686                     uPD98401_CLPM_1 : uPD98401_CLPM_0));
 687                 dsc[1] = 0;
 688                 dsc[2] = ATM_SKB(skb)->iovcnt * uPD98401_TXBD_SIZE;
 689                 dsc[3] = virt_to_bus(put);
 690                 for (i = 0; i < ATM_SKB(skb)->iovcnt; i++) {
 691                         *put++ = ((struct iovec *) skb->data)[i].iov_len;
 692                         *put++ = virt_to_bus(((struct iovec *)
 693                             skb->data)[i].iov_base);
 694                 }
 695                 put[-2] |= uPD98401_TXBD_LAST;
 696 #endif
 697         }
 698         ZATM_PRV_DSC(skb) = dsc;
 699         skb_queue_tail(&zatm_vcc->tx_queue,skb);
 700         DPRINTK("QRP=0x%08lx\n",zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+
 701           uPD98401_TXVC_QRP));
 702         zwait();
 703         zout(uPD98401_TX_READY | (zatm_vcc->tx_chan <<
 704             uPD98401_CHAN_ADDR_SHIFT),CMR);
 705         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 706         EVENT("done\n",0,0);
 707         return 0;
 708 }
 709
 710
 711 static inline void dequeue_tx(struct atm_vcc *vcc)
 712 {
 713         struct zatm_vcc *zatm_vcc;
 714         struct sk_buff *skb;
 715
 716         EVENT("dequeue_tx\n",0,0);
 717         zatm_vcc = ZATM_VCC(vcc);
 718         skb = skb_dequeue(&zatm_vcc->tx_queue);
 719         if (!skb) {
 720                 printk(KERN_CRIT DEV_LABEL "(itf %d): dequeue_tx but not "
 721                     "txing\n",vcc->dev->number);
 722                 return;
 723         }
 724 #if 0 /* @@@ would fail on CLP */
 725 if (*ZATM_PRV_DSC(skb) != (uPD98401_TXPD_V | uPD98401_TXPD_DP |
 726   uPD98401_TXPD_SM | uPD98401_TXPD_AAL5)) printk("@#*$!!!!  (%08x)\n",
 727   *ZATM_PRV_DSC(skb));
 728 #endif
 729         *ZATM_PRV_DSC(skb) = 0; /* mark as invalid */
 730         zatm_vcc->txing--;
 731         if (vcc->pop) vcc->pop(vcc,skb);
 732         else dev_kfree_skb_irq(skb);
 733         while ((skb = skb_dequeue(&zatm_vcc->backlog)))
 734                 if (do_tx(skb) == RING_BUSY) {
 735                         skb_queue_head(&zatm_vcc->backlog,skb);
 736                         break;
 737                 }
 738         atomic_inc(&vcc->stats->tx);
 739         wake_up(&zatm_vcc->tx_wait);
 740 }
 741
 742
 743 static void poll_tx(struct atm_dev *dev,int mbx)
 744 {
 745         struct zatm_dev *zatm_dev;
 746         unsigned long pos;
 747         u32 x;
 748
 749         EVENT("poll_tx\n",0,0);
 750         zatm_dev = ZATM_DEV(dev);
 751         pos = (zatm_dev->mbx_start[mbx] & ~0xffffUL) | zin(MTA(mbx));
 752         while (x = zin(MWA(mbx)), (pos & 0xffff) != x) {
 753                 int chan;
 754
 755 #if 1
 756                 u32 data,*addr;
 757
 758                 EVENT("MBX: host 0x%lx, nic 0x%x\n",pos,x);
 759                 addr = (u32 *) pos;
 760                 data = *addr;
 761                 chan = (data & uPD98401_TXI_CONN) >> uPD98401_TXI_CONN_SHIFT;
 762                 EVENT("addr = 0x%lx, data = 0x%08x,",(unsigned long) addr,
 763                     data);
 764                 EVENT("chan = %d\n",chan,0);
 765 #else
 766 NO !
 767                 chan = (zatm_dev->mbx_start[mbx][pos >> 2] & uPD98401_TXI_CONN)
 768                 >> uPD98401_TXI_CONN_SHIFT;
 769 #endif
 770                 if (chan < zatm_dev->chans && zatm_dev->tx_map[chan])
 771                         dequeue_tx(zatm_dev->tx_map[chan]);
 772                 else {
 773                         printk(KERN_CRIT DEV_LABEL "(itf %d): TX indication "
 774                             "for non-existing channel %d\n",dev->number,chan);
 775                         event_dump();
 776                 }
 777                 if (((pos += 4) & 0xffff) == zatm_dev->mbx_end[mbx])
 778                         pos = zatm_dev->mbx_start[mbx];
 779         }
 780         zout(pos & 0xffff,MTA(mbx));
 781 }
 782
 783
 784 /*
 785  * BUG BUG BUG: Doesn't handle "new-style" rate specification yet.
 786  */
 787
 788 static int alloc_shaper(struct atm_dev *dev,int *pcr,int min,int max,int ubr)
 789 {
 790         struct zatm_dev *zatm_dev;
 791         unsigned long flags;
 792         unsigned long i,m,c;
 793         int shaper;
 794
 795         DPRINTK("alloc_shaper (min = %d, max = %d)\n",min,max);
 796         zatm_dev = ZATM_DEV(dev);
 797         if (!zatm_dev->free_shapers) return -EAGAIN;
 798         for (shaper = 0; !((zatm_dev->free_shapers >> shaper) & 1); shaper++);
 799         zatm_dev->free_shapers &= ~1 << shaper;
 800         if (ubr) {
 801                 c = 5;
 802                 i = m = 1;
 803                 zatm_dev->ubr_ref_cnt++;
 804                 zatm_dev->ubr = shaper;
 805                 *pcr = 0;
 806         }
 807         else {
 808                 if (min) {
 809                         if (min <= 255) {
 810                                 i = min;
 811                                 m = ATM_OC3_PCR;
 812                         }
 813                         else {
 814                                 i = 255;
 815                                 m = ATM_OC3_PCR*255/min;
 816                         }
 817                 }
 818                 else {
 819                         if (max > zatm_dev->tx_bw) max = zatm_dev->tx_bw;
 820                         if (max <= 255) {
 821                                 i = max;
 822                                 m = ATM_OC3_PCR;
 823                         }
 824                         else {
 825                                 i = 255;
 826                                 m = DIV_ROUND_UP(ATM_OC3_PCR*255, max);
 827                         }
 828                 }
 829                 if (i > m) {
 830                         printk(KERN_CRIT DEV_LABEL "shaper algorithm botched "
 831                             "[%d,%d] -> i=%ld,m=%ld\n",min,max,i,m);
 832                         m = i;
 833                 }
 834                 *pcr = i*ATM_OC3_PCR/m;
 835                 c = 20; /* @@@ should use max_cdv ! */
 836                 if ((min && *pcr < min) || (max && *pcr > max)) return -EINVAL;
 837                 if (zatm_dev->tx_bw < *pcr) return -EAGAIN;
 838                 zatm_dev->tx_bw -= *pcr;
 839         }
 840         spin_lock_irqsave(&zatm_dev->lock, flags);
 841         DPRINTK("i = %d, m = %d, PCR = %d\n",i,m,*pcr);
 842         zpokel(zatm_dev,(i << uPD98401_IM_I_SHIFT) | m,uPD98401_IM(shaper));
 843         zpokel(zatm_dev,c << uPD98401_PC_C_SHIFT,uPD98401_PC(shaper));
 844         zpokel(zatm_dev,0,uPD98401_X(shaper));
 845         zpokel(zatm_dev,0,uPD98401_Y(shaper));
 846         zpokel(zatm_dev,uPD98401_PS_E,uPD98401_PS(shaper));
 847         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 848         return shaper;
 849 }
 850
 851
 852 static void dealloc_shaper(struct atm_dev *dev,int shaper)
 853 {
 854         struct zatm_dev *zatm_dev;
 855         unsigned long flags;
 856
 857         zatm_dev = ZATM_DEV(dev);
 858         if (shaper == zatm_dev->ubr) {
 859                 if (--zatm_dev->ubr_ref_cnt) return;
 860                 zatm_dev->ubr = -1;
 861         }
 862         spin_lock_irqsave(&zatm_dev->lock, flags);
 863         zpokel(zatm_dev,zpeekl(zatm_dev,uPD98401_PS(shaper)) & ~uPD98401_PS_E,
 864             uPD98401_PS(shaper));
 865         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 866         zatm_dev->free_shapers |= 1 << shaper;
 867 }
 868
 869
 870 static void close_tx(struct atm_vcc *vcc)
 871 {
 872         struct zatm_dev *zatm_dev;
 873         struct zatm_vcc *zatm_vcc;
 874         unsigned long flags;
 875         int chan;
 876
 877         zatm_vcc = ZATM_VCC(vcc);
 878         zatm_dev = ZATM_DEV(vcc->dev);
 879         chan = zatm_vcc->tx_chan;
 880         if (!chan) return;
 881         DPRINTK("close_tx\n");
 882         if (skb_peek(&zatm_vcc->backlog)) {
 883                 printk("waiting for backlog to drain ...\n");
 884                 event_dump();
 885                 wait_event(zatm_vcc->tx_wait, !skb_peek(&zatm_vcc->backlog));
 886         }
 887         if (skb_peek(&zatm_vcc->tx_queue)) {
 888                 printk("waiting for TX queue to drain ...\n");
 889                 event_dump();
 890                 wait_event(zatm_vcc->tx_wait, !skb_peek(&zatm_vcc->tx_queue));
 891         }
 892         spin_lock_irqsave(&zatm_dev->lock, flags);
 893 #if 0
 894         zwait();
 895         zout(uPD98401_DEACT_CHAN | (chan << uPD98401_CHAN_ADDR_SHIFT),CMR);
 896 #endif
 897         zwait();
 898         zout(uPD98401_CLOSE_CHAN | (chan << uPD98401_CHAN_ADDR_SHIFT),CMR);
 899         zwait();
 900         if (!(zin(CMR) & uPD98401_CHAN_ADDR))
 901                 printk(KERN_CRIT DEV_LABEL "(itf %d): can't close TX channel "
 902                     "%d\n",vcc->dev->number,chan);
 903         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 904         zatm_vcc->tx_chan = 0;
 905         zatm_dev->tx_map[chan] = NULL;
 906         if (zatm_vcc->shaper != zatm_dev->ubr) {
 907                 zatm_dev->tx_bw += vcc->qos.txtp.min_pcr;
 908                 dealloc_shaper(vcc->dev,zatm_vcc->shaper);
 909         }
 910         kfree(zatm_vcc->ring);
 911 }
 912
 913
 914 static int open_tx_first(struct atm_vcc *vcc)
 915 {
 916         struct zatm_dev *zatm_dev;
 917         struct zatm_vcc *zatm_vcc;
 918         unsigned long flags;
 919         u32 *loop;
 920         unsigned short chan;
 921         int unlimited;
 922
 923         DPRINTK("open_tx_first\n");
 924         zatm_dev = ZATM_DEV(vcc->dev);
 925         zatm_vcc = ZATM_VCC(vcc);
 926         zatm_vcc->tx_chan = 0;
 927         if (vcc->qos.txtp.traffic_class == ATM_NONE) return 0;
 928         spin_lock_irqsave(&zatm_dev->lock, flags);
 929         zwait();
 930         zout(uPD98401_OPEN_CHAN,CMR);
 931         zwait();
 932         DPRINTK("0x%x 0x%x\n",zin(CMR),zin(CER));
 933         chan = (zin(CMR) & uPD98401_CHAN_ADDR) >> uPD98401_CHAN_ADDR_SHIFT;
 934         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 935         DPRINTK("chan is %d\n",chan);
 936         if (!chan) return -EAGAIN;
 937         unlimited = vcc->qos.txtp.traffic_class == ATM_UBR &&
 938             (!vcc->qos.txtp.max_pcr || vcc->qos.txtp.max_pcr == ATM_MAX_PCR ||
 939             vcc->qos.txtp.max_pcr >= ATM_OC3_PCR);
 940         if (unlimited && zatm_dev->ubr != -1) zatm_vcc->shaper = zatm_dev->ubr;
 941         else {
 942                 int uninitialized_var(pcr);
 943
 944                 if (unlimited) vcc->qos.txtp.max_sdu = ATM_MAX_AAL5_PDU;
 945                 if ((zatm_vcc->shaper = alloc_shaper(vcc->dev,&pcr,
 946                     vcc->qos.txtp.min_pcr,vcc->qos.txtp.max_pcr,unlimited))
 947                     < 0) {
 948                         close_tx(vcc);
 949                         return zatm_vcc->shaper;
 950                 }
 951                 if (pcr > ATM_OC3_PCR) pcr = ATM_OC3_PCR;
 952                 vcc->qos.txtp.min_pcr = vcc->qos.txtp.max_pcr = pcr;
 953         }
 954         zatm_vcc->tx_chan = chan;
 955         skb_queue_head_init(&zatm_vcc->tx_queue);
 956         init_waitqueue_head(&zatm_vcc->tx_wait);
 957         /* initialize ring */
 958         zatm_vcc->ring = kzalloc(RING_SIZE,GFP_KERNEL);
 959         if (!zatm_vcc->ring) return -ENOMEM;
 960         loop = zatm_vcc->ring+RING_ENTRIES*RING_WORDS;
 961         loop[0] = uPD98401_TXPD_V;
 962         loop[1] = loop[2] = 0;
 963         loop[3] = virt_to_bus(zatm_vcc->ring);
 964         zatm_vcc->ring_curr = 0;
 965         zatm_vcc->txing = 0;
 966         skb_queue_head_init(&zatm_vcc->backlog);
 967         zpokel(zatm_dev,virt_to_bus(zatm_vcc->ring),
 968             chan*VC_SIZE/4+uPD98401_TXVC_QRP);
 969         return 0;
 970 }
 971
 972
 973 static int open_tx_second(struct atm_vcc *vcc)
 974 {
 975         struct zatm_dev *zatm_dev;
 976         struct zatm_vcc *zatm_vcc;
 977         unsigned long flags;
 978
 979         DPRINTK("open_tx_second\n");
 980         zatm_dev = ZATM_DEV(vcc->dev);
 981         zatm_vcc = ZATM_VCC(vcc);
 982         if (!zatm_vcc->tx_chan) return 0;
 983         /* set up VC descriptor */
 984         spin_lock_irqsave(&zatm_dev->lock, flags);
 985         zpokel(zatm_dev,0,zatm_vcc->tx_chan*VC_SIZE/4);
 986         zpokel(zatm_dev,uPD98401_TXVC_L | (zatm_vcc->shaper <<
 987             uPD98401_TXVC_SHP_SHIFT) | (vcc->vpi << uPD98401_TXVC_VPI_SHIFT) |
 988             vcc->vci,zatm_vcc->tx_chan*VC_SIZE/4+1);
 989         zpokel(zatm_dev,0,zatm_vcc->tx_chan*VC_SIZE/4+2);
 990         spin_unlock_irqrestore(&zatm_dev->lock, flags);
 991         zatm_dev->tx_map[zatm_vcc->tx_chan] = vcc;
 992         return 0;
 993 }
 994
 995
 996 static int start_tx(struct atm_dev *dev)
 997 {
 998         struct zatm_dev *zatm_dev;
 999         int i;
1000
1001         DPRINTK("start_tx\n");
1002         zatm_dev = ZATM_DEV(dev);
1003         zatm_dev->tx_map = kmalloc_array(zatm_dev->chans,
1004                                          sizeof(*zatm_dev->tx_map),
1005                                          GFP_KERNEL);
1006         if (!zatm_dev->tx_map) return -ENOMEM;
1007         zatm_dev->tx_bw = ATM_OC3_PCR;
1008         zatm_dev->free_shapers = (1 << NR_SHAPERS)-1;
1009         zatm_dev->ubr = -1;
1010         zatm_dev->ubr_ref_cnt = 0;
1011         /* initialize shapers */
1012         for (i = 0; i < NR_SHAPERS; i++) zpokel(zatm_dev,0,uPD98401_PS(i));
1013         return 0;
1014 }
1015
1016
1017 /*------------------------------- interrupts --------------------------------*/
1018
1019
1020 static irqreturn_t zatm_int(int irq,void *dev_id)
1021 {
1022         struct atm_dev *dev;
1023         struct zatm_dev *zatm_dev;
1024         u32 reason;
1025         int handled = 0;
1026
1027         dev = dev_id;
1028         zatm_dev = ZATM_DEV(dev);
1029         while ((reason = zin(GSR))) {
1030                 handled = 1;
1031                 EVENT("reason 0x%x\n",reason,0);
1032                 if (reason & uPD98401_INT_PI) {
1033                         EVENT("PHY int\n",0,0);
1034                         dev->phy->interrupt(dev);
1035                 }
1036                 if (reason & uPD98401_INT_RQA) {
1037                         unsigned long pools;
1038                         int i;
1039
1040                         pools = zin(RQA);
1041                         EVENT("RQA (0x%08x)\n",pools,0);
1042                         for (i = 0; pools; i++) {
1043                                 if (pools & 1) {
1044                                         refill_pool(dev,i);
1045                                         zatm_dev->pool_info[i].rqa_count++;
1046                                 }
1047                                 pools >>= 1;
1048                         }
1049                 }
1050                 if (reason & uPD98401_INT_RQU) {
1051                         unsigned long pools;
1052                         int i;
1053                         pools = zin(RQU);
1054                         printk(KERN_WARNING DEV_LABEL "(itf %d): RQU 0x%08lx\n",
1055                             dev->number,pools);
1056                         event_dump();
1057                         for (i = 0; pools; i++) {
1058                                 if (pools & 1) {
1059                                         refill_pool(dev,i);
1060                                         zatm_dev->pool_info[i].rqu_count++;
1061                                 }
1062                                 pools >>= 1;
1063                         }
1064                 }
1065                 /* don't handle RD */
1066                 if (reason & uPD98401_INT_SPE)
1067                         printk(KERN_ALERT DEV_LABEL "(itf %d): system parity "
1068                             "error at 0x%08x\n",dev->number,zin(ADDR));
1069                 if (reason & uPD98401_INT_CPE)
1070                         printk(KERN_ALERT DEV_LABEL "(itf %d): control memory "
1071                             "parity error at 0x%08x\n",dev->number,zin(ADDR));
1072                 if (reason & uPD98401_INT_SBE) {
1073                         printk(KERN_ALERT DEV_LABEL "(itf %d): system bus "
1074                             "error at 0x%08x\n",dev->number,zin(ADDR));
1075                         event_dump();
1076                 }
1077                 /* don't handle IND */
1078                 if (reason & uPD98401_INT_MF) {
1079                         printk(KERN_CRIT DEV_LABEL "(itf %d): mailbox full "
1080                             "(0x%x)\n",dev->number,(reason & uPD98401_INT_MF)
1081                             >> uPD98401_INT_MF_SHIFT);
1082                         event_dump();
1083                             /* @@@ should try to recover */
1084                 }
1085                 if (reason & uPD98401_INT_MM) {
1086                         if (reason & 1) poll_rx(dev,0);
1087                         if (reason & 2) poll_rx(dev,1);
1088                         if (reason & 4) poll_tx(dev,2);
1089                         if (reason & 8) poll_tx(dev,3);
1090                 }
1091                 /* @@@ handle RCRn */
1092         }
1093         return IRQ_RETVAL(handled);
1094 }
1095
1096
1097 /*----------------------------- (E)EPROM access -----------------------------*/
1098
1099
1100 static void eprom_set(struct zatm_dev *zatm_dev, unsigned long value,
1101                       unsigned short cmd)
1102 {
1103         int error;
1104
1105         if ((error = pci_write_config_dword(zatm_dev->pci_dev,cmd,value)))
1106                 printk(KERN_ERR DEV_LABEL ": PCI write failed (0x%02x)\n",
1107                     error);
1108 }
1109
1110
1111 static unsigned long eprom_get(struct zatm_dev *zatm_dev, unsigned short cmd)
1112 {
1113         unsigned int value;
1114         int error;
1115
1116         if ((error = pci_read_config_dword(zatm_dev->pci_dev,cmd,&value)))
1117                 printk(KERN_ERR DEV_LABEL ": PCI read failed (0x%02x)\n",
1118                     error);
1119         return value;
1120 }
1121
1122
1123 static void eprom_put_bits(struct zatm_dev *zatm_dev, unsigned long data,
1124                            int bits, unsigned short cmd)
1125 {
1126         unsigned long value;
1127         int i;
1128
1129         for (i = bits-1; i >= 0; i--) {
1130                 value = ZEPROM_CS | (((data >> i) & 1) ? ZEPROM_DI : 0);
1131                 eprom_set(zatm_dev,value,cmd);
1132                 eprom_set(zatm_dev,value | ZEPROM_SK,cmd);
1133                 eprom_set(zatm_dev,value,cmd);
1134         }
1135 }
1136
1137
1138 static void eprom_get_byte(struct zatm_dev *zatm_dev, unsigned char *byte,
1139                            unsigned short cmd)
1140 {
1141         int i;
1142
1143         *byte = 0;
1144         for (i = 8; i; i--) {
1145                 eprom_set(zatm_dev,ZEPROM_CS,cmd);
1146                 eprom_set(zatm_dev,ZEPROM_CS | ZEPROM_SK,cmd);
1147                 *byte <<= 1;
1148                 if (eprom_get(zatm_dev,cmd) & ZEPROM_DO) *byte |= 1;
1149                 eprom_set(zatm_dev,ZEPROM_CS,cmd);
1150         }
1151 }
1152
1153
1154 static int eprom_try_esi(struct atm_dev *dev, unsigned short cmd, int offset,
1155                          int swap)
1156 {
1157         unsigned char buf[ZEPROM_SIZE];
1158         struct zatm_dev *zatm_dev;
1159         int i;
1160
1161         zatm_dev = ZATM_DEV(dev);
1162         for (i = 0; i < ZEPROM_SIZE; i += 2) {
1163                 eprom_set(zatm_dev,ZEPROM_CS,cmd); /* select EPROM */
1164                 eprom_put_bits(zatm_dev,ZEPROM_CMD_READ,ZEPROM_CMD_LEN,cmd);
1165                 eprom_put_bits(zatm_dev,i >> 1,ZEPROM_ADDR_LEN,cmd);
1166                 eprom_get_byte(zatm_dev,buf+i+swap,cmd);
1167                 eprom_get_byte(zatm_dev,buf+i+1-swap,cmd);
1168                 eprom_set(zatm_dev,0,cmd); /* deselect EPROM */
1169         }
1170         memcpy(dev->esi,buf+offset,ESI_LEN);
1171         return memcmp(dev->esi,"\0\0\0\0\0",ESI_LEN); /* assumes ESI_LEN == 6 */
1172 }
1173
1174
1175 static void eprom_get_esi(struct atm_dev *dev)
1176 {
1177         if (eprom_try_esi(dev,ZEPROM_V1_REG,ZEPROM_V1_ESI_OFF,1)) return;
1178         (void) eprom_try_esi(dev,ZEPROM_V2_REG,ZEPROM_V2_ESI_OFF,0);
1179 }
1180
1181
1182 /*--------------------------------- entries ---------------------------------*/
1183
1184
1185 static int zatm_init(struct atm_dev *dev)
1186 {
1187         struct zatm_dev *zatm_dev;
1188         struct pci_dev *pci_dev;
1189         unsigned short command;
1190         int error,i,last;
1191         unsigned long t0,t1,t2;
1192
1193         DPRINTK(">zatm_init\n");
1194         zatm_dev = ZATM_DEV(dev);
1195         spin_lock_init(&zatm_dev->lock);
1196         pci_dev = zatm_dev->pci_dev;
1197         zatm_dev->base = pci_resource_start(pci_dev, 0);
1198         zatm_dev->irq = pci_dev->irq;
1199         if ((error = pci_read_config_word(pci_dev,PCI_COMMAND,&command))) {
1200                 printk(KERN_ERR DEV_LABEL "(itf %d): init error 0x%02x\n",
1201                     dev->number,error);
1202                 return -EINVAL;
1203         }
1204         if ((error = pci_write_config_word(pci_dev,PCI_COMMAND,
1205             command | PCI_COMMAND_IO | PCI_COMMAND_MASTER))) {
1206                 printk(KERN_ERR DEV_LABEL "(itf %d): can't enable IO (0x%02x)"
1207                     "\n",dev->number,error);
1208                 return -EIO;
1209         }
1210         eprom_get_esi(dev);
1211         printk(KERN_NOTICE DEV_LABEL "(itf %d): rev.%d,base=0x%x,irq=%d,",
1212             dev->number,pci_dev->revision,zatm_dev->base,zatm_dev->irq);
1213         /* reset uPD98401 */
1214         zout(0,SWR);
1215         while (!(zin(GSR) & uPD98401_INT_IND));
1216         zout(uPD98401_GMR_ONE /*uPD98401_BURST4*/,GMR);
1217         last = MAX_CRAM_SIZE;
1218         for (i = last-RAM_INCREMENT; i >= 0; i -= RAM_INCREMENT) {
1219                 zpokel(zatm_dev,0x55555555,i);
1220                 if (zpeekl(zatm_dev,i) != 0x55555555) last = i;
1221                 else {
1222                         zpokel(zatm_dev,0xAAAAAAAA,i);
1223                         if (zpeekl(zatm_dev,i) != 0xAAAAAAAA) last = i;
1224                         else zpokel(zatm_dev,i,i);
1225                 }
1226         }
1227         for (i = 0; i < last; i += RAM_INCREMENT)
1228                 if (zpeekl(zatm_dev,i) != i) break;
1229         zatm_dev->mem = i << 2;
1230         while (i) zpokel(zatm_dev,0,--i);
1231         /* reset again to rebuild memory pointers */
1232         zout(0,SWR);
1233         while (!(zin(GSR) & uPD98401_INT_IND));
1234         zout(uPD98401_GMR_ONE | uPD98401_BURST8 | uPD98401_BURST4 |
1235             uPD98401_BURST2 | uPD98401_GMR_PM | uPD98401_GMR_DR,GMR);
1236         /* TODO: should shrink allocation now */
1237         printk("mem=%dkB,%s (",zatm_dev->mem >> 10,zatm_dev->copper ? "UTP" :
1238             "MMF");
1239         for (i = 0; i < ESI_LEN; i++)
1240                 printk("%02X%s",dev->esi[i],i == ESI_LEN-1 ? ")\n" : "-");
1241         do {
1242                 unsigned long flags;
1243
1244                 spin_lock_irqsave(&zatm_dev->lock, flags);
1245                 t0 = zpeekl(zatm_dev,uPD98401_TSR);
1246                 udelay(10);
1247                 t1 = zpeekl(zatm_dev,uPD98401_TSR);
1248                 udelay(1010);
1249                 t2 = zpeekl(zatm_dev,uPD98401_TSR);
1250                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
1251         }
1252         while (t0 > t1 || t1 > t2); /* loop if wrapping ... */
1253         zatm_dev->khz = t2-2*t1+t0;
1254         printk(KERN_NOTICE DEV_LABEL "(itf %d): uPD98401 %d.%d at %d.%03d "
1255             "MHz\n",dev->number,
1256             (zin(VER) & uPD98401_MAJOR) >> uPD98401_MAJOR_SHIFT,
1257             zin(VER) & uPD98401_MINOR,zatm_dev->khz/1000,zatm_dev->khz % 1000);
1258         return uPD98402_init(dev);
1259 }
1260
1261
1262 static int zatm_start(struct atm_dev *dev)
1263 {
1264         struct zatm_dev *zatm_dev = ZATM_DEV(dev);
1265         struct pci_dev *pdev = zatm_dev->pci_dev;
1266         unsigned long curr;
1267         int pools,vccs,rx;
1268         int error, i, ld;
1269
1270         DPRINTK("zatm_start\n");
1271         zatm_dev->rx_map = zatm_dev->tx_map = NULL;
1272         for (i = 0; i < NR_MBX; i++)
1273                 zatm_dev->mbx_start[i] = 0;
1274         error = request_irq(zatm_dev->irq, zatm_int, IRQF_SHARED, DEV_LABEL, dev);
1275         if (error < 0) {
1276                 printk(KERN_ERR DEV_LABEL "(itf %d): IRQ%d is already in use\n",
1277                     dev->number,zatm_dev->irq);
1278                 goto done;
1279         }
1280         /* define memory regions */
1281         pools = NR_POOLS;
1282         if (NR_SHAPERS*SHAPER_SIZE > pools*POOL_SIZE)
1283                 pools = NR_SHAPERS*SHAPER_SIZE/POOL_SIZE;
1284         vccs = (zatm_dev->mem-NR_SHAPERS*SHAPER_SIZE-pools*POOL_SIZE)/
1285             (2*VC_SIZE+RX_SIZE);
1286         ld = -1;
1287         for (rx = 1; rx < vccs; rx <<= 1) ld++;
1288         dev->ci_range.vpi_bits = 0; /* @@@ no VPI for now */
1289         dev->ci_range.vci_bits = ld;
1290         dev->link_rate = ATM_OC3_PCR;
1291         zatm_dev->chans = vccs; /* ??? */
1292         curr = rx*RX_SIZE/4;
1293         DPRINTK("RX pool 0x%08lx\n",curr);
1294         zpokel(zatm_dev,curr,uPD98401_PMA); /* receive pool */
1295         zatm_dev->pool_base = curr;
1296         curr += pools*POOL_SIZE/4;
1297         DPRINTK("Shapers 0x%08lx\n",curr);
1298         zpokel(zatm_dev,curr,uPD98401_SMA); /* shapers */
1299         curr += NR_SHAPERS*SHAPER_SIZE/4;
1300         DPRINTK("Free    0x%08lx\n",curr);
1301         zpokel(zatm_dev,curr,uPD98401_TOS); /* free pool */
1302         printk(KERN_INFO DEV_LABEL "(itf %d): %d shapers, %d pools, %d RX, "
1303             "%ld VCs\n",dev->number,NR_SHAPERS,pools,rx,
1304             (zatm_dev->mem-curr*4)/VC_SIZE);
1305         /* create mailboxes */
1306         for (i = 0; i < NR_MBX; i++) {
1307                 void *mbx;
1308                 dma_addr_t mbx_dma;
1309
1310                 if (!mbx_entries[i])
1311                         continue;
1312                 mbx = dma_alloc_coherent(&pdev->dev,
1313                                          2 * MBX_SIZE(i), &mbx_dma, GFP_KERNEL);
1314                 if (!mbx) {
1315                         error = -ENOMEM;
1316                         goto out;
1317                 }
1318                 /*
1319                  * Alignment provided by dma_alloc_coherent() isn't enough
1320                  * for this device.
1321                  */
1322                 if (((unsigned long)mbx ^ mbx_dma) & 0xffff) {
1323                         printk(KERN_ERR DEV_LABEL "(itf %d): system "
1324                                "bus incompatible with driver\n", dev->number);
1325                         dma_free_coherent(&pdev->dev, 2*MBX_SIZE(i), mbx, mbx_dma);
1326                         error = -ENODEV;
1327                         goto out;
1328                 }
1329                 DPRINTK("mbx@0x%08lx-0x%08lx\n", mbx, mbx + MBX_SIZE(i));
1330                 zatm_dev->mbx_start[i] = (unsigned long)mbx;
1331                 zatm_dev->mbx_dma[i] = mbx_dma;
1332                 zatm_dev->mbx_end[i] = (zatm_dev->mbx_start[i] + MBX_SIZE(i)) &
1333                                         0xffff;
1334                 zout(mbx_dma >> 16, MSH(i));
1335                 zout(mbx_dma, MSL(i));
1336                 zout(zatm_dev->mbx_end[i], MBA(i));
1337                 zout((unsigned long)mbx & 0xffff, MTA(i));
1338                 zout((unsigned long)mbx & 0xffff, MWA(i));
1339         }
1340         error = start_tx(dev);
1341         if (error)
1342                 goto out;
1343         error = start_rx(dev);
1344         if (error)
1345                 goto out_tx;
1346         error = dev->phy->start(dev);
1347         if (error)
1348                 goto out_rx;
1349         zout(0xffffffff,IMR); /* enable interrupts */
1350         /* enable TX & RX */
1351         zout(zin(GMR) | uPD98401_GMR_SE | uPD98401_GMR_RE,GMR);
1352 done:
1353         return error;
1354
1355 out_rx:
1356         kfree(zatm_dev->rx_map);
1357 out_tx:
1358         kfree(zatm_dev->tx_map);
1359 out:
1360         while (i-- > 0) {
1361                 dma_free_coherent(&pdev->dev, 2 * MBX_SIZE(i),
1362                                   (void *)zatm_dev->mbx_start[i],
1363                                   zatm_dev->mbx_dma[i]);
1364         }
1365         free_irq(zatm_dev->irq, dev);
1366         goto done;
1367 }
1368
1369
1370 static void zatm_close(struct atm_vcc *vcc)
1371 {
1372         DPRINTK(">zatm_close\n");
1373         if (!ZATM_VCC(vcc)) return;
1374         clear_bit(ATM_VF_READY,&vcc->flags);
1375         close_rx(vcc);
1376         EVENT("close_tx\n",0,0);
1377         close_tx(vcc);
1378         DPRINTK("zatm_close: done waiting\n");
1379         /* deallocate memory */
1380         kfree(ZATM_VCC(vcc));
1381         vcc->dev_data = NULL;
1382         clear_bit(ATM_VF_ADDR,&vcc->flags);
1383 }
1384
1385
1386 static int zatm_open(struct atm_vcc *vcc)
1387 {
1388         struct zatm_vcc *zatm_vcc;
1389         short vpi = vcc->vpi;
1390         int vci = vcc->vci;
1391         int error;
1392
1393         DPRINTK(">zatm_open\n");
1394         if (!test_bit(ATM_VF_PARTIAL,&vcc->flags))
1395                 vcc->dev_data = NULL;
1396         if (vci != ATM_VPI_UNSPEC && vpi != ATM_VCI_UNSPEC)
1397                 set_bit(ATM_VF_ADDR,&vcc->flags);
1398         if (vcc->qos.aal != ATM_AAL5) return -EINVAL; /* @@@ AAL0 */
1399         DPRINTK(DEV_LABEL "(itf %d): open %d.%d\n",vcc->dev->number,vcc->vpi,
1400             vcc->vci);
1401         if (!test_bit(ATM_VF_PARTIAL,&vcc->flags)) {
1402                 zatm_vcc = kmalloc(sizeof(*zatm_vcc), GFP_KERNEL);
1403                 if (!zatm_vcc) {
1404                         clear_bit(ATM_VF_ADDR,&vcc->flags);
1405                         return -ENOMEM;
1406                 }
1407                 vcc->dev_data = zatm_vcc;
1408                 ZATM_VCC(vcc)->tx_chan = 0; /* for zatm_close after open_rx */
1409                 if ((error = open_rx_first(vcc))) {
1410                         zatm_close(vcc);
1411                         return error;
1412                 }
1413                 if ((error = open_tx_first(vcc))) {
1414                         zatm_close(vcc);
1415                         return error;
1416                 }
1417         }
1418         if (vci == ATM_VPI_UNSPEC || vpi == ATM_VCI_UNSPEC) return 0;
1419         if ((error = open_rx_second(vcc))) {
1420                 zatm_close(vcc);
1421                 return error;
1422         }
1423         if ((error = open_tx_second(vcc))) {
1424                 zatm_close(vcc);
1425                 return error;
1426         }
1427         set_bit(ATM_VF_READY,&vcc->flags);
1428         return 0;
1429 }
1430
1431
1432 static int zatm_change_qos(struct atm_vcc *vcc,struct atm_qos *qos,int flags)
1433 {
1434         printk("Not yet implemented\n");
1435         return -ENOSYS;
1436         /* @@@ */
1437 }
1438
1439
1440 static int zatm_ioctl(struct atm_dev *dev,unsigned int cmd,void __user *arg)
1441 {
1442         struct zatm_dev *zatm_dev;
1443         unsigned long flags;
1444
1445         zatm_dev = ZATM_DEV(dev);
1446         switch (cmd) {
1447                 case ZATM_GETPOOLZ:
1448                         if (!capable(CAP_NET_ADMIN)) return -EPERM;
1449                         /* fall through */
1450                 case ZATM_GETPOOL:
1451                         {
1452                                 struct zatm_pool_info info;
1453                                 int pool;
1454
1455                                 if (get_user(pool,
1456                                     &((struct zatm_pool_req __user *) arg)->pool_num))
1457                                         return -EFAULT;
1458                                 if (pool < 0 || pool > ZATM_LAST_POOL)
1459                                         return -EINVAL;
1460                                 pool = array_index_nospec(pool,
1461                                                           ZATM_LAST_POOL + 1);
1462                                 spin_lock_irqsave(&zatm_dev->lock, flags);
1463                                 info = zatm_dev->pool_info[pool];
1464                                 if (cmd == ZATM_GETPOOLZ) {
1465                                         zatm_dev->pool_info[pool].rqa_count = 0;
1466                                         zatm_dev->pool_info[pool].rqu_count = 0;
1467                                 }
1468                                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
1469                                 return copy_to_user(
1470                                     &((struct zatm_pool_req __user *) arg)->info,
1471                                     &info,sizeof(info)) ? -EFAULT : 0;
1472                         }
1473                 case ZATM_SETPOOL:
1474                         {
1475                                 struct zatm_pool_info info;
1476                                 int pool;
1477
1478                                 if (!capable(CAP_NET_ADMIN)) return -EPERM;
1479                                 if (get_user(pool,
1480                                     &((struct zatm_pool_req __user *) arg)->pool_num))
1481                                         return -EFAULT;
1482                                 if (pool < 0 || pool > ZATM_LAST_POOL)
1483                                         return -EINVAL;
1484                                 pool = array_index_nospec(pool,
1485                                                           ZATM_LAST_POOL + 1);
1486                                 if (copy_from_user(&info,
1487                                     &((struct zatm_pool_req __user *) arg)->info,
1488                                     sizeof(info))) return -EFAULT;
1489                                 if (!info.low_water)
1490                                         info.low_water = zatm_dev->
1491                                             pool_info[pool].low_water;
1492                                 if (!info.high_water)
1493                                         info.high_water = zatm_dev->
1494                                             pool_info[pool].high_water;
1495                                 if (!info.next_thres)
1496                                         info.next_thres = zatm_dev->
1497                                             pool_info[pool].next_thres;
1498                                 if (info.low_water >= info.high_water ||
1499                                     info.low_water < 0)
1500                                         return -EINVAL;
1501                                 spin_lock_irqsave(&zatm_dev->lock, flags);
1502                                 zatm_dev->pool_info[pool].low_water =
1503                                     info.low_water;
1504                                 zatm_dev->pool_info[pool].high_water =
1505                                     info.high_water;
1506                                 zatm_dev->pool_info[pool].next_thres =
1507                                     info.next_thres;
1508                                 spin_unlock_irqrestore(&zatm_dev->lock, flags);
1509                                 return 0;
1510                         }
1511                 default:
1512                         if (!dev->phy->ioctl) return -ENOIOCTLCMD;
1513                         return dev->phy->ioctl(dev,cmd,arg);
1514         }
1515 }
1516
1517
1518 static int zatm_getsockopt(struct atm_vcc *vcc,int level,int optname,
1519     void __user *optval,int optlen)
1520 {
1521         return -EINVAL;
1522 }
1523
1524
1525 static int zatm_setsockopt(struct atm_vcc *vcc,int level,int optname,
1526     void __user *optval,unsigned int optlen)
1527 {
1528         return -EINVAL;
1529 }
1530
1531 static int zatm_send(struct atm_vcc *vcc,struct sk_buff *skb)
1532 {
1533         int error;
1534
1535         EVENT(">zatm_send 0x%lx\n",(unsigned long) skb,0);
1536         if (!ZATM_VCC(vcc)->tx_chan || !test_bit(ATM_VF_READY,&vcc->flags)) {
1537                 if (vcc->pop) vcc->pop(vcc,skb);
1538                 else dev_kfree_skb(skb);
1539                 return -EINVAL;
1540         }
1541         if (!skb) {
1542                 printk(KERN_CRIT "!skb in zatm_send ?\n");
1543                 if (vcc->pop) vcc->pop(vcc,skb);
1544                 return -EINVAL;
1545         }
1546         ATM_SKB(skb)->vcc = vcc;
1547         error = do_tx(skb);
1548         if (error != RING_BUSY) return error;
1549         skb_queue_tail(&ZATM_VCC(vcc)->backlog,skb);
1550         return 0;
1551 }
1552
1553
1554 static void zatm_phy_put(struct atm_dev *dev,unsigned char value,
1555     unsigned long addr)
1556 {
1557         struct zatm_dev *zatm_dev;
1558
1559         zatm_dev = ZATM_DEV(dev);
1560         zwait();
1561         zout(value,CER);
1562         zout(uPD98401_IND_ACC | uPD98401_IA_B0 |
1563             (uPD98401_IA_TGT_PHY << uPD98401_IA_TGT_SHIFT) | addr,CMR);
1564 }
1565
1566
1567 static unsigned char zatm_phy_get(struct atm_dev *dev,unsigned long addr)
1568 {
1569         struct zatm_dev *zatm_dev;
1570
1571         zatm_dev = ZATM_DEV(dev);
1572         zwait();
1573         zout(uPD98401_IND_ACC | uPD98401_IA_B0 | uPD98401_IA_RW |
1574           (uPD98401_IA_TGT_PHY << uPD98401_IA_TGT_SHIFT) | addr,CMR);
1575         zwait();
1576         return zin(CER) & 0xff;
1577 }
1578
1579
1580 static const struct atmdev_ops ops = {
1581         .open           = zatm_open,
1582         .close          = zatm_close,
1583         .ioctl          = zatm_ioctl,
1584         .getsockopt     = zatm_getsockopt,
1585         .setsockopt     = zatm_setsockopt,
1586         .send           = zatm_send,
1587         .phy_put        = zatm_phy_put,
1588         .phy_get        = zatm_phy_get,
1589         .change_qos     = zatm_change_qos,
1590 };
1591
1592 static int zatm_init_one(struct pci_dev *pci_dev,
1593                          const struct pci_device_id *ent)
1594 {
1595         struct atm_dev *dev;
1596         struct zatm_dev *zatm_dev;
1597         int ret = -ENOMEM;
1598
1599         zatm_dev = kmalloc(sizeof(*zatm_dev), GFP_KERNEL);
1600         if (!zatm_dev) {
1601                 printk(KERN_EMERG "%s: memory shortage\n", DEV_LABEL);
1602                 goto out;
1603         }
1604
1605         dev = atm_dev_register(DEV_LABEL, &pci_dev->dev, &ops, -1, NULL);
1606         if (!dev)
1607                 goto out_free;
1608
1609         ret = pci_enable_device(pci_dev);
1610         if (ret < 0)
1611                 goto out_deregister;
1612
1613         ret = pci_request_regions(pci_dev, DEV_LABEL);
1614         if (ret < 0)
1615                 goto out_disable;
1616
1617         ret = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32));
1618         if (ret < 0)
1619                 goto out_release;
1620
1621         zatm_dev->pci_dev = pci_dev;
1622         dev->dev_data = zatm_dev;
1623         zatm_dev->copper = (int)ent->driver_data;
1624         if ((ret = zatm_init(dev)) || (ret = zatm_start(dev)))
1625                 goto out_release;
1626
1627         pci_set_drvdata(pci_dev, dev);
1628         zatm_dev->more = zatm_boards;
1629         zatm_boards = dev;
1630         ret = 0;
1631 out:
1632         return ret;
1633
1634 out_release:
1635         pci_release_regions(pci_dev);
1636 out_disable:
1637         pci_disable_device(pci_dev);
1638 out_deregister:
1639         atm_dev_deregister(dev);
1640 out_free:
1641         kfree(zatm_dev);
1642         goto out;
1643 }
1644
1645
1646 MODULE_LICENSE("GPL");
1647
1648 static const struct pci_device_id zatm_pci_tbl[] = {
1649         { PCI_VDEVICE(ZEITNET, PCI_DEVICE_ID_ZEITNET_1221), ZATM_COPPER },
1650         { PCI_VDEVICE(ZEITNET, PCI_DEVICE_ID_ZEITNET_1225), 0 },
1651         { 0, }
1652 };
1653 MODULE_DEVICE_TABLE(pci, zatm_pci_tbl);
1654
1655 static struct pci_driver zatm_driver = {
1656         .name =         DEV_LABEL,
1657         .id_table =     zatm_pci_tbl,
1658         .probe =        zatm_init_one,
1659 };
1660
1661 static int __init zatm_init_module(void)
1662 {
1663         return pci_register_driver(&zatm_driver);
1664 }
1665
1666 module_init(zatm_init_module);
1667 /* module_exit not defined so not unloadable */