drivers/net/ethernet/intel/igc/igc_ptp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2019 Intel Corporation */
   3
   4 #include "igc.h"
   5
   6 #include <linux/module.h>
   7 #include <linux/device.h>
   8 #include <linux/pci.h>
   9 #include <linux/ptp_classify.h>
  10 #include <linux/clocksource.h>
  11
  12 #define INCVALUE_MASK           0x7fffffff
  13 #define ISGN                    0x80000000
  14
  15 #define IGC_SYSTIM_OVERFLOW_PERIOD      (HZ * 60 * 9)
  16 #define IGC_PTP_TX_TIMEOUT              (HZ * 15)
  17
  18 /* SYSTIM read access for I225 */
  19 static void igc_ptp_read_i225(struct igc_adapter *adapter,
  20                               struct timespec64 *ts)
  21 {
  22         struct igc_hw *hw = &adapter->hw;
  23         u32 sec, nsec;
  24
  25         /* The timestamp latches on lowest register read. For I210/I211, the
  26          * lowest register is SYSTIMR. Since we only need to provide nanosecond
  27          * resolution, we can ignore it.
  28          */
  29         rd32(IGC_SYSTIMR);
  30         nsec = rd32(IGC_SYSTIML);
  31         sec = rd32(IGC_SYSTIMH);
  32
  33         ts->tv_sec = sec;
  34         ts->tv_nsec = nsec;
  35 }
  36
  37 static void igc_ptp_write_i225(struct igc_adapter *adapter,
  38                                const struct timespec64 *ts)
  39 {
  40         struct igc_hw *hw = &adapter->hw;
  41
  42         /* Writing the SYSTIMR register is not necessary as it only
  43          * provides sub-nanosecond resolution.
  44          */
  45         wr32(IGC_SYSTIML, ts->tv_nsec);
  46         wr32(IGC_SYSTIMH, ts->tv_sec);
  47 }
  48
  49 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
  50 {
  51         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  52                                                ptp_caps);
  53         struct igc_hw *hw = &igc->hw;
  54         int neg_adj = 0;
  55         u64 rate;
  56         u32 inca;
  57
  58         if (scaled_ppm < 0) {
  59                 neg_adj = 1;
  60                 scaled_ppm = -scaled_ppm;
  61         }
  62         rate = scaled_ppm;
  63         rate <<= 14;
  64         rate = div_u64(rate, 78125);
  65
  66         inca = rate & INCVALUE_MASK;
  67         if (neg_adj)
  68                 inca |= ISGN;
  69
  70         wr32(IGC_TIMINCA, inca);
  71
  72         return 0;
  73 }
  74
  75 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
  76 {
  77         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  78                                                ptp_caps);
  79         struct timespec64 now, then = ns_to_timespec64(delta);
  80         unsigned long flags;
  81
  82         spin_lock_irqsave(&igc->tmreg_lock, flags);
  83
  84         igc_ptp_read_i225(igc, &now);
  85         now = timespec64_add(now, then);
  86         igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
  87
  88         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
  89
  90         return 0;
  91 }
  92
  93 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
  94                                    struct timespec64 *ts,
  95                                    struct ptp_system_timestamp *sts)
  96 {
  97         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  98                                                ptp_caps);
  99         struct igc_hw *hw = &igc->hw;
 100         unsigned long flags;
 101
 102         spin_lock_irqsave(&igc->tmreg_lock, flags);
 103
 104         ptp_read_system_prets(sts);
 105         rd32(IGC_SYSTIMR);
 106         ptp_read_system_postts(sts);
 107         ts->tv_nsec = rd32(IGC_SYSTIML);
 108         ts->tv_sec = rd32(IGC_SYSTIMH);
 109
 110         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 111
 112         return 0;
 113 }
 114
 115 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
 116                                 const struct timespec64 *ts)
 117 {
 118         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
 119                                                ptp_caps);
 120         unsigned long flags;
 121
 122         spin_lock_irqsave(&igc->tmreg_lock, flags);
 123
 124         igc_ptp_write_i225(igc, ts);
 125
 126         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 127
 128         return 0;
 129 }
 130
 131 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
 132                                        struct ptp_clock_request *rq, int on)
 133 {
 134         return -EOPNOTSUPP;
 135 }
 136
 137 /**
 138  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
 139  * @adapter: board private structure
 140  * @hwtstamps: timestamp structure to update
 141  * @systim: unsigned 64bit system time value
 142  *
 143  * We need to convert the system time value stored in the RX/TXSTMP registers
 144  * into a hwtstamp which can be used by the upper level timestamping functions.
 145  **/
 146 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 147                                        struct skb_shared_hwtstamps *hwtstamps,
 148                                        u64 systim)
 149 {
 150         switch (adapter->hw.mac.type) {
 151         case igc_i225:
 152                 memset(hwtstamps, 0, sizeof(*hwtstamps));
 153                 /* Upper 32 bits contain s, lower 32 bits contain ns. */
 154                 hwtstamps->hwtstamp = ktime_set(systim >> 32,
 155                                                 systim & 0xFFFFFFFF);
 156                 break;
 157         default:
 158                 break;
 159         }
 160 }
 161
 162 /**
 163  * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp
 164  * @q_vector: Pointer to interrupt specific structure
 165  * @va: Pointer to address containing Rx buffer
 166  * @skb: Buffer containing timestamp and packet
 167  *
 168  * This function is meant to retrieve the first timestamp from the
 169  * first buffer of an incoming frame. The value is stored in little
 170  * endian format starting on byte 0. There's a second timestamp
 171  * starting on byte 8.
 172  **/
 173 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va,
 174                          struct sk_buff *skb)
 175 {
 176         struct igc_adapter *adapter = q_vector->adapter;
 177         __le64 *regval = (__le64 *)va;
 178         int adjust = 0;
 179
 180         /* The timestamp is recorded in little endian format.
 181          * DWORD: | 0          | 1           | 2          | 3
 182          * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High
 183          */
 184         igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
 185                                    le64_to_cpu(regval[0]));
 186
 187         /* adjust timestamp for the RX latency based on link speed */
 188         if (adapter->hw.mac.type == igc_i225) {
 189                 switch (adapter->link_speed) {
 190                 case SPEED_10:
 191                         adjust = IGC_I225_RX_LATENCY_10;
 192                         break;
 193                 case SPEED_100:
 194                         adjust = IGC_I225_RX_LATENCY_100;
 195                         break;
 196                 case SPEED_1000:
 197                         adjust = IGC_I225_RX_LATENCY_1000;
 198                         break;
 199                 case SPEED_2500:
 200                         adjust = IGC_I225_RX_LATENCY_2500;
 201                         break;
 202                 }
 203         }
 204         skb_hwtstamps(skb)->hwtstamp =
 205                 ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
 206 }
 207
 208 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
 209 {
 210         struct igc_hw *hw = &adapter->hw;
 211         u32 val;
 212         int i;
 213
 214         wr32(IGC_TSYNCRXCTL, 0);
 215
 216         for (i = 0; i < adapter->num_rx_queues; i++) {
 217                 val = rd32(IGC_SRRCTL(i));
 218                 val &= ~IGC_SRRCTL_TIMESTAMP;
 219                 wr32(IGC_SRRCTL(i), val);
 220         }
 221
 222         val = rd32(IGC_RXPBS);
 223         val &= ~IGC_RXPBS_CFG_TS_EN;
 224         wr32(IGC_RXPBS, val);
 225 }
 226
 227 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 228 {
 229         struct igc_hw *hw = &adapter->hw;
 230         u32 val;
 231         int i;
 232
 233         val = rd32(IGC_RXPBS);
 234         val |= IGC_RXPBS_CFG_TS_EN;
 235         wr32(IGC_RXPBS, val);
 236
 237         for (i = 0; i < adapter->num_rx_queues; i++) {
 238                 val = rd32(IGC_SRRCTL(i));
 239                 /* FIXME: For now, only support retrieving RX timestamps from
 240                  * timer 0.
 241                  */
 242                 val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
 243                        IGC_SRRCTL_TIMESTAMP;
 244                 wr32(IGC_SRRCTL(i), val);
 245         }
 246
 247         val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
 248               IGC_TSYNCRXCTL_RXSYNSIG;
 249         wr32(IGC_TSYNCRXCTL, val);
 250 }
 251
 252 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 253 {
 254         struct igc_hw *hw = &adapter->hw;
 255
 256         wr32(IGC_TSYNCTXCTL, 0);
 257 }
 258
 259 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
 260 {
 261         struct igc_hw *hw = &adapter->hw;
 262
 263         wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
 264
 265         /* Read TXSTMP registers to discard any timestamp previously stored. */
 266         rd32(IGC_TXSTMPL);
 267         rd32(IGC_TXSTMPH);
 268 }
 269
 270 /**
 271  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
 272  * @adapter: networking device structure
 273  * @config: hwtstamp configuration
 274  *
 275  * Return: 0 in case of success, negative errno code otherwise.
 276  */
 277 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
 278                                       struct hwtstamp_config *config)
 279 {
 280         /* reserved for future extensions */
 281         if (config->flags)
 282                 return -EINVAL;
 283
 284         switch (config->tx_type) {
 285         case HWTSTAMP_TX_OFF:
 286                 igc_ptp_disable_tx_timestamp(adapter);
 287                 break;
 288         case HWTSTAMP_TX_ON:
 289                 igc_ptp_enable_tx_timestamp(adapter);
 290                 break;
 291         default:
 292                 return -ERANGE;
 293         }
 294
 295         switch (config->rx_filter) {
 296         case HWTSTAMP_FILTER_NONE:
 297                 igc_ptp_disable_rx_timestamp(adapter);
 298                 break;
 299         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 300         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 301         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 302         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 303         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 304         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 305         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 306         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 307         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 308         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 309         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 310         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 311         case HWTSTAMP_FILTER_NTP_ALL:
 312         case HWTSTAMP_FILTER_ALL:
 313                 igc_ptp_enable_rx_timestamp(adapter);
 314                 config->rx_filter = HWTSTAMP_FILTER_ALL;
 315                 break;
 316         default:
 317                 return -ERANGE;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
 324 {
 325         struct igc_hw *hw = &adapter->hw;
 326
 327         dev_kfree_skb_any(adapter->ptp_tx_skb);
 328         adapter->ptp_tx_skb = NULL;
 329         adapter->tx_hwtstamp_timeouts++;
 330         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 331         /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
 332         rd32(IGC_TXSTMPH);
 333         netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
 334 }
 335
 336 void igc_ptp_tx_hang(struct igc_adapter *adapter)
 337 {
 338         bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
 339                                               IGC_PTP_TX_TIMEOUT);
 340
 341         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 342                 return;
 343
 344         /* If we haven't received a timestamp within the timeout, it is
 345          * reasonable to assume that it will never occur, so we can unlock the
 346          * timestamp bit when this occurs.
 347          */
 348         if (timeout) {
 349                 cancel_work_sync(&adapter->ptp_tx_work);
 350                 igc_ptp_tx_timeout(adapter);
 351         }
 352 }
 353
 354 /**
 355  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 356  * @adapter: Board private structure
 357  *
 358  * If we were asked to do hardware stamping and such a time stamp is
 359  * available, then it must have been for this skb here because we only
 360  * allow only one such packet into the queue.
 361  */
 362 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
 363 {
 364         struct sk_buff *skb = adapter->ptp_tx_skb;
 365         struct skb_shared_hwtstamps shhwtstamps;
 366         struct igc_hw *hw = &adapter->hw;
 367         int adjust = 0;
 368         u64 regval;
 369
 370         if (WARN_ON_ONCE(!skb))
 371                 return;
 372
 373         regval = rd32(IGC_TXSTMPL);
 374         regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 375         igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
 376
 377         switch (adapter->link_speed) {
 378         case SPEED_10:
 379                 adjust = IGC_I225_TX_LATENCY_10;
 380                 break;
 381         case SPEED_100:
 382                 adjust = IGC_I225_TX_LATENCY_100;
 383                 break;
 384         case SPEED_1000:
 385                 adjust = IGC_I225_TX_LATENCY_1000;
 386                 break;
 387         case SPEED_2500:
 388                 adjust = IGC_I225_TX_LATENCY_2500;
 389                 break;
 390         }
 391
 392         shhwtstamps.hwtstamp =
 393                 ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 394
 395         /* Clear the lock early before calling skb_tstamp_tx so that
 396          * applications are not woken up before the lock bit is clear. We use
 397          * a copy of the skb pointer to ensure other threads can't change it
 398          * while we're notifying the stack.
 399          */
 400         adapter->ptp_tx_skb = NULL;
 401         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 402
 403         /* Notify the stack and free the skb after we've unlocked */
 404         skb_tstamp_tx(skb, &shhwtstamps);
 405         dev_kfree_skb_any(skb);
 406 }
 407
 408 /**
 409  * igc_ptp_tx_work
 410  * @work: pointer to work struct
 411  *
 412  * This work function polls the TSYNCTXCTL valid bit to determine when a
 413  * timestamp has been taken for the current stored skb.
 414  */
 415 static void igc_ptp_tx_work(struct work_struct *work)
 416 {
 417         struct igc_adapter *adapter = container_of(work, struct igc_adapter,
 418                                                    ptp_tx_work);
 419         struct igc_hw *hw = &adapter->hw;
 420         u32 tsynctxctl;
 421
 422         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 423                 return;
 424
 425         if (time_is_before_jiffies(adapter->ptp_tx_start +
 426                                    IGC_PTP_TX_TIMEOUT)) {
 427                 igc_ptp_tx_timeout(adapter);
 428                 return;
 429         }
 430
 431         tsynctxctl = rd32(IGC_TSYNCTXCTL);
 432         if (tsynctxctl & IGC_TSYNCTXCTL_VALID)
 433                 igc_ptp_tx_hwtstamp(adapter);
 434         else
 435                 /* reschedule to check later */
 436                 schedule_work(&adapter->ptp_tx_work);
 437 }
 438
 439 /**
 440  * igc_ptp_set_ts_config - set hardware time stamping config
 441  * @netdev: network interface device structure
 442  * @ifreq: interface request data
 443  *
 444  **/
 445 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
 446 {
 447         struct igc_adapter *adapter = netdev_priv(netdev);
 448         struct hwtstamp_config config;
 449         int err;
 450
 451         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 452                 return -EFAULT;
 453
 454         err = igc_ptp_set_timestamp_mode(adapter, &config);
 455         if (err)
 456                 return err;
 457
 458         /* save these settings for future reference */
 459         memcpy(&adapter->tstamp_config, &config,
 460                sizeof(adapter->tstamp_config));
 461
 462         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 463                 -EFAULT : 0;
 464 }
 465
 466 /**
 467  * igc_ptp_get_ts_config - get hardware time stamping config
 468  * @netdev: network interface device structure
 469  * @ifreq: interface request data
 470  *
 471  * Get the hwtstamp_config settings to return to the user. Rather than attempt
 472  * to deconstruct the settings from the registers, just return a shadow copy
 473  * of the last known settings.
 474  **/
 475 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
 476 {
 477         struct igc_adapter *adapter = netdev_priv(netdev);
 478         struct hwtstamp_config *config = &adapter->tstamp_config;
 479
 480         return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
 481                 -EFAULT : 0;
 482 }
 483
 484 /**
 485  * igc_ptp_init - Initialize PTP functionality
 486  * @adapter: Board private structure
 487  *
 488  * This function is called at device probe to initialize the PTP
 489  * functionality.
 490  */
 491 void igc_ptp_init(struct igc_adapter *adapter)
 492 {
 493         struct net_device *netdev = adapter->netdev;
 494         struct igc_hw *hw = &adapter->hw;
 495
 496         switch (hw->mac.type) {
 497         case igc_i225:
 498                 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
 499                 adapter->ptp_caps.owner = THIS_MODULE;
 500                 adapter->ptp_caps.max_adj = 62499999;
 501                 adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
 502                 adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
 503                 adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
 504                 adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
 505                 adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
 506                 break;
 507         default:
 508                 adapter->ptp_clock = NULL;
 509                 return;
 510         }
 511
 512         spin_lock_init(&adapter->tmreg_lock);
 513         INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
 514
 515         adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
 516         adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
 517
 518         adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
 519                                                 &adapter->pdev->dev);
 520         if (IS_ERR(adapter->ptp_clock)) {
 521                 adapter->ptp_clock = NULL;
 522                 netdev_err(netdev, "ptp_clock_register failed\n");
 523         } else if (adapter->ptp_clock) {
 524                 netdev_info(netdev, "PHC added\n");
 525                 adapter->ptp_flags |= IGC_PTP_ENABLED;
 526         }
 527 }
 528
 529 /**
 530  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
 531  * @adapter: Board private structure
 532  *
 533  * This function stops the overflow check work and PTP Tx timestamp work, and
 534  * will prepare the device for OS suspend.
 535  */
 536 void igc_ptp_suspend(struct igc_adapter *adapter)
 537 {
 538         if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
 539                 return;
 540
 541         cancel_work_sync(&adapter->ptp_tx_work);
 542         dev_kfree_skb_any(adapter->ptp_tx_skb);
 543         adapter->ptp_tx_skb = NULL;
 544         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 545 }
 546
 547 /**
 548  * igc_ptp_stop - Disable PTP device and stop the overflow check.
 549  * @adapter: Board private structure.
 550  *
 551  * This function stops the PTP support and cancels the delayed work.
 552  **/
 553 void igc_ptp_stop(struct igc_adapter *adapter)
 554 {
 555         igc_ptp_suspend(adapter);
 556
 557         if (adapter->ptp_clock) {
 558                 ptp_clock_unregister(adapter->ptp_clock);
 559                 netdev_info(adapter->netdev, "PHC removed\n");
 560                 adapter->ptp_flags &= ~IGC_PTP_ENABLED;
 561         }
 562 }
 563
 564 /**
 565  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
 566  * @adapter: Board private structure.
 567  *
 568  * This function handles the reset work required to re-enable the PTP device.
 569  **/
 570 void igc_ptp_reset(struct igc_adapter *adapter)
 571 {
 572         struct igc_hw *hw = &adapter->hw;
 573         unsigned long flags;
 574
 575         /* reset the tstamp_config */
 576         igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
 577
 578         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 579
 580         switch (adapter->hw.mac.type) {
 581         case igc_i225:
 582                 wr32(IGC_TSAUXC, 0x0);
 583                 wr32(IGC_TSSDP, 0x0);
 584                 wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
 585                 wr32(IGC_IMS, IGC_IMS_TS);
 586                 break;
 587         default:
 588                 /* No work to do. */
 589                 goto out;
 590         }
 591
 592         /* Re-initialize the timer. */
 593         if (hw->mac.type == igc_i225) {
 594                 struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real());
 595
 596                 igc_ptp_write_i225(adapter, &ts64);
 597         } else {
 598                 timecounter_init(&adapter->tc, &adapter->cc,
 599                                  ktime_to_ns(ktime_get_real()));
 600         }
 601 out:
 602         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 603
 604         wrfl();
 605 }