drivers/net/ethernet/intel/igc/igc_ptp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2019 Intel Corporation */
   3
   4 #include "igc.h"
   5
   6 #include <linux/module.h>
   7 #include <linux/device.h>
   8 #include <linux/pci.h>
   9 #include <linux/ptp_classify.h>
  10 #include <linux/clocksource.h>
  11 #include <linux/ktime.h>
  12
  13 #define INCVALUE_MASK           0x7fffffff
  14 #define ISGN                    0x80000000
  15
  16 #define IGC_SYSTIM_OVERFLOW_PERIOD      (HZ * 60 * 9)
  17 #define IGC_PTP_TX_TIMEOUT              (HZ * 15)
  18
  19 /* SYSTIM read access for I225 */
  20 void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts)
  21 {
  22         struct igc_hw *hw = &adapter->hw;
  23         u32 sec, nsec;
  24
  25         /* The timestamp is latched when SYSTIML is read. */
  26         nsec = rd32(IGC_SYSTIML);
  27         sec = rd32(IGC_SYSTIMH);
  28
  29         ts->tv_sec = sec;
  30         ts->tv_nsec = nsec;
  31 }
  32
  33 static void igc_ptp_write_i225(struct igc_adapter *adapter,
  34                                const struct timespec64 *ts)
  35 {
  36         struct igc_hw *hw = &adapter->hw;
  37
  38         wr32(IGC_SYSTIML, ts->tv_nsec);
  39         wr32(IGC_SYSTIMH, ts->tv_sec);
  40 }
  41
  42 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
  43 {
  44         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  45                                                ptp_caps);
  46         struct igc_hw *hw = &igc->hw;
  47         int neg_adj = 0;
  48         u64 rate;
  49         u32 inca;
  50
  51         if (scaled_ppm < 0) {
  52                 neg_adj = 1;
  53                 scaled_ppm = -scaled_ppm;
  54         }
  55         rate = scaled_ppm;
  56         rate <<= 14;
  57         rate = div_u64(rate, 78125);
  58
  59         inca = rate & INCVALUE_MASK;
  60         if (neg_adj)
  61                 inca |= ISGN;
  62
  63         wr32(IGC_TIMINCA, inca);
  64
  65         return 0;
  66 }
  67
  68 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
  69 {
  70         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  71                                                ptp_caps);
  72         struct timespec64 now, then = ns_to_timespec64(delta);
  73         unsigned long flags;
  74
  75         spin_lock_irqsave(&igc->tmreg_lock, flags);
  76
  77         igc_ptp_read(igc, &now);
  78         now = timespec64_add(now, then);
  79         igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
  80
  81         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
  82
  83         return 0;
  84 }
  85
  86 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
  87                                    struct timespec64 *ts,
  88                                    struct ptp_system_timestamp *sts)
  89 {
  90         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  91                                                ptp_caps);
  92         struct igc_hw *hw = &igc->hw;
  93         unsigned long flags;
  94
  95         spin_lock_irqsave(&igc->tmreg_lock, flags);
  96
  97         ptp_read_system_prets(sts);
  98         ts->tv_nsec = rd32(IGC_SYSTIML);
  99         ts->tv_sec = rd32(IGC_SYSTIMH);
 100         ptp_read_system_postts(sts);
 101
 102         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 103
 104         return 0;
 105 }
 106
 107 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
 108                                 const struct timespec64 *ts)
 109 {
 110         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
 111                                                ptp_caps);
 112         unsigned long flags;
 113
 114         spin_lock_irqsave(&igc->tmreg_lock, flags);
 115
 116         igc_ptp_write_i225(igc, ts);
 117
 118         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 119
 120         return 0;
 121 }
 122
 123 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
 124                                        struct ptp_clock_request *rq, int on)
 125 {
 126         return -EOPNOTSUPP;
 127 }
 128
 129 /**
 130  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
 131  * @adapter: board private structure
 132  * @hwtstamps: timestamp structure to update
 133  * @systim: unsigned 64bit system time value
 134  *
 135  * We need to convert the system time value stored in the RX/TXSTMP registers
 136  * into a hwtstamp which can be used by the upper level timestamping functions.
 137  **/
 138 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 139                                        struct skb_shared_hwtstamps *hwtstamps,
 140                                        u64 systim)
 141 {
 142         switch (adapter->hw.mac.type) {
 143         case igc_i225:
 144                 memset(hwtstamps, 0, sizeof(*hwtstamps));
 145                 /* Upper 32 bits contain s, lower 32 bits contain ns. */
 146                 hwtstamps->hwtstamp = ktime_set(systim >> 32,
 147                                                 systim & 0xFFFFFFFF);
 148                 break;
 149         default:
 150                 break;
 151         }
 152 }
 153
 154 /**
 155  * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp
 156  * @q_vector: Pointer to interrupt specific structure
 157  * @va: Pointer to address containing Rx buffer
 158  * @skb: Buffer containing timestamp and packet
 159  *
 160  * This function is meant to retrieve the first timestamp from the
 161  * first buffer of an incoming frame. The value is stored in little
 162  * endian format starting on byte 0. There's a second timestamp
 163  * starting on byte 8.
 164  **/
 165 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va,
 166                          struct sk_buff *skb)
 167 {
 168         struct igc_adapter *adapter = q_vector->adapter;
 169         __le64 *regval = (__le64 *)va;
 170         int adjust = 0;
 171
 172         /* The timestamp is recorded in little endian format.
 173          * DWORD: | 0          | 1           | 2          | 3
 174          * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High
 175          */
 176         igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
 177                                    le64_to_cpu(regval[0]));
 178
 179         /* adjust timestamp for the RX latency based on link speed */
 180         if (adapter->hw.mac.type == igc_i225) {
 181                 switch (adapter->link_speed) {
 182                 case SPEED_10:
 183                         adjust = IGC_I225_RX_LATENCY_10;
 184                         break;
 185                 case SPEED_100:
 186                         adjust = IGC_I225_RX_LATENCY_100;
 187                         break;
 188                 case SPEED_1000:
 189                         adjust = IGC_I225_RX_LATENCY_1000;
 190                         break;
 191                 case SPEED_2500:
 192                         adjust = IGC_I225_RX_LATENCY_2500;
 193                         break;
 194                 }
 195         }
 196         skb_hwtstamps(skb)->hwtstamp =
 197                 ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
 198 }
 199
 200 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
 201 {
 202         struct igc_hw *hw = &adapter->hw;
 203         u32 val;
 204         int i;
 205
 206         wr32(IGC_TSYNCRXCTL, 0);
 207
 208         for (i = 0; i < adapter->num_rx_queues; i++) {
 209                 val = rd32(IGC_SRRCTL(i));
 210                 val &= ~IGC_SRRCTL_TIMESTAMP;
 211                 wr32(IGC_SRRCTL(i), val);
 212         }
 213
 214         val = rd32(IGC_RXPBS);
 215         val &= ~IGC_RXPBS_CFG_TS_EN;
 216         wr32(IGC_RXPBS, val);
 217 }
 218
 219 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 220 {
 221         struct igc_hw *hw = &adapter->hw;
 222         u32 val;
 223         int i;
 224
 225         val = rd32(IGC_RXPBS);
 226         val |= IGC_RXPBS_CFG_TS_EN;
 227         wr32(IGC_RXPBS, val);
 228
 229         for (i = 0; i < adapter->num_rx_queues; i++) {
 230                 val = rd32(IGC_SRRCTL(i));
 231                 /* FIXME: For now, only support retrieving RX timestamps from
 232                  * timer 0.
 233                  */
 234                 val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
 235                        IGC_SRRCTL_TIMESTAMP;
 236                 wr32(IGC_SRRCTL(i), val);
 237         }
 238
 239         val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
 240               IGC_TSYNCRXCTL_RXSYNSIG;
 241         wr32(IGC_TSYNCRXCTL, val);
 242 }
 243
 244 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 245 {
 246         struct igc_hw *hw = &adapter->hw;
 247
 248         wr32(IGC_TSYNCTXCTL, 0);
 249 }
 250
 251 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
 252 {
 253         struct igc_hw *hw = &adapter->hw;
 254
 255         wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
 256
 257         /* Read TXSTMP registers to discard any timestamp previously stored. */
 258         rd32(IGC_TXSTMPL);
 259         rd32(IGC_TXSTMPH);
 260 }
 261
 262 /**
 263  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
 264  * @adapter: networking device structure
 265  * @config: hwtstamp configuration
 266  *
 267  * Return: 0 in case of success, negative errno code otherwise.
 268  */
 269 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
 270                                       struct hwtstamp_config *config)
 271 {
 272         /* reserved for future extensions */
 273         if (config->flags)
 274                 return -EINVAL;
 275
 276         switch (config->tx_type) {
 277         case HWTSTAMP_TX_OFF:
 278                 igc_ptp_disable_tx_timestamp(adapter);
 279                 break;
 280         case HWTSTAMP_TX_ON:
 281                 igc_ptp_enable_tx_timestamp(adapter);
 282                 break;
 283         default:
 284                 return -ERANGE;
 285         }
 286
 287         switch (config->rx_filter) {
 288         case HWTSTAMP_FILTER_NONE:
 289                 igc_ptp_disable_rx_timestamp(adapter);
 290                 break;
 291         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 292         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 293         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 294         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 295         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 296         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 297         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 298         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 299         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 300         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 301         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 302         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 303         case HWTSTAMP_FILTER_NTP_ALL:
 304         case HWTSTAMP_FILTER_ALL:
 305                 igc_ptp_enable_rx_timestamp(adapter);
 306                 config->rx_filter = HWTSTAMP_FILTER_ALL;
 307                 break;
 308         default:
 309                 return -ERANGE;
 310         }
 311
 312         return 0;
 313 }
 314
 315 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
 316 {
 317         struct igc_hw *hw = &adapter->hw;
 318
 319         dev_kfree_skb_any(adapter->ptp_tx_skb);
 320         adapter->ptp_tx_skb = NULL;
 321         adapter->tx_hwtstamp_timeouts++;
 322         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 323         /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
 324         rd32(IGC_TXSTMPH);
 325         netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
 326 }
 327
 328 void igc_ptp_tx_hang(struct igc_adapter *adapter)
 329 {
 330         bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
 331                                               IGC_PTP_TX_TIMEOUT);
 332
 333         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 334                 return;
 335
 336         /* If we haven't received a timestamp within the timeout, it is
 337          * reasonable to assume that it will never occur, so we can unlock the
 338          * timestamp bit when this occurs.
 339          */
 340         if (timeout) {
 341                 cancel_work_sync(&adapter->ptp_tx_work);
 342                 igc_ptp_tx_timeout(adapter);
 343         }
 344 }
 345
 346 /**
 347  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 348  * @adapter: Board private structure
 349  *
 350  * If we were asked to do hardware stamping and such a time stamp is
 351  * available, then it must have been for this skb here because we only
 352  * allow only one such packet into the queue.
 353  */
 354 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
 355 {
 356         struct sk_buff *skb = adapter->ptp_tx_skb;
 357         struct skb_shared_hwtstamps shhwtstamps;
 358         struct igc_hw *hw = &adapter->hw;
 359         int adjust = 0;
 360         u64 regval;
 361
 362         if (WARN_ON_ONCE(!skb))
 363                 return;
 364
 365         regval = rd32(IGC_TXSTMPL);
 366         regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 367         igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
 368
 369         switch (adapter->link_speed) {
 370         case SPEED_10:
 371                 adjust = IGC_I225_TX_LATENCY_10;
 372                 break;
 373         case SPEED_100:
 374                 adjust = IGC_I225_TX_LATENCY_100;
 375                 break;
 376         case SPEED_1000:
 377                 adjust = IGC_I225_TX_LATENCY_1000;
 378                 break;
 379         case SPEED_2500:
 380                 adjust = IGC_I225_TX_LATENCY_2500;
 381                 break;
 382         }
 383
 384         shhwtstamps.hwtstamp =
 385                 ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 386
 387         /* Clear the lock early before calling skb_tstamp_tx so that
 388          * applications are not woken up before the lock bit is clear. We use
 389          * a copy of the skb pointer to ensure other threads can't change it
 390          * while we're notifying the stack.
 391          */
 392         adapter->ptp_tx_skb = NULL;
 393         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 394
 395         /* Notify the stack and free the skb after we've unlocked */
 396         skb_tstamp_tx(skb, &shhwtstamps);
 397         dev_kfree_skb_any(skb);
 398 }
 399
 400 /**
 401  * igc_ptp_tx_work
 402  * @work: pointer to work struct
 403  *
 404  * This work function polls the TSYNCTXCTL valid bit to determine when a
 405  * timestamp has been taken for the current stored skb.
 406  */
 407 static void igc_ptp_tx_work(struct work_struct *work)
 408 {
 409         struct igc_adapter *adapter = container_of(work, struct igc_adapter,
 410                                                    ptp_tx_work);
 411         struct igc_hw *hw = &adapter->hw;
 412         u32 tsynctxctl;
 413
 414         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 415                 return;
 416
 417         tsynctxctl = rd32(IGC_TSYNCTXCTL);
 418         if (WARN_ON_ONCE(!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0)))
 419                 return;
 420
 421         igc_ptp_tx_hwtstamp(adapter);
 422 }
 423
 424 /**
 425  * igc_ptp_set_ts_config - set hardware time stamping config
 426  * @netdev: network interface device structure
 427  * @ifr: interface request data
 428  *
 429  **/
 430 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
 431 {
 432         struct igc_adapter *adapter = netdev_priv(netdev);
 433         struct hwtstamp_config config;
 434         int err;
 435
 436         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 437                 return -EFAULT;
 438
 439         err = igc_ptp_set_timestamp_mode(adapter, &config);
 440         if (err)
 441                 return err;
 442
 443         /* save these settings for future reference */
 444         memcpy(&adapter->tstamp_config, &config,
 445                sizeof(adapter->tstamp_config));
 446
 447         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 448                 -EFAULT : 0;
 449 }
 450
 451 /**
 452  * igc_ptp_get_ts_config - get hardware time stamping config
 453  * @netdev: network interface device structure
 454  * @ifr: interface request data
 455  *
 456  * Get the hwtstamp_config settings to return to the user. Rather than attempt
 457  * to deconstruct the settings from the registers, just return a shadow copy
 458  * of the last known settings.
 459  **/
 460 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
 461 {
 462         struct igc_adapter *adapter = netdev_priv(netdev);
 463         struct hwtstamp_config *config = &adapter->tstamp_config;
 464
 465         return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
 466                 -EFAULT : 0;
 467 }
 468
 469 /**
 470  * igc_ptp_init - Initialize PTP functionality
 471  * @adapter: Board private structure
 472  *
 473  * This function is called at device probe to initialize the PTP
 474  * functionality.
 475  */
 476 void igc_ptp_init(struct igc_adapter *adapter)
 477 {
 478         struct net_device *netdev = adapter->netdev;
 479         struct igc_hw *hw = &adapter->hw;
 480
 481         switch (hw->mac.type) {
 482         case igc_i225:
 483                 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
 484                 adapter->ptp_caps.owner = THIS_MODULE;
 485                 adapter->ptp_caps.max_adj = 62499999;
 486                 adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
 487                 adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
 488                 adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
 489                 adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
 490                 adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
 491                 break;
 492         default:
 493                 adapter->ptp_clock = NULL;
 494                 return;
 495         }
 496
 497         spin_lock_init(&adapter->tmreg_lock);
 498         INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
 499
 500         adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
 501         adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
 502
 503         adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real());
 504         adapter->ptp_reset_start = ktime_get();
 505
 506         adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
 507                                                 &adapter->pdev->dev);
 508         if (IS_ERR(adapter->ptp_clock)) {
 509                 adapter->ptp_clock = NULL;
 510                 netdev_err(netdev, "ptp_clock_register failed\n");
 511         } else if (adapter->ptp_clock) {
 512                 netdev_info(netdev, "PHC added\n");
 513                 adapter->ptp_flags |= IGC_PTP_ENABLED;
 514         }
 515 }
 516
 517 static void igc_ptp_time_save(struct igc_adapter *adapter)
 518 {
 519         igc_ptp_read(adapter, &adapter->prev_ptp_time);
 520         adapter->ptp_reset_start = ktime_get();
 521 }
 522
 523 static void igc_ptp_time_restore(struct igc_adapter *adapter)
 524 {
 525         struct timespec64 ts = adapter->prev_ptp_time;
 526         ktime_t delta;
 527
 528         delta = ktime_sub(ktime_get(), adapter->ptp_reset_start);
 529
 530         timespec64_add_ns(&ts, ktime_to_ns(delta));
 531
 532         igc_ptp_write_i225(adapter, &ts);
 533 }
 534
 535 /**
 536  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
 537  * @adapter: Board private structure
 538  *
 539  * This function stops the overflow check work and PTP Tx timestamp work, and
 540  * will prepare the device for OS suspend.
 541  */
 542 void igc_ptp_suspend(struct igc_adapter *adapter)
 543 {
 544         if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
 545                 return;
 546
 547         cancel_work_sync(&adapter->ptp_tx_work);
 548         dev_kfree_skb_any(adapter->ptp_tx_skb);
 549         adapter->ptp_tx_skb = NULL;
 550         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 551
 552         igc_ptp_time_save(adapter);
 553 }
 554
 555 /**
 556  * igc_ptp_stop - Disable PTP device and stop the overflow check.
 557  * @adapter: Board private structure.
 558  *
 559  * This function stops the PTP support and cancels the delayed work.
 560  **/
 561 void igc_ptp_stop(struct igc_adapter *adapter)
 562 {
 563         igc_ptp_suspend(adapter);
 564
 565         if (adapter->ptp_clock) {
 566                 ptp_clock_unregister(adapter->ptp_clock);
 567                 netdev_info(adapter->netdev, "PHC removed\n");
 568                 adapter->ptp_flags &= ~IGC_PTP_ENABLED;
 569         }
 570 }
 571
 572 /**
 573  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
 574  * @adapter: Board private structure.
 575  *
 576  * This function handles the reset work required to re-enable the PTP device.
 577  **/
 578 void igc_ptp_reset(struct igc_adapter *adapter)
 579 {
 580         struct igc_hw *hw = &adapter->hw;
 581         unsigned long flags;
 582
 583         /* reset the tstamp_config */
 584         igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
 585
 586         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 587
 588         switch (adapter->hw.mac.type) {
 589         case igc_i225:
 590                 wr32(IGC_TSAUXC, 0x0);
 591                 wr32(IGC_TSSDP, 0x0);
 592                 wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
 593                 wr32(IGC_IMS, IGC_IMS_TS);
 594                 break;
 595         default:
 596                 /* No work to do. */
 597                 goto out;
 598         }
 599
 600         /* Re-initialize the timer. */
 601         if (hw->mac.type == igc_i225) {
 602                 igc_ptp_time_restore(adapter);
 603         } else {
 604                 timecounter_init(&adapter->tc, &adapter->cc,
 605                                  ktime_to_ns(ktime_get_real()));
 606         }
 607 out:
 608         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 609
 610         wrfl();
 611 }