drivers/net/ethernet/intel/igc/igc_ptp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2019 Intel Corporation */
   3
   4 #include "igc.h"
   5
   6 #include <linux/module.h>
   7 #include <linux/device.h>
   8 #include <linux/pci.h>
   9 #include <linux/ptp_classify.h>
  10 #include <linux/clocksource.h>
  11
  12 #define INCVALUE_MASK           0x7fffffff
  13 #define ISGN                    0x80000000
  14
  15 #define IGC_SYSTIM_OVERFLOW_PERIOD      (HZ * 60 * 9)
  16 #define IGC_PTP_TX_TIMEOUT              (HZ * 15)
  17
  18 /* SYSTIM read access for I225 */
  19 static void igc_ptp_read_i225(struct igc_adapter *adapter,
  20                               struct timespec64 *ts)
  21 {
  22         struct igc_hw *hw = &adapter->hw;
  23         u32 sec, nsec;
  24
  25         /* The timestamp latches on lowest register read. For I210/I211, the
  26          * lowest register is SYSTIMR. Since we only need to provide nanosecond
  27          * resolution, we can ignore it.
  28          */
  29         rd32(IGC_SYSTIMR);
  30         nsec = rd32(IGC_SYSTIML);
  31         sec = rd32(IGC_SYSTIMH);
  32
  33         ts->tv_sec = sec;
  34         ts->tv_nsec = nsec;
  35 }
  36
  37 static void igc_ptp_write_i225(struct igc_adapter *adapter,
  38                                const struct timespec64 *ts)
  39 {
  40         struct igc_hw *hw = &adapter->hw;
  41
  42         /* Writing the SYSTIMR register is not necessary as it only
  43          * provides sub-nanosecond resolution.
  44          */
  45         wr32(IGC_SYSTIML, ts->tv_nsec);
  46         wr32(IGC_SYSTIMH, ts->tv_sec);
  47 }
  48
  49 static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
  50 {
  51         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  52                                                ptp_caps);
  53         struct igc_hw *hw = &igc->hw;
  54         int neg_adj = 0;
  55         u64 rate;
  56         u32 inca;
  57
  58         if (scaled_ppm < 0) {
  59                 neg_adj = 1;
  60                 scaled_ppm = -scaled_ppm;
  61         }
  62         rate = scaled_ppm;
  63         rate <<= 14;
  64         rate = div_u64(rate, 78125);
  65
  66         inca = rate & INCVALUE_MASK;
  67         if (neg_adj)
  68                 inca |= ISGN;
  69
  70         wr32(IGC_TIMINCA, inca);
  71
  72         return 0;
  73 }
  74
  75 static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
  76 {
  77         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  78                                                ptp_caps);
  79         struct timespec64 now, then = ns_to_timespec64(delta);
  80         unsigned long flags;
  81
  82         spin_lock_irqsave(&igc->tmreg_lock, flags);
  83
  84         igc_ptp_read_i225(igc, &now);
  85         now = timespec64_add(now, then);
  86         igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
  87
  88         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
  89
  90         return 0;
  91 }
  92
  93 static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
  94                                    struct timespec64 *ts,
  95                                    struct ptp_system_timestamp *sts)
  96 {
  97         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  98                                                ptp_caps);
  99         struct igc_hw *hw = &igc->hw;
 100         unsigned long flags;
 101
 102         spin_lock_irqsave(&igc->tmreg_lock, flags);
 103
 104         ptp_read_system_prets(sts);
 105         rd32(IGC_SYSTIMR);
 106         ptp_read_system_postts(sts);
 107         ts->tv_nsec = rd32(IGC_SYSTIML);
 108         ts->tv_sec = rd32(IGC_SYSTIMH);
 109
 110         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 111
 112         return 0;
 113 }
 114
 115 static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
 116                                 const struct timespec64 *ts)
 117 {
 118         struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
 119                                                ptp_caps);
 120         unsigned long flags;
 121
 122         spin_lock_irqsave(&igc->tmreg_lock, flags);
 123
 124         igc_ptp_write_i225(igc, ts);
 125
 126         spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 127
 128         return 0;
 129 }
 130
 131 static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
 132                                        struct ptp_clock_request *rq, int on)
 133 {
 134         return -EOPNOTSUPP;
 135 }
 136
 137 /**
 138  * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
 139  * @adapter: board private structure
 140  * @hwtstamps: timestamp structure to update
 141  * @systim: unsigned 64bit system time value
 142  *
 143  * We need to convert the system time value stored in the RX/TXSTMP registers
 144  * into a hwtstamp which can be used by the upper level timestamping functions.
 145  **/
 146 static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 147                                        struct skb_shared_hwtstamps *hwtstamps,
 148                                        u64 systim)
 149 {
 150         switch (adapter->hw.mac.type) {
 151         case igc_i225:
 152                 memset(hwtstamps, 0, sizeof(*hwtstamps));
 153                 /* Upper 32 bits contain s, lower 32 bits contain ns. */
 154                 hwtstamps->hwtstamp = ktime_set(systim >> 32,
 155                                                 systim & 0xFFFFFFFF);
 156                 break;
 157         default:
 158                 break;
 159         }
 160 }
 161
 162 /**
 163  * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp
 164  * @q_vector: Pointer to interrupt specific structure
 165  * @va: Pointer to address containing Rx buffer
 166  * @skb: Buffer containing timestamp and packet
 167  *
 168  * This function is meant to retrieve the first timestamp from the
 169  * first buffer of an incoming frame. The value is stored in little
 170  * endian format starting on byte 0. There's a second timestamp
 171  * starting on byte 8.
 172  **/
 173 void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va,
 174                          struct sk_buff *skb)
 175 {
 176         struct igc_adapter *adapter = q_vector->adapter;
 177         __le64 *regval = (__le64 *)va;
 178         int adjust = 0;
 179
 180         /* The timestamp is recorded in little endian format.
 181          * DWORD: | 0          | 1           | 2          | 3
 182          * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High
 183          */
 184         igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
 185                                    le64_to_cpu(regval[0]));
 186
 187         /* adjust timestamp for the RX latency based on link speed */
 188         if (adapter->hw.mac.type == igc_i225) {
 189                 switch (adapter->link_speed) {
 190                 case SPEED_10:
 191                         adjust = IGC_I225_RX_LATENCY_10;
 192                         break;
 193                 case SPEED_100:
 194                         adjust = IGC_I225_RX_LATENCY_100;
 195                         break;
 196                 case SPEED_1000:
 197                         adjust = IGC_I225_RX_LATENCY_1000;
 198                         break;
 199                 case SPEED_2500:
 200                         adjust = IGC_I225_RX_LATENCY_2500;
 201                         break;
 202                 }
 203         }
 204         skb_hwtstamps(skb)->hwtstamp =
 205                 ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
 206 }
 207
 208 static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
 209 {
 210         struct igc_hw *hw = &adapter->hw;
 211         u32 val;
 212         int i;
 213
 214         wr32(IGC_TSYNCRXCTL, 0);
 215
 216         for (i = 0; i < adapter->num_rx_queues; i++) {
 217                 val = rd32(IGC_SRRCTL(i));
 218                 val &= ~IGC_SRRCTL_TIMESTAMP;
 219                 wr32(IGC_SRRCTL(i), val);
 220         }
 221
 222         val = rd32(IGC_RXPBS);
 223         val &= ~IGC_RXPBS_CFG_TS_EN;
 224         wr32(IGC_RXPBS, val);
 225 }
 226
 227 static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 228 {
 229         struct igc_hw *hw = &adapter->hw;
 230         u32 val;
 231         int i;
 232
 233         val = rd32(IGC_RXPBS);
 234         val |= IGC_RXPBS_CFG_TS_EN;
 235         wr32(IGC_RXPBS, val);
 236
 237         for (i = 0; i < adapter->num_rx_queues; i++) {
 238                 val = rd32(IGC_SRRCTL(i));
 239                 /* FIXME: For now, only support retrieving RX timestamps from
 240                  * timer 0.
 241                  */
 242                 val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
 243                        IGC_SRRCTL_TIMESTAMP;
 244                 wr32(IGC_SRRCTL(i), val);
 245         }
 246
 247         val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
 248               IGC_TSYNCRXCTL_RXSYNSIG;
 249         wr32(IGC_TSYNCRXCTL, val);
 250 }
 251
 252 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 253 {
 254         struct igc_hw *hw = &adapter->hw;
 255
 256         wr32(IGC_TSYNCTXCTL, 0);
 257 }
 258
 259 static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
 260 {
 261         struct igc_hw *hw = &adapter->hw;
 262
 263         wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
 264
 265         /* Read TXSTMP registers to discard any timestamp previously stored. */
 266         rd32(IGC_TXSTMPL);
 267         rd32(IGC_TXSTMPH);
 268 }
 269
 270 /**
 271  * igc_ptp_set_timestamp_mode - setup hardware for timestamping
 272  * @adapter: networking device structure
 273  * @config: hwtstamp configuration
 274  *
 275  * Return: 0 in case of success, negative errno code otherwise.
 276  */
 277 static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
 278                                       struct hwtstamp_config *config)
 279 {
 280         /* reserved for future extensions */
 281         if (config->flags)
 282                 return -EINVAL;
 283
 284         switch (config->tx_type) {
 285         case HWTSTAMP_TX_OFF:
 286                 igc_ptp_disable_tx_timestamp(adapter);
 287                 break;
 288         case HWTSTAMP_TX_ON:
 289                 igc_ptp_enable_tx_timestamp(adapter);
 290                 break;
 291         default:
 292                 return -ERANGE;
 293         }
 294
 295         switch (config->rx_filter) {
 296         case HWTSTAMP_FILTER_NONE:
 297                 igc_ptp_disable_rx_timestamp(adapter);
 298                 break;
 299         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 300         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 301         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 302         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 303         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 304         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 305         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 306         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 307         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 308         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 309         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 310         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 311         case HWTSTAMP_FILTER_NTP_ALL:
 312         case HWTSTAMP_FILTER_ALL:
 313                 igc_ptp_enable_rx_timestamp(adapter);
 314                 config->rx_filter = HWTSTAMP_FILTER_ALL;
 315                 break;
 316         default:
 317                 return -ERANGE;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static void igc_ptp_tx_timeout(struct igc_adapter *adapter)
 324 {
 325         struct igc_hw *hw = &adapter->hw;
 326
 327         dev_kfree_skb_any(adapter->ptp_tx_skb);
 328         adapter->ptp_tx_skb = NULL;
 329         adapter->tx_hwtstamp_timeouts++;
 330         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 331         /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
 332         rd32(IGC_TXSTMPH);
 333         netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
 334 }
 335
 336 void igc_ptp_tx_hang(struct igc_adapter *adapter)
 337 {
 338         bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
 339                                               IGC_PTP_TX_TIMEOUT);
 340
 341         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 342                 return;
 343
 344         /* If we haven't received a timestamp within the timeout, it is
 345          * reasonable to assume that it will never occur, so we can unlock the
 346          * timestamp bit when this occurs.
 347          */
 348         if (timeout) {
 349                 cancel_work_sync(&adapter->ptp_tx_work);
 350                 igc_ptp_tx_timeout(adapter);
 351         }
 352 }
 353
 354 /**
 355  * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 356  * @adapter: Board private structure
 357  *
 358  * If we were asked to do hardware stamping and such a time stamp is
 359  * available, then it must have been for this skb here because we only
 360  * allow only one such packet into the queue.
 361  */
 362 static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
 363 {
 364         struct sk_buff *skb = adapter->ptp_tx_skb;
 365         struct skb_shared_hwtstamps shhwtstamps;
 366         struct igc_hw *hw = &adapter->hw;
 367         u64 regval;
 368
 369         if (WARN_ON_ONCE(!skb))
 370                 return;
 371
 372         regval = rd32(IGC_TXSTMPL);
 373         regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 374         igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
 375
 376         /* Clear the lock early before calling skb_tstamp_tx so that
 377          * applications are not woken up before the lock bit is clear. We use
 378          * a copy of the skb pointer to ensure other threads can't change it
 379          * while we're notifying the stack.
 380          */
 381         adapter->ptp_tx_skb = NULL;
 382         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 383
 384         /* Notify the stack and free the skb after we've unlocked */
 385         skb_tstamp_tx(skb, &shhwtstamps);
 386         dev_kfree_skb_any(skb);
 387 }
 388
 389 /**
 390  * igc_ptp_tx_work
 391  * @work: pointer to work struct
 392  *
 393  * This work function polls the TSYNCTXCTL valid bit to determine when a
 394  * timestamp has been taken for the current stored skb.
 395  */
 396 static void igc_ptp_tx_work(struct work_struct *work)
 397 {
 398         struct igc_adapter *adapter = container_of(work, struct igc_adapter,
 399                                                    ptp_tx_work);
 400         struct igc_hw *hw = &adapter->hw;
 401         u32 tsynctxctl;
 402
 403         if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state))
 404                 return;
 405
 406         if (time_is_before_jiffies(adapter->ptp_tx_start +
 407                                    IGC_PTP_TX_TIMEOUT)) {
 408                 igc_ptp_tx_timeout(adapter);
 409                 return;
 410         }
 411
 412         tsynctxctl = rd32(IGC_TSYNCTXCTL);
 413         if (tsynctxctl & IGC_TSYNCTXCTL_VALID)
 414                 igc_ptp_tx_hwtstamp(adapter);
 415         else
 416                 /* reschedule to check later */
 417                 schedule_work(&adapter->ptp_tx_work);
 418 }
 419
 420 /**
 421  * igc_ptp_set_ts_config - set hardware time stamping config
 422  * @netdev: network interface device structure
 423  * @ifreq: interface request data
 424  *
 425  **/
 426 int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
 427 {
 428         struct igc_adapter *adapter = netdev_priv(netdev);
 429         struct hwtstamp_config config;
 430         int err;
 431
 432         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 433                 return -EFAULT;
 434
 435         err = igc_ptp_set_timestamp_mode(adapter, &config);
 436         if (err)
 437                 return err;
 438
 439         /* save these settings for future reference */
 440         memcpy(&adapter->tstamp_config, &config,
 441                sizeof(adapter->tstamp_config));
 442
 443         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 444                 -EFAULT : 0;
 445 }
 446
 447 /**
 448  * igc_ptp_get_ts_config - get hardware time stamping config
 449  * @netdev: network interface device structure
 450  * @ifreq: interface request data
 451  *
 452  * Get the hwtstamp_config settings to return to the user. Rather than attempt
 453  * to deconstruct the settings from the registers, just return a shadow copy
 454  * of the last known settings.
 455  **/
 456 int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
 457 {
 458         struct igc_adapter *adapter = netdev_priv(netdev);
 459         struct hwtstamp_config *config = &adapter->tstamp_config;
 460
 461         return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
 462                 -EFAULT : 0;
 463 }
 464
 465 /**
 466  * igc_ptp_init - Initialize PTP functionality
 467  * @adapter: Board private structure
 468  *
 469  * This function is called at device probe to initialize the PTP
 470  * functionality.
 471  */
 472 void igc_ptp_init(struct igc_adapter *adapter)
 473 {
 474         struct net_device *netdev = adapter->netdev;
 475         struct igc_hw *hw = &adapter->hw;
 476
 477         switch (hw->mac.type) {
 478         case igc_i225:
 479                 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
 480                 adapter->ptp_caps.owner = THIS_MODULE;
 481                 adapter->ptp_caps.max_adj = 62499999;
 482                 adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
 483                 adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
 484                 adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
 485                 adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
 486                 adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
 487                 break;
 488         default:
 489                 adapter->ptp_clock = NULL;
 490                 return;
 491         }
 492
 493         spin_lock_init(&adapter->tmreg_lock);
 494         INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work);
 495
 496         adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
 497         adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
 498
 499         igc_ptp_reset(adapter);
 500
 501         adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
 502                                                 &adapter->pdev->dev);
 503         if (IS_ERR(adapter->ptp_clock)) {
 504                 adapter->ptp_clock = NULL;
 505                 netdev_err(netdev, "ptp_clock_register failed\n");
 506         } else if (adapter->ptp_clock) {
 507                 netdev_info(netdev, "PHC added\n");
 508                 adapter->ptp_flags |= IGC_PTP_ENABLED;
 509         }
 510 }
 511
 512 /**
 513  * igc_ptp_suspend - Disable PTP work items and prepare for suspend
 514  * @adapter: Board private structure
 515  *
 516  * This function stops the overflow check work and PTP Tx timestamp work, and
 517  * will prepare the device for OS suspend.
 518  */
 519 void igc_ptp_suspend(struct igc_adapter *adapter)
 520 {
 521         if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
 522                 return;
 523
 524         cancel_work_sync(&adapter->ptp_tx_work);
 525         dev_kfree_skb_any(adapter->ptp_tx_skb);
 526         adapter->ptp_tx_skb = NULL;
 527         clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
 528 }
 529
 530 /**
 531  * igc_ptp_stop - Disable PTP device and stop the overflow check.
 532  * @adapter: Board private structure.
 533  *
 534  * This function stops the PTP support and cancels the delayed work.
 535  **/
 536 void igc_ptp_stop(struct igc_adapter *adapter)
 537 {
 538         igc_ptp_suspend(adapter);
 539
 540         if (adapter->ptp_clock) {
 541                 ptp_clock_unregister(adapter->ptp_clock);
 542                 netdev_info(adapter->netdev, "PHC removed\n");
 543                 adapter->ptp_flags &= ~IGC_PTP_ENABLED;
 544         }
 545 }
 546
 547 /**
 548  * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
 549  * @adapter: Board private structure.
 550  *
 551  * This function handles the reset work required to re-enable the PTP device.
 552  **/
 553 void igc_ptp_reset(struct igc_adapter *adapter)
 554 {
 555         struct igc_hw *hw = &adapter->hw;
 556         unsigned long flags;
 557
 558         /* reset the tstamp_config */
 559         igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
 560
 561         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 562
 563         switch (adapter->hw.mac.type) {
 564         case igc_i225:
 565                 wr32(IGC_TSAUXC, 0x0);
 566                 wr32(IGC_TSSDP, 0x0);
 567                 wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS);
 568                 wr32(IGC_IMS, IGC_IMS_TS);
 569                 break;
 570         default:
 571                 /* No work to do. */
 572                 goto out;
 573         }
 574
 575         /* Re-initialize the timer. */
 576         if (hw->mac.type == igc_i225) {
 577                 struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real());
 578
 579                 igc_ptp_write_i225(adapter, &ts64);
 580         } else {
 581                 timecounter_init(&adapter->tc, &adapter->cc,
 582                                  ktime_to_ns(ktime_get_real()));
 583         }
 584 out:
 585         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 586
 587         wrfl();
 588 }