drivers/net/ethernet/mediatek/mtk_ppe.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/io.h>
   6 #include <linux/iopoll.h>
   7 #include <linux/etherdevice.h>
   8 #include <linux/platform_device.h>
   9 #include "mtk_ppe.h"
  10 #include "mtk_ppe_regs.h"
  11
  12 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
  13 {
  14         writel(val, ppe->base + reg);
  15 }
  16
  17 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)
  18 {
  19         return readl(ppe->base + reg);
  20 }
  21
  22 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)
  23 {
  24         u32 val;
  25
  26         val = ppe_r32(ppe, reg);
  27         val &= ~mask;
  28         val |= set;
  29         ppe_w32(ppe, reg, val);
  30
  31         return val;
  32 }
  33
  34 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)
  35 {
  36         return ppe_m32(ppe, reg, 0, val);
  37 }
  38
  39 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
  40 {
  41         return ppe_m32(ppe, reg, val, 0);
  42 }
  43
  44 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
  45 {
  46         int ret;
  47         u32 val;
  48
  49         ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,
  50                                  !(val & MTK_PPE_GLO_CFG_BUSY),
  51                                  20, MTK_PPE_WAIT_TIMEOUT_US);
  52
  53         if (ret)
  54                 dev_err(ppe->dev, "PPE table busy");
  55
  56         return ret;
  57 }
  58
  59 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
  60 {
  61         ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
  62         ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
  63 }
  64
  65 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
  66 {
  67         mtk_ppe_cache_clear(ppe);
  68
  69         ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,
  70                 enable * MTK_PPE_CACHE_CTL_EN);
  71 }
  72
  73 static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e)
  74 {
  75         u32 hv1, hv2, hv3;
  76         u32 hash;
  77
  78         switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) {
  79                 case MTK_PPE_PKT_TYPE_BRIDGE:
  80                         hv1 = e->bridge.src_mac_lo;
  81                         hv1 ^= ((e->bridge.src_mac_hi & 0xffff) << 16);
  82                         hv2 = e->bridge.src_mac_hi >> 16;
  83                         hv2 ^= e->bridge.dest_mac_lo;
  84                         hv3 = e->bridge.dest_mac_hi;
  85                         break;
  86                 case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
  87                 case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
  88                         hv1 = e->ipv4.orig.ports;
  89                         hv2 = e->ipv4.orig.dest_ip;
  90                         hv3 = e->ipv4.orig.src_ip;
  91                         break;
  92                 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
  93                 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
  94                         hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3];
  95                         hv1 ^= e->ipv6.ports;
  96
  97                         hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2];
  98                         hv2 ^= e->ipv6.dest_ip[0];
  99
 100                         hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1];
 101                         hv3 ^= e->ipv6.src_ip[0];
 102                         break;
 103                 case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 104                 case MTK_PPE_PKT_TYPE_IPV6_6RD:
 105                 default:
 106                         WARN_ON_ONCE(1);
 107                         return MTK_PPE_HASH_MASK;
 108         }
 109
 110         hash = (hv1 & hv2) | ((~hv1) & hv3);
 111         hash = (hash >> 24) | ((hash & 0xffffff) << 8);
 112         hash ^= hv1 ^ hv2 ^ hv3;
 113         hash ^= hash >> 16;
 114         hash <<= 1;
 115         hash &= MTK_PPE_ENTRIES - 1;
 116
 117         return hash;
 118 }
 119
 120 static inline struct mtk_foe_mac_info *
 121 mtk_foe_entry_l2(struct mtk_foe_entry *entry)
 122 {
 123         int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
 124
 125         if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 126                 return &entry->ipv6.l2;
 127
 128         return &entry->ipv4.l2;
 129 }
 130
 131 static inline u32 *
 132 mtk_foe_entry_ib2(struct mtk_foe_entry *entry)
 133 {
 134         int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
 135
 136         if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 137                 return &entry->ipv6.ib2;
 138
 139         return &entry->ipv4.ib2;
 140 }
 141
 142 int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
 143                           u8 pse_port, u8 *src_mac, u8 *dest_mac)
 144 {
 145         struct mtk_foe_mac_info *l2;
 146         u32 ports_pad, val;
 147
 148         memset(entry, 0, sizeof(*entry));
 149
 150         val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
 151               FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
 152               FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
 153               MTK_FOE_IB1_BIND_TTL |
 154               MTK_FOE_IB1_BIND_CACHE;
 155         entry->ib1 = val;
 156
 157         val = FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) |
 158               FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f) |
 159               FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port);
 160
 161         if (is_multicast_ether_addr(dest_mac))
 162                 val |= MTK_FOE_IB2_MULTICAST;
 163
 164         ports_pad = 0xa5a5a500 | (l4proto & 0xff);
 165         if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
 166                 entry->ipv4.orig.ports = ports_pad;
 167         if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
 168                 entry->ipv6.ports = ports_pad;
 169
 170         if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
 171                 entry->ipv6.ib2 = val;
 172                 l2 = &entry->ipv6.l2;
 173         } else {
 174                 entry->ipv4.ib2 = val;
 175                 l2 = &entry->ipv4.l2;
 176         }
 177
 178         l2->dest_mac_hi = get_unaligned_be32(dest_mac);
 179         l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4);
 180         l2->src_mac_hi = get_unaligned_be32(src_mac);
 181         l2->src_mac_lo = get_unaligned_be16(src_mac + 4);
 182
 183         if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
 184                 l2->etype = ETH_P_IPV6;
 185         else
 186                 l2->etype = ETH_P_IP;
 187
 188         return 0;
 189 }
 190
 191 int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port)
 192 {
 193         u32 *ib2 = mtk_foe_entry_ib2(entry);
 194         u32 val;
 195
 196         val = *ib2;
 197         val &= ~MTK_FOE_IB2_DEST_PORT;
 198         val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
 199         *ib2 = val;
 200
 201         return 0;
 202 }
 203
 204 int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress,
 205                                  __be32 src_addr, __be16 src_port,
 206                                  __be32 dest_addr, __be16 dest_port)
 207 {
 208         int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
 209         struct mtk_ipv4_tuple *t;
 210
 211         switch (type) {
 212         case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
 213                 if (egress) {
 214                         t = &entry->ipv4.new;
 215                         break;
 216                 }
 217                 fallthrough;
 218         case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 219         case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
 220                 t = &entry->ipv4.orig;
 221                 break;
 222         case MTK_PPE_PKT_TYPE_IPV6_6RD:
 223                 entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr);
 224                 entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr);
 225                 return 0;
 226         default:
 227                 WARN_ON_ONCE(1);
 228                 return -EINVAL;
 229         }
 230
 231         t->src_ip = be32_to_cpu(src_addr);
 232         t->dest_ip = be32_to_cpu(dest_addr);
 233
 234         if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
 235                 return 0;
 236
 237         t->src_port = be16_to_cpu(src_port);
 238         t->dest_port = be16_to_cpu(dest_port);
 239
 240         return 0;
 241 }
 242
 243 int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
 244                                  __be32 *src_addr, __be16 src_port,
 245                                  __be32 *dest_addr, __be16 dest_port)
 246 {
 247         int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
 248         u32 *src, *dest;
 249         int i;
 250
 251         switch (type) {
 252         case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 253                 src = entry->dslite.tunnel_src_ip;
 254                 dest = entry->dslite.tunnel_dest_ip;
 255                 break;
 256         case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
 257         case MTK_PPE_PKT_TYPE_IPV6_6RD:
 258                 entry->ipv6.src_port = be16_to_cpu(src_port);
 259                 entry->ipv6.dest_port = be16_to_cpu(dest_port);
 260                 fallthrough;
 261         case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
 262                 src = entry->ipv6.src_ip;
 263                 dest = entry->ipv6.dest_ip;
 264                 break;
 265         default:
 266                 WARN_ON_ONCE(1);
 267                 return -EINVAL;
 268         }
 269
 270         for (i = 0; i < 4; i++)
 271                 src[i] = be32_to_cpu(src_addr[i]);
 272         for (i = 0; i < 4; i++)
 273                 dest[i] = be32_to_cpu(dest_addr[i]);
 274
 275         return 0;
 276 }
 277
 278 int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port)
 279 {
 280         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
 281
 282         l2->etype = BIT(port);
 283
 284         if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER))
 285                 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
 286         else
 287                 l2->etype |= BIT(8);
 288
 289         entry->ib1 &= ~MTK_FOE_IB1_BIND_VLAN_TAG;
 290
 291         return 0;
 292 }
 293
 294 int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid)
 295 {
 296         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
 297
 298         switch (FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, entry->ib1)) {
 299         case 0:
 300                 entry->ib1 |= MTK_FOE_IB1_BIND_VLAN_TAG |
 301                               FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
 302                 l2->vlan1 = vid;
 303                 return 0;
 304         case 1:
 305                 if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) {
 306                         l2->vlan1 = vid;
 307                         l2->etype |= BIT(8);
 308                 } else {
 309                         l2->vlan2 = vid;
 310                         entry->ib1 += FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
 311                 }
 312                 return 0;
 313         default:
 314                 return -ENOSPC;
 315         }
 316 }
 317
 318 int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid)
 319 {
 320         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
 321
 322         if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER) ||
 323             (entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG))
 324                 l2->etype = ETH_P_PPP_SES;
 325
 326         entry->ib1 |= MTK_FOE_IB1_BIND_PPPOE;
 327         l2->pppoe_id = sid;
 328
 329         return 0;
 330 }
 331
 332 static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry)
 333 {
 334         return !(entry->ib1 & MTK_FOE_IB1_STATIC) &&
 335                FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;
 336 }
 337
 338 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
 339                          u16 timestamp)
 340 {
 341         struct mtk_foe_entry *hwe;
 342         u32 hash;
 343
 344         timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
 345         entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
 346         entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
 347
 348         hash = mtk_ppe_hash_entry(entry);
 349         hwe = &ppe->foe_table[hash];
 350         if (!mtk_foe_entry_usable(hwe)) {
 351                 hwe++;
 352                 hash++;
 353
 354                 if (!mtk_foe_entry_usable(hwe))
 355                         return -ENOSPC;
 356         }
 357
 358         memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
 359         wmb();
 360         hwe->ib1 = entry->ib1;
 361
 362         dma_wmb();
 363
 364         mtk_ppe_cache_clear(ppe);
 365
 366         return hash;
 367 }
 368
 369 int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base,
 370                  int version)
 371 {
 372         struct mtk_foe_entry *foe;
 373
 374         /* need to allocate a separate device, since it PPE DMA access is
 375          * not coherent.
 376          */
 377         ppe->base = base;
 378         ppe->dev = dev;
 379         ppe->version = version;
 380
 381         foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe),
 382                                   &ppe->foe_phys, GFP_KERNEL);
 383         if (!foe)
 384                 return -ENOMEM;
 385
 386         ppe->foe_table = foe;
 387
 388         mtk_ppe_debugfs_init(ppe);
 389
 390         return 0;
 391 }
 392
 393 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
 394 {
 395         static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
 396         int i, k;
 397
 398         memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(ppe->foe_table));
 399
 400         if (!IS_ENABLED(CONFIG_SOC_MT7621))
 401                 return;
 402
 403         /* skip all entries that cross the 1024 byte boundary */
 404         for (i = 0; i < MTK_PPE_ENTRIES; i += 128)
 405                 for (k = 0; k < ARRAY_SIZE(skip); k++)
 406                         ppe->foe_table[i + skip[k]].ib1 |= MTK_FOE_IB1_STATIC;
 407 }
 408
 409 int mtk_ppe_start(struct mtk_ppe *ppe)
 410 {
 411         u32 val;
 412
 413         mtk_ppe_init_foe_table(ppe);
 414         ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
 415
 416         val = MTK_PPE_TB_CFG_ENTRY_80B |
 417               MTK_PPE_TB_CFG_AGE_NON_L4 |
 418               MTK_PPE_TB_CFG_AGE_UNBIND |
 419               MTK_PPE_TB_CFG_AGE_TCP |
 420               MTK_PPE_TB_CFG_AGE_UDP |
 421               MTK_PPE_TB_CFG_AGE_TCP_FIN |
 422               FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS,
 423                          MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) |
 424               FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE,
 425                          MTK_PPE_KEEPALIVE_DISABLE) |
 426               FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) |
 427               FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE,
 428                          MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
 429               FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
 430                          MTK_PPE_ENTRIES_SHIFT);
 431         ppe_w32(ppe, MTK_PPE_TB_CFG, val);
 432
 433         ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
 434                 MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6);
 435
 436         mtk_ppe_cache_enable(ppe, true);
 437
 438         val = MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
 439               MTK_PPE_FLOW_CFG_IP4_UDP_FRAG |
 440               MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
 441               MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
 442               MTK_PPE_FLOW_CFG_IP6_6RD |
 443               MTK_PPE_FLOW_CFG_IP4_NAT |
 444               MTK_PPE_FLOW_CFG_IP4_NAPT |
 445               MTK_PPE_FLOW_CFG_IP4_DSLITE |
 446               MTK_PPE_FLOW_CFG_L2_BRIDGE |
 447               MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
 448         ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
 449
 450         val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
 451               FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3);
 452         ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);
 453
 454         val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) |
 455               FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1);
 456         ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);
 457
 458         val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
 459               FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7);
 460         ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);
 461
 462         val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF;
 463         ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);
 464
 465         val = MTK_PPE_BIND_LIMIT1_FULL |
 466               FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1);
 467         ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);
 468
 469         val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) |
 470               FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1);
 471         ppe_w32(ppe, MTK_PPE_BIND_RATE, val);
 472
 473         /* enable PPE */
 474         val = MTK_PPE_GLO_CFG_EN |
 475               MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
 476               MTK_PPE_GLO_CFG_IP4_CS_DROP |
 477               MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
 478         ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
 479
 480         ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
 481
 482         return 0;
 483 }
 484
 485 int mtk_ppe_stop(struct mtk_ppe *ppe)
 486 {
 487         u32 val;
 488         int i;
 489
 490         for (i = 0; i < MTK_PPE_ENTRIES; i++)
 491                 ppe->foe_table[i].ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
 492                                                    MTK_FOE_STATE_INVALID);
 493
 494         mtk_ppe_cache_enable(ppe, false);
 495
 496         /* disable offload engine */
 497         ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);
 498         ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);
 499
 500         /* disable aging */
 501         val = MTK_PPE_TB_CFG_AGE_NON_L4 |
 502               MTK_PPE_TB_CFG_AGE_UNBIND |
 503               MTK_PPE_TB_CFG_AGE_TCP |
 504               MTK_PPE_TB_CFG_AGE_UDP |
 505               MTK_PPE_TB_CFG_AGE_TCP_FIN;
 506         ppe_clear(ppe, MTK_PPE_TB_CFG, val);
 507
 508         return mtk_ppe_wait_busy(ppe);
 509 }