net/ipv4/udp_tunnel_nic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (c) 2020 Facebook Inc.
   3
   4 #include <linux/netdevice.h>
   5 #include <linux/slab.h>
   6 #include <linux/types.h>
   7 #include <linux/workqueue.h>
   8 #include <net/udp_tunnel.h>
   9
  10 enum udp_tunnel_nic_table_entry_flags {
  11         UDP_TUNNEL_NIC_ENTRY_ADD        = BIT(0),
  12         UDP_TUNNEL_NIC_ENTRY_DEL        = BIT(1),
  13         UDP_TUNNEL_NIC_ENTRY_OP_FAIL    = BIT(2),
  14         UDP_TUNNEL_NIC_ENTRY_FROZEN     = BIT(3),
  15 };
  16
  17 struct udp_tunnel_nic_table_entry {
  18         __be16 port;
  19         u8 type;
  20         u8 use_cnt;
  21         u8 flags;
  22         u8 hw_priv;
  23 };
  24
  25 /**
  26  * struct udp_tunnel_nic - UDP tunnel port offload state
  27  * @work:       async work for talking to hardware from process context
  28  * @dev:        netdev pointer
  29  * @need_sync:  at least one port start changed
  30  * @need_replay: space was freed, we need a replay of all ports
  31  * @work_pending: @work is currently scheduled
  32  * @n_tables:   number of tables under @entries
  33  * @missed:     bitmap of tables which overflown
  34  * @entries:    table of tables of ports currently offloaded
  35  */
  36 struct udp_tunnel_nic {
  37         struct work_struct work;
  38
  39         struct net_device *dev;
  40
  41         u8 need_sync:1;
  42         u8 need_replay:1;
  43         u8 work_pending:1;
  44
  45         unsigned int n_tables;
  46         unsigned long missed;
  47         struct udp_tunnel_nic_table_entry **entries;
  48 };
  49
  50 /* We ensure all work structs are done using driver state, but not the code.
  51  * We need a workqueue we can flush before module gets removed.
  52  */
  53 static struct workqueue_struct *udp_tunnel_nic_workqueue;
  54
  55 static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
  56 {
  57         switch (type) {
  58         case UDP_TUNNEL_TYPE_VXLAN:
  59                 return "vxlan";
  60         case UDP_TUNNEL_TYPE_GENEVE:
  61                 return "geneve";
  62         case UDP_TUNNEL_TYPE_VXLAN_GPE:
  63                 return "vxlan-gpe";
  64         default:
  65                 return "unknown";
  66         }
  67 }
  68
  69 static bool
  70 udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
  71 {
  72         return entry->use_cnt == 0 && !entry->flags;
  73 }
  74
  75 static bool
  76 udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
  77 {
  78         return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
  79 }
  80
  81 static void
  82 udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
  83 {
  84         if (!udp_tunnel_nic_entry_is_free(entry))
  85                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
  86 }
  87
  88 static void
  89 udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
  90 {
  91         entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
  92 }
  93
  94 static bool
  95 udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
  96 {
  97         return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
  98                                UDP_TUNNEL_NIC_ENTRY_DEL);
  99 }
 100
 101 static void
 102 udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
 103                            struct udp_tunnel_nic_table_entry *entry,
 104                            unsigned int flag)
 105 {
 106         entry->flags |= flag;
 107         utn->need_sync = 1;
 108 }
 109
 110 static void
 111 udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
 112                              struct udp_tunnel_info *ti)
 113 {
 114         memset(ti, 0, sizeof(*ti));
 115         ti->port = entry->port;
 116         ti->type = entry->type;
 117         ti->hw_priv = entry->hw_priv;
 118 }
 119
 120 static bool
 121 udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
 122 {
 123         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 124         unsigned int i, j;
 125
 126         for (i = 0; i < utn->n_tables; i++)
 127                 for (j = 0; j < info->tables[i].n_entries; j++)
 128                         if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 129                                 return false;
 130         return true;
 131 }
 132
 133 static bool
 134 udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 135 {
 136         const struct udp_tunnel_nic_table_info *table;
 137         unsigned int i, j;
 138
 139         if (!utn->missed)
 140                 return false;
 141
 142         for (i = 0; i < utn->n_tables; i++) {
 143                 table = &dev->udp_tunnel_nic_info->tables[i];
 144                 if (!test_bit(i, &utn->missed))
 145                         continue;
 146
 147                 for (j = 0; j < table->n_entries; j++)
 148                         if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 149                                 return true;
 150         }
 151
 152         return false;
 153 }
 154
 155 static void
 156 __udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
 157                           unsigned int idx, struct udp_tunnel_info *ti)
 158 {
 159         struct udp_tunnel_nic_table_entry *entry;
 160         struct udp_tunnel_nic *utn;
 161
 162         utn = dev->udp_tunnel_nic;
 163         entry = &utn->entries[table][idx];
 164
 165         if (entry->use_cnt)
 166                 udp_tunnel_nic_ti_from_entry(entry, ti);
 167 }
 168
 169 static void
 170 __udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
 171                                unsigned int idx, u8 priv)
 172 {
 173         dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
 174 }
 175
 176 static void
 177 udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
 178                                  int err)
 179 {
 180         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 181
 182         WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 183                      entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);
 184
 185         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 186             (!err || (err == -EEXIST && dodgy)))
 187                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;
 188
 189         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
 190             (!err || (err == -ENOENT && dodgy)))
 191                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;
 192
 193         if (!err)
 194                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 195         else
 196                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 197 }
 198
 199 static void
 200 udp_tunnel_nic_device_sync_one(struct net_device *dev,
 201                                struct udp_tunnel_nic *utn,
 202                                unsigned int table, unsigned int idx)
 203 {
 204         struct udp_tunnel_nic_table_entry *entry;
 205         struct udp_tunnel_info ti;
 206         int err;
 207
 208         entry = &utn->entries[table][idx];
 209         if (!udp_tunnel_nic_entry_is_queued(entry))
 210                 return;
 211
 212         udp_tunnel_nic_ti_from_entry(entry, &ti);
 213         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
 214                 err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
 215         else
 216                 err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
 217                                                            &ti);
 218         udp_tunnel_nic_entry_update_done(entry, err);
 219
 220         if (err)
 221                 netdev_warn(dev,
 222                             "UDP tunnel port sync failed port %d type %s: %d\n",
 223                             be16_to_cpu(entry->port),
 224                             udp_tunnel_nic_tunnel_type_name(entry->type),
 225                             err);
 226 }
 227
 228 static void
 229 udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
 230                                    struct udp_tunnel_nic *utn)
 231 {
 232         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 233         unsigned int i, j;
 234
 235         for (i = 0; i < utn->n_tables; i++)
 236                 for (j = 0; j < info->tables[i].n_entries; j++)
 237                         udp_tunnel_nic_device_sync_one(dev, utn, i, j);
 238 }
 239
 240 static void
 241 udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
 242                                     struct udp_tunnel_nic *utn)
 243 {
 244         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 245         unsigned int i, j;
 246         int err;
 247
 248         for (i = 0; i < utn->n_tables; i++) {
 249                 /* Find something that needs sync in this table */
 250                 for (j = 0; j < info->tables[i].n_entries; j++)
 251                         if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
 252                                 break;
 253                 if (j == info->tables[i].n_entries)
 254                         continue;
 255
 256                 err = info->sync_table(dev, i);
 257                 if (err)
 258                         netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
 259                                     i, err);
 260
 261                 for (j = 0; j < info->tables[i].n_entries; j++) {
 262                         struct udp_tunnel_nic_table_entry *entry;
 263
 264                         entry = &utn->entries[i][j];
 265                         if (udp_tunnel_nic_entry_is_queued(entry))
 266                                 udp_tunnel_nic_entry_update_done(entry, err);
 267                 }
 268         }
 269 }
 270
 271 static void
 272 __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 273 {
 274         if (!utn->need_sync)
 275                 return;
 276
 277         if (dev->udp_tunnel_nic_info->sync_table)
 278                 udp_tunnel_nic_device_sync_by_table(dev, utn);
 279         else
 280                 udp_tunnel_nic_device_sync_by_port(dev, utn);
 281
 282         utn->need_sync = 0;
 283         /* Can't replay directly here, in case we come from the tunnel driver's
 284          * notification - trying to replay may deadlock inside tunnel driver.
 285          */
 286         utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
 287 }
 288
 289 static void
 290 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 291 {
 292         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 293         bool may_sleep;
 294
 295         if (!utn->need_sync)
 296                 return;
 297
 298         /* Drivers which sleep in the callback need to update from
 299          * the workqueue, if we come from the tunnel driver's notification.
 300          */
 301         may_sleep = info->flags & UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
 302         if (!may_sleep)
 303                 __udp_tunnel_nic_device_sync(dev, utn);
 304         if (may_sleep || utn->need_replay) {
 305                 queue_work(udp_tunnel_nic_workqueue, &utn->work);
 306                 utn->work_pending = 1;
 307         }
 308 }
 309
 310 static bool
 311 udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
 312                                 struct udp_tunnel_info *ti)
 313 {
 314         return table->tunnel_types & ti->type;
 315 }
 316
 317 static bool
 318 udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
 319                           struct udp_tunnel_info *ti)
 320 {
 321         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 322         unsigned int i;
 323
 324         /* Special case IPv4-only NICs */
 325         if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
 326             ti->sa_family != AF_INET)
 327                 return false;
 328
 329         for (i = 0; i < utn->n_tables; i++)
 330                 if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
 331                         return true;
 332         return false;
 333 }
 334
 335 static int
 336 udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
 337                              struct udp_tunnel_info *ti)
 338 {
 339         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 340         struct udp_tunnel_nic_table_entry *entry;
 341         unsigned int i, j;
 342
 343         for (i = 0; i < utn->n_tables; i++)
 344                 for (j = 0; j < info->tables[i].n_entries; j++) {
 345                         entry = &utn->entries[i][j];
 346
 347                         if (!udp_tunnel_nic_entry_is_free(entry) &&
 348                             entry->port == ti->port &&
 349                             entry->type != ti->type) {
 350                                 __set_bit(i, &utn->missed);
 351                                 return true;
 352                         }
 353                 }
 354         return false;
 355 }
 356
 357 static void
 358 udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
 359                          unsigned int table, unsigned int idx, int use_cnt_adj)
 360 {
 361         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 362         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 363         unsigned int from, to;
 364
 365         /* If not going from used to unused or vice versa - all done.
 366          * For dodgy entries make sure we try to sync again (queue the entry).
 367          */
 368         entry->use_cnt += use_cnt_adj;
 369         if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
 370                 return;
 371
 372         /* Cancel the op before it was sent to the device, if possible,
 373          * otherwise we'd need to take special care to issue commands
 374          * in the same order the ports arrived.
 375          */
 376         if (use_cnt_adj < 0) {
 377                 from = UDP_TUNNEL_NIC_ENTRY_ADD;
 378                 to = UDP_TUNNEL_NIC_ENTRY_DEL;
 379         } else {
 380                 from = UDP_TUNNEL_NIC_ENTRY_DEL;
 381                 to = UDP_TUNNEL_NIC_ENTRY_ADD;
 382         }
 383
 384         if (entry->flags & from) {
 385                 entry->flags &= ~from;
 386                 if (!dodgy)
 387                         return;
 388         }
 389
 390         udp_tunnel_nic_entry_queue(utn, entry, to);
 391 }
 392
 393 static bool
 394 udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
 395                              unsigned int table, unsigned int idx,
 396                              struct udp_tunnel_info *ti, int use_cnt_adj)
 397 {
 398         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 399
 400         if (udp_tunnel_nic_entry_is_free(entry) ||
 401             entry->port != ti->port ||
 402             entry->type != ti->type)
 403                 return false;
 404
 405         if (udp_tunnel_nic_entry_is_frozen(entry))
 406                 return true;
 407
 408         udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
 409         return true;
 410 }
 411
 412 /* Try to find existing matching entry and adjust its use count, instead of
 413  * adding a new one. Returns true if entry was found. In case of delete the
 414  * entry may have gotten removed in the process, in which case it will be
 415  * queued for removal.
 416  */
 417 static bool
 418 udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 419                             struct udp_tunnel_info *ti, int use_cnt_adj)
 420 {
 421         const struct udp_tunnel_nic_table_info *table;
 422         unsigned int i, j;
 423
 424         for (i = 0; i < utn->n_tables; i++) {
 425                 table = &dev->udp_tunnel_nic_info->tables[i];
 426                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 427                         continue;
 428
 429                 for (j = 0; j < table->n_entries; j++)
 430                         if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
 431                                                          use_cnt_adj))
 432                                 return true;
 433         }
 434
 435         return false;
 436 }
 437
 438 static bool
 439 udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 440                             struct udp_tunnel_info *ti)
 441 {
 442         return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
 443 }
 444
 445 static bool
 446 udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 447                             struct udp_tunnel_info *ti)
 448 {
 449         return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
 450 }
 451
 452 static bool
 453 udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
 454                        struct udp_tunnel_info *ti)
 455 {
 456         const struct udp_tunnel_nic_table_info *table;
 457         unsigned int i, j;
 458
 459         for (i = 0; i < utn->n_tables; i++) {
 460                 table = &dev->udp_tunnel_nic_info->tables[i];
 461                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 462                         continue;
 463
 464                 for (j = 0; j < table->n_entries; j++) {
 465                         struct udp_tunnel_nic_table_entry *entry;
 466
 467                         entry = &utn->entries[i][j];
 468                         if (!udp_tunnel_nic_entry_is_free(entry))
 469                                 continue;
 470
 471                         entry->port = ti->port;
 472                         entry->type = ti->type;
 473                         entry->use_cnt = 1;
 474                         udp_tunnel_nic_entry_queue(utn, entry,
 475                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 476                         return true;
 477                 }
 478
 479                 /* The different table may still fit this port in, but there
 480                  * are no devices currently which have multiple tables accepting
 481                  * the same tunnel type, and false positives are okay.
 482                  */
 483                 __set_bit(i, &utn->missed);
 484         }
 485
 486         return false;
 487 }
 488
 489 static void
 490 __udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
 491 {
 492         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 493         struct udp_tunnel_nic *utn;
 494
 495         utn = dev->udp_tunnel_nic;
 496         if (!utn)
 497                 return;
 498         if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
 499                 return;
 500
 501         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 502                 return;
 503
 504         /* It may happen that a tunnel of one type is removed and different
 505          * tunnel type tries to reuse its port before the device was informed.
 506          * Rely on utn->missed to re-add this port later.
 507          */
 508         if (udp_tunnel_nic_has_collision(dev, utn, ti))
 509                 return;
 510
 511         if (!udp_tunnel_nic_add_existing(dev, utn, ti))
 512                 udp_tunnel_nic_add_new(dev, utn, ti);
 513
 514         udp_tunnel_nic_device_sync(dev, utn);
 515 }
 516
 517 static void
 518 __udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
 519 {
 520         struct udp_tunnel_nic *utn;
 521
 522         utn = dev->udp_tunnel_nic;
 523         if (!utn)
 524                 return;
 525
 526         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 527                 return;
 528
 529         udp_tunnel_nic_del_existing(dev, utn, ti);
 530
 531         udp_tunnel_nic_device_sync(dev, utn);
 532 }
 533
 534 static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
 535 {
 536         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 537         struct udp_tunnel_nic *utn;
 538         unsigned int i, j;
 539
 540         ASSERT_RTNL();
 541
 542         utn = dev->udp_tunnel_nic;
 543         if (!utn)
 544                 return;
 545
 546         utn->need_sync = false;
 547         for (i = 0; i < utn->n_tables; i++)
 548                 for (j = 0; j < info->tables[i].n_entries; j++) {
 549                         struct udp_tunnel_nic_table_entry *entry;
 550
 551                         entry = &utn->entries[i][j];
 552
 553                         entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
 554                                           UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
 555                         /* We don't release rtnl across ops */
 556                         WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
 557                         if (!entry->use_cnt)
 558                                 continue;
 559
 560                         udp_tunnel_nic_entry_queue(utn, entry,
 561                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 562                 }
 563
 564         __udp_tunnel_nic_device_sync(dev, utn);
 565 }
 566
 567 static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
 568         .get_port       = __udp_tunnel_nic_get_port,
 569         .set_port_priv  = __udp_tunnel_nic_set_port_priv,
 570         .add_port       = __udp_tunnel_nic_add_port,
 571         .del_port       = __udp_tunnel_nic_del_port,
 572         .reset_ntf      = __udp_tunnel_nic_reset_ntf,
 573 };
 574
 575 static void
 576 udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
 577 {
 578         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 579         unsigned int i, j;
 580
 581         for (i = 0; i < utn->n_tables; i++)
 582                 for (j = 0; j < info->tables[i].n_entries; j++) {
 583                         int adj_cnt = -utn->entries[i][j].use_cnt;
 584
 585                         if (adj_cnt)
 586                                 udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
 587                 }
 588
 589         __udp_tunnel_nic_device_sync(dev, utn);
 590
 591         for (i = 0; i < utn->n_tables; i++)
 592                 memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
 593                                                       sizeof(**utn->entries)));
 594         WARN_ON(utn->need_sync);
 595         utn->need_replay = 0;
 596 }
 597
 598 static void
 599 udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 600 {
 601         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 602         unsigned int i, j;
 603
 604         /* Freeze all the ports we are already tracking so that the replay
 605          * does not double up the refcount.
 606          */
 607         for (i = 0; i < utn->n_tables; i++)
 608                 for (j = 0; j < info->tables[i].n_entries; j++)
 609                         udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
 610         utn->missed = 0;
 611         utn->need_replay = 0;
 612
 613         udp_tunnel_get_rx_info(dev);
 614
 615         for (i = 0; i < utn->n_tables; i++)
 616                 for (j = 0; j < info->tables[i].n_entries; j++)
 617                         udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
 618 }
 619
 620 static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
 621 {
 622         struct udp_tunnel_nic *utn =
 623                 container_of(work, struct udp_tunnel_nic, work);
 624
 625         rtnl_lock();
 626         utn->work_pending = 0;
 627         __udp_tunnel_nic_device_sync(utn->dev, utn);
 628
 629         if (utn->need_replay)
 630                 udp_tunnel_nic_replay(utn->dev, utn);
 631         rtnl_unlock();
 632 }
 633
 634 static struct udp_tunnel_nic *
 635 udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
 636                      unsigned int n_tables)
 637 {
 638         struct udp_tunnel_nic *utn;
 639         unsigned int i;
 640
 641         utn = kzalloc(sizeof(*utn), GFP_KERNEL);
 642         if (!utn)
 643                 return NULL;
 644         utn->n_tables = n_tables;
 645         INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);
 646
 647         utn->entries = kmalloc_array(n_tables, sizeof(void *), GFP_KERNEL);
 648         if (!utn->entries)
 649                 goto err_free_utn;
 650
 651         for (i = 0; i < n_tables; i++) {
 652                 utn->entries[i] = kcalloc(info->tables[i].n_entries,
 653                                           sizeof(*utn->entries[i]), GFP_KERNEL);
 654                 if (!utn->entries[i])
 655                         goto err_free_prev_entries;
 656         }
 657
 658         return utn;
 659
 660 err_free_prev_entries:
 661         while (i--)
 662                 kfree(utn->entries[i]);
 663         kfree(utn->entries);
 664 err_free_utn:
 665         kfree(utn);
 666         return NULL;
 667 }
 668
 669 static int udp_tunnel_nic_register(struct net_device *dev)
 670 {
 671         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 672         struct udp_tunnel_nic *utn;
 673         unsigned int n_tables, i;
 674
 675         BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
 676                      UDP_TUNNEL_NIC_MAX_TABLES);
 677
 678         if (WARN_ON(!info->set_port != !info->unset_port) ||
 679             WARN_ON(!info->set_port == !info->sync_table) ||
 680             WARN_ON(!info->tables[0].n_entries))
 681                 return -EINVAL;
 682
 683         n_tables = 1;
 684         for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
 685                 if (!info->tables[i].n_entries)
 686                         continue;
 687
 688                 n_tables++;
 689                 if (WARN_ON(!info->tables[i - 1].n_entries))
 690                         return -EINVAL;
 691         }
 692
 693         utn = udp_tunnel_nic_alloc(info, n_tables);
 694         if (!utn)
 695                 return -ENOMEM;
 696
 697         utn->dev = dev;
 698         dev_hold(dev);
 699         dev->udp_tunnel_nic = utn;
 700
 701         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 702                 udp_tunnel_get_rx_info(dev);
 703
 704         return 0;
 705 }
 706
 707 static void
 708 udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
 709 {
 710         unsigned int i;
 711
 712         /* Flush before we check work, so we don't waste time adding entries
 713          * from the work which we will boot immediately.
 714          */
 715         udp_tunnel_nic_flush(dev, utn);
 716
 717         /* Wait for the work to be done using the state, netdev core will
 718          * retry unregister until we give up our reference on this device.
 719          */
 720         if (utn->work_pending)
 721                 return;
 722
 723         for (i = 0; i < utn->n_tables; i++)
 724                 kfree(utn->entries[i]);
 725         kfree(utn->entries);
 726         kfree(utn);
 727         dev->udp_tunnel_nic = NULL;
 728         dev_put(dev);
 729 }
 730
 731 static int
 732 udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
 733                                unsigned long event, void *ptr)
 734 {
 735         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 736         const struct udp_tunnel_nic_info *info;
 737         struct udp_tunnel_nic *utn;
 738
 739         info = dev->udp_tunnel_nic_info;
 740         if (!info)
 741                 return NOTIFY_DONE;
 742
 743         if (event == NETDEV_REGISTER) {
 744                 int err;
 745
 746                 err = udp_tunnel_nic_register(dev);
 747                 if (err)
 748                         netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
 749                 return notifier_from_errno(err);
 750         }
 751         /* All other events will need the udp_tunnel_nic state */
 752         utn = dev->udp_tunnel_nic;
 753         if (!utn)
 754                 return NOTIFY_DONE;
 755
 756         if (event == NETDEV_UNREGISTER) {
 757                 udp_tunnel_nic_unregister(dev, utn);
 758                 return NOTIFY_OK;
 759         }
 760
 761         /* All other events only matter if NIC has to be programmed open */
 762         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 763                 return NOTIFY_DONE;
 764
 765         if (event == NETDEV_UP) {
 766                 WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
 767                 udp_tunnel_get_rx_info(dev);
 768                 return NOTIFY_OK;
 769         }
 770         if (event == NETDEV_GOING_DOWN) {
 771                 udp_tunnel_nic_flush(dev, utn);
 772                 return NOTIFY_OK;
 773         }
 774
 775         return NOTIFY_DONE;
 776 }
 777
 778 static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
 779         .notifier_call = udp_tunnel_nic_netdevice_event,
 780 };
 781
 782 static int __init udp_tunnel_nic_init_module(void)
 783 {
 784         int err;
 785
 786         udp_tunnel_nic_workqueue = alloc_workqueue("udp_tunnel_nic", 0, 0);
 787         if (!udp_tunnel_nic_workqueue)
 788                 return -ENOMEM;
 789
 790         rtnl_lock();
 791         udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
 792         rtnl_unlock();
 793
 794         err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 795         if (err)
 796                 goto err_unset_ops;
 797
 798         return 0;
 799
 800 err_unset_ops:
 801         rtnl_lock();
 802         udp_tunnel_nic_ops = NULL;
 803         rtnl_unlock();
 804         destroy_workqueue(udp_tunnel_nic_workqueue);
 805         return err;
 806 }
 807 late_initcall(udp_tunnel_nic_init_module);
 808
 809 static void __exit udp_tunnel_nic_cleanup_module(void)
 810 {
 811         unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 812
 813         rtnl_lock();
 814         udp_tunnel_nic_ops = NULL;
 815         rtnl_unlock();
 816
 817         destroy_workqueue(udp_tunnel_nic_workqueue);
 818 }
 819 module_exit(udp_tunnel_nic_cleanup_module);
 820
 821 MODULE_LICENSE("GPL");