drivers/net/ethernet/google/gve/gve_adminq.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6
   7 #include <linux/etherdevice.h>
   8 #include <linux/pci.h>
   9 #include "gve.h"
  10 #include "gve_adminq.h"
  11 #include "gve_register.h"
  12
  13 #define GVE_MAX_ADMINQ_RELEASE_CHECK    500
  14 #define GVE_ADMINQ_SLEEP_LEN            20
  15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK      100
  16
  17 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
  18 {
  19         priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
  20                                           &priv->adminq_bus_addr, GFP_KERNEL);
  21         if (unlikely(!priv->adminq))
  22                 return -ENOMEM;
  23
  24         priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
  25         priv->adminq_prod_cnt = 0;
  26         priv->adminq_cmd_fail = 0;
  27         priv->adminq_timeouts = 0;
  28         priv->adminq_describe_device_cnt = 0;
  29         priv->adminq_cfg_device_resources_cnt = 0;
  30         priv->adminq_register_page_list_cnt = 0;
  31         priv->adminq_unregister_page_list_cnt = 0;
  32         priv->adminq_create_tx_queue_cnt = 0;
  33         priv->adminq_create_rx_queue_cnt = 0;
  34         priv->adminq_destroy_tx_queue_cnt = 0;
  35         priv->adminq_destroy_rx_queue_cnt = 0;
  36         priv->adminq_dcfg_device_resources_cnt = 0;
  37         priv->adminq_set_driver_parameter_cnt = 0;
  38
  39         /* Setup Admin queue with the device */
  40         iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
  41                     &priv->reg_bar0->adminq_pfn);
  42
  43         gve_set_admin_queue_ok(priv);
  44         return 0;
  45 }
  46
  47 void gve_adminq_release(struct gve_priv *priv)
  48 {
  49         int i = 0;
  50
  51         /* Tell the device the adminq is leaving */
  52         iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
  53         while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
  54                 /* If this is reached the device is unrecoverable and still
  55                  * holding memory. Continue looping to avoid memory corruption,
  56                  * but WARN so it is visible what is going on.
  57                  */
  58                 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
  59                         WARN(1, "Unrecoverable platform error!");
  60                 i++;
  61                 msleep(GVE_ADMINQ_SLEEP_LEN);
  62         }
  63         gve_clear_device_rings_ok(priv);
  64         gve_clear_device_resources_ok(priv);
  65         gve_clear_admin_queue_ok(priv);
  66 }
  67
  68 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
  69 {
  70         if (!gve_get_admin_queue_ok(priv))
  71                 return;
  72         gve_adminq_release(priv);
  73         dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
  74         gve_clear_admin_queue_ok(priv);
  75 }
  76
  77 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
  78 {
  79         iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
  80 }
  81
  82 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
  83 {
  84         int i;
  85
  86         for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
  87                 if (ioread32be(&priv->reg_bar0->adminq_event_counter)
  88                     == prod_cnt)
  89                         return true;
  90                 msleep(GVE_ADMINQ_SLEEP_LEN);
  91         }
  92
  93         return false;
  94 }
  95
  96 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
  97 {
  98         if (status != GVE_ADMINQ_COMMAND_PASSED &&
  99             status != GVE_ADMINQ_COMMAND_UNSET) {
 100                 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
 101                 priv->adminq_cmd_fail++;
 102         }
 103         switch (status) {
 104         case GVE_ADMINQ_COMMAND_PASSED:
 105                 return 0;
 106         case GVE_ADMINQ_COMMAND_UNSET:
 107                 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
 108                 return -EINVAL;
 109         case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
 110         case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
 111         case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
 112         case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
 113         case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
 114                 return -EAGAIN;
 115         case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
 116         case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
 117         case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
 118         case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
 119         case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
 120         case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
 121                 return -EINVAL;
 122         case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
 123                 return -ETIME;
 124         case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
 125         case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
 126                 return -EACCES;
 127         case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
 128                 return -ENOMEM;
 129         case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
 130                 return -ENOTSUPP;
 131         default:
 132                 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
 133                 return -EINVAL;
 134         }
 135 }
 136
 137 /* This function is not threadsafe - the caller is responsible for any
 138  * necessary locks.
 139  */
 140 int gve_adminq_execute_cmd(struct gve_priv *priv,
 141                            union gve_adminq_command *cmd_orig)
 142 {
 143         union gve_adminq_command *cmd;
 144         u32 status = 0;
 145         u32 prod_cnt;
 146         u32 opcode;
 147
 148         cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
 149         priv->adminq_prod_cnt++;
 150         prod_cnt = priv->adminq_prod_cnt;
 151
 152         memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
 153         opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
 154
 155         switch (opcode) {
 156         case GVE_ADMINQ_DESCRIBE_DEVICE:
 157                 priv->adminq_describe_device_cnt++;
 158                 break;
 159         case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
 160                 priv->adminq_cfg_device_resources_cnt++;
 161                 break;
 162         case GVE_ADMINQ_REGISTER_PAGE_LIST:
 163                 priv->adminq_register_page_list_cnt++;
 164                 break;
 165         case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
 166                 priv->adminq_unregister_page_list_cnt++;
 167                 break;
 168         case GVE_ADMINQ_CREATE_TX_QUEUE:
 169                 priv->adminq_create_tx_queue_cnt++;
 170                 break;
 171         case GVE_ADMINQ_CREATE_RX_QUEUE:
 172                 priv->adminq_create_rx_queue_cnt++;
 173                 break;
 174         case GVE_ADMINQ_DESTROY_TX_QUEUE:
 175                 priv->adminq_destroy_tx_queue_cnt++;
 176                 break;
 177         case GVE_ADMINQ_DESTROY_RX_QUEUE:
 178                 priv->adminq_destroy_rx_queue_cnt++;
 179                 break;
 180         case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
 181                 priv->adminq_dcfg_device_resources_cnt++;
 182                 break;
 183         case GVE_ADMINQ_SET_DRIVER_PARAMETER:
 184                 priv->adminq_set_driver_parameter_cnt++;
 185                 break;
 186         default:
 187                 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
 188         }
 189
 190         gve_adminq_kick_cmd(priv, prod_cnt);
 191         if (!gve_adminq_wait_for_cmd(priv, prod_cnt)) {
 192                 dev_err(&priv->pdev->dev, "AQ command timed out, need to reset AQ\n");
 193                 priv->adminq_timeouts++;
 194                 return -ENOTRECOVERABLE;
 195         }
 196
 197         memcpy(cmd_orig, cmd, sizeof(*cmd));
 198         status = be32_to_cpu(READ_ONCE(cmd->status));
 199         return gve_adminq_parse_err(priv, status);
 200 }
 201
 202 /* The device specifies that the management vector can either be the first irq
 203  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
 204  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
 205  * the management vector is first.
 206  *
 207  * gve arranges the msix vectors so that the management vector is last.
 208  */
 209 #define GVE_NTFY_BLK_BASE_MSIX_IDX      0
 210 int gve_adminq_configure_device_resources(struct gve_priv *priv,
 211                                           dma_addr_t counter_array_bus_addr,
 212                                           u32 num_counters,
 213                                           dma_addr_t db_array_bus_addr,
 214                                           u32 num_ntfy_blks)
 215 {
 216         union gve_adminq_command cmd;
 217
 218         memset(&cmd, 0, sizeof(cmd));
 219         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
 220         cmd.configure_device_resources =
 221                 (struct gve_adminq_configure_device_resources) {
 222                 .counter_array = cpu_to_be64(counter_array_bus_addr),
 223                 .num_counters = cpu_to_be32(num_counters),
 224                 .irq_db_addr = cpu_to_be64(db_array_bus_addr),
 225                 .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
 226                 .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
 227                 .ntfy_blk_msix_base_idx =
 228                                         cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
 229         };
 230
 231         return gve_adminq_execute_cmd(priv, &cmd);
 232 }
 233
 234 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
 235 {
 236         union gve_adminq_command cmd;
 237
 238         memset(&cmd, 0, sizeof(cmd));
 239         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
 240
 241         return gve_adminq_execute_cmd(priv, &cmd);
 242 }
 243
 244 int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
 245 {
 246         struct gve_tx_ring *tx = &priv->tx[queue_index];
 247         union gve_adminq_command cmd;
 248
 249         memset(&cmd, 0, sizeof(cmd));
 250         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
 251         cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
 252                 .queue_id = cpu_to_be32(queue_index),
 253                 .reserved = 0,
 254                 .queue_resources_addr = cpu_to_be64(tx->q_resources_bus),
 255                 .tx_ring_addr = cpu_to_be64(tx->bus),
 256                 .queue_page_list_id = cpu_to_be32(tx->tx_fifo.qpl->id),
 257                 .ntfy_id = cpu_to_be32(tx->ntfy_id),
 258         };
 259
 260         return gve_adminq_execute_cmd(priv, &cmd);
 261 }
 262
 263 int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
 264 {
 265         struct gve_rx_ring *rx = &priv->rx[queue_index];
 266         union gve_adminq_command cmd;
 267
 268         memset(&cmd, 0, sizeof(cmd));
 269         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
 270         cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
 271                 .queue_id = cpu_to_be32(queue_index),
 272                 .index = cpu_to_be32(queue_index),
 273                 .reserved = 0,
 274                 .ntfy_id = cpu_to_be32(rx->ntfy_id),
 275                 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
 276                 .rx_desc_ring_addr = cpu_to_be64(rx->desc.bus),
 277                 .rx_data_ring_addr = cpu_to_be64(rx->data.data_bus),
 278                 .queue_page_list_id = cpu_to_be32(rx->data.qpl->id),
 279         };
 280
 281         return gve_adminq_execute_cmd(priv, &cmd);
 282 }
 283
 284 int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
 285 {
 286         union gve_adminq_command cmd;
 287
 288         memset(&cmd, 0, sizeof(cmd));
 289         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
 290         cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
 291                 .queue_id = cpu_to_be32(queue_index),
 292         };
 293
 294         return gve_adminq_execute_cmd(priv, &cmd);
 295 }
 296
 297 int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
 298 {
 299         union gve_adminq_command cmd;
 300
 301         memset(&cmd, 0, sizeof(cmd));
 302         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
 303         cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
 304                 .queue_id = cpu_to_be32(queue_index),
 305         };
 306
 307         return gve_adminq_execute_cmd(priv, &cmd);
 308 }
 309
 310 int gve_adminq_describe_device(struct gve_priv *priv)
 311 {
 312         struct gve_device_descriptor *descriptor;
 313         union gve_adminq_command cmd;
 314         dma_addr_t descriptor_bus;
 315         int err = 0;
 316         u8 *mac;
 317         u16 mtu;
 318
 319         memset(&cmd, 0, sizeof(cmd));
 320         descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
 321                                         &descriptor_bus, GFP_KERNEL);
 322         if (!descriptor)
 323                 return -ENOMEM;
 324         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
 325         cmd.describe_device.device_descriptor_addr =
 326                                                 cpu_to_be64(descriptor_bus);
 327         cmd.describe_device.device_descriptor_version =
 328                         cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
 329         cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
 330
 331         err = gve_adminq_execute_cmd(priv, &cmd);
 332         if (err)
 333                 goto free_device_descriptor;
 334
 335         priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
 336         if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
 337                 netif_err(priv, drv, priv->dev, "Tx desc count %d too low\n",
 338                           priv->tx_desc_cnt);
 339                 err = -EINVAL;
 340                 goto free_device_descriptor;
 341         }
 342         priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
 343         if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
 344             < PAGE_SIZE ||
 345             priv->rx_desc_cnt * sizeof(priv->rx->data.data_ring[0])
 346             < PAGE_SIZE) {
 347                 netif_err(priv, drv, priv->dev, "Rx desc count %d too low\n",
 348                           priv->rx_desc_cnt);
 349                 err = -EINVAL;
 350                 goto free_device_descriptor;
 351         }
 352         priv->max_registered_pages =
 353                                 be64_to_cpu(descriptor->max_registered_pages);
 354         mtu = be16_to_cpu(descriptor->mtu);
 355         if (mtu < ETH_MIN_MTU) {
 356                 netif_err(priv, drv, priv->dev, "MTU %d below minimum MTU\n",
 357                           mtu);
 358                 err = -EINVAL;
 359                 goto free_device_descriptor;
 360         }
 361         priv->dev->max_mtu = mtu;
 362         priv->num_event_counters = be16_to_cpu(descriptor->counters);
 363         ether_addr_copy(priv->dev->dev_addr, descriptor->mac);
 364         mac = descriptor->mac;
 365         netif_info(priv, drv, priv->dev, "MAC addr: %pM\n", mac);
 366         priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
 367         priv->rx_pages_per_qpl = be16_to_cpu(descriptor->rx_pages_per_qpl);
 368         if (priv->rx_pages_per_qpl < priv->rx_desc_cnt) {
 369                 netif_err(priv, drv, priv->dev, "rx_pages_per_qpl cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
 370                           priv->rx_pages_per_qpl);
 371                 priv->rx_desc_cnt = priv->rx_pages_per_qpl;
 372         }
 373         priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
 374
 375 free_device_descriptor:
 376         dma_free_coherent(&priv->pdev->dev, sizeof(*descriptor), descriptor,
 377                           descriptor_bus);
 378         return err;
 379 }
 380
 381 int gve_adminq_register_page_list(struct gve_priv *priv,
 382                                   struct gve_queue_page_list *qpl)
 383 {
 384         struct device *hdev = &priv->pdev->dev;
 385         u32 num_entries = qpl->num_entries;
 386         u32 size = num_entries * sizeof(qpl->page_buses[0]);
 387         union gve_adminq_command cmd;
 388         dma_addr_t page_list_bus;
 389         __be64 *page_list;
 390         int err;
 391         int i;
 392
 393         memset(&cmd, 0, sizeof(cmd));
 394         page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
 395         if (!page_list)
 396                 return -ENOMEM;
 397
 398         for (i = 0; i < num_entries; i++)
 399                 page_list[i] = cpu_to_be64(qpl->page_buses[i]);
 400
 401         cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
 402         cmd.reg_page_list = (struct gve_adminq_register_page_list) {
 403                 .page_list_id = cpu_to_be32(qpl->id),
 404                 .num_pages = cpu_to_be32(num_entries),
 405                 .page_address_list_addr = cpu_to_be64(page_list_bus),
 406         };
 407
 408         err = gve_adminq_execute_cmd(priv, &cmd);
 409         dma_free_coherent(hdev, size, page_list, page_list_bus);
 410         return err;
 411 }
 412
 413 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
 414 {
 415         union gve_adminq_command cmd;
 416
 417         memset(&cmd, 0, sizeof(cmd));
 418         cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
 419         cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
 420                 .page_list_id = cpu_to_be32(page_list_id),
 421         };
 422
 423         return gve_adminq_execute_cmd(priv, &cmd);
 424 }
 425
 426 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
 427 {
 428         union gve_adminq_command cmd;
 429
 430         memset(&cmd, 0, sizeof(cmd));
 431         cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
 432         cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
 433                 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
 434                 .parameter_value = cpu_to_be64(mtu),
 435         };
 436
 437         return gve_adminq_execute_cmd(priv, &cmd);
 438 }