drivers/net/ethernet/google/gve/gve_adminq.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6
   7 #include <linux/etherdevice.h>
   8 #include <linux/pci.h>
   9 #include "gve.h"
  10 #include "gve_adminq.h"
  11 #include "gve_register.h"
  12
  13 #define GVE_MAX_ADMINQ_RELEASE_CHECK    500
  14 #define GVE_ADMINQ_SLEEP_LEN            20
  15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK      100
  16
  17 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
  18 {
  19         priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
  20                                           &priv->adminq_bus_addr, GFP_KERNEL);
  21         if (unlikely(!priv->adminq))
  22                 return -ENOMEM;
  23
  24         priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
  25         priv->adminq_prod_cnt = 0;
  26
  27         /* Setup Admin queue with the device */
  28         iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
  29                     &priv->reg_bar0->adminq_pfn);
  30
  31         gve_set_admin_queue_ok(priv);
  32         return 0;
  33 }
  34
  35 void gve_adminq_release(struct gve_priv *priv)
  36 {
  37         int i = 0;
  38
  39         /* Tell the device the adminq is leaving */
  40         iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
  41         while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
  42                 /* If this is reached the device is unrecoverable and still
  43                  * holding memory. Continue looping to avoid memory corruption,
  44                  * but WARN so it is visible what is going on.
  45                  */
  46                 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
  47                         WARN(1, "Unrecoverable platform error!");
  48                 i++;
  49                 msleep(GVE_ADMINQ_SLEEP_LEN);
  50         }
  51         gve_clear_device_rings_ok(priv);
  52         gve_clear_device_resources_ok(priv);
  53         gve_clear_admin_queue_ok(priv);
  54 }
  55
  56 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
  57 {
  58         if (!gve_get_admin_queue_ok(priv))
  59                 return;
  60         gve_adminq_release(priv);
  61         dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
  62         gve_clear_admin_queue_ok(priv);
  63 }
  64
  65 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
  66 {
  67         iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
  68 }
  69
  70 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
  71 {
  72         int i;
  73
  74         for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
  75                 if (ioread32be(&priv->reg_bar0->adminq_event_counter)
  76                     == prod_cnt)
  77                         return true;
  78                 msleep(GVE_ADMINQ_SLEEP_LEN);
  79         }
  80
  81         return false;
  82 }
  83
  84 static int gve_adminq_parse_err(struct device *dev, u32 status)
  85 {
  86         if (status != GVE_ADMINQ_COMMAND_PASSED &&
  87             status != GVE_ADMINQ_COMMAND_UNSET)
  88                 dev_err(dev, "AQ command failed with status %d\n", status);
  89
  90         switch (status) {
  91         case GVE_ADMINQ_COMMAND_PASSED:
  92                 return 0;
  93         case GVE_ADMINQ_COMMAND_UNSET:
  94                 dev_err(dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
  95                 return -EINVAL;
  96         case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
  97         case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
  98         case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
  99         case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
 100         case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
 101                 return -EAGAIN;
 102         case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
 103         case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
 104         case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
 105         case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
 106         case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
 107         case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
 108                 return -EINVAL;
 109         case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
 110                 return -ETIME;
 111         case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
 112         case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
 113                 return -EACCES;
 114         case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
 115                 return -ENOMEM;
 116         case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
 117                 return -ENOTSUPP;
 118         default:
 119                 dev_err(dev, "parse_aq_err: unknown status code %d\n", status);
 120                 return -EINVAL;
 121         }
 122 }
 123
 124 /* This function is not threadsafe - the caller is responsible for any
 125  * necessary locks.
 126  */
 127 int gve_adminq_execute_cmd(struct gve_priv *priv,
 128                            union gve_adminq_command *cmd_orig)
 129 {
 130         union gve_adminq_command *cmd;
 131         u32 status = 0;
 132         u32 prod_cnt;
 133
 134         cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
 135         priv->adminq_prod_cnt++;
 136         prod_cnt = priv->adminq_prod_cnt;
 137
 138         memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
 139
 140         gve_adminq_kick_cmd(priv, prod_cnt);
 141         if (!gve_adminq_wait_for_cmd(priv, prod_cnt)) {
 142                 dev_err(&priv->pdev->dev, "AQ command timed out, need to reset AQ\n");
 143                 return -ENOTRECOVERABLE;
 144         }
 145
 146         memcpy(cmd_orig, cmd, sizeof(*cmd));
 147         status = be32_to_cpu(READ_ONCE(cmd->status));
 148         return gve_adminq_parse_err(&priv->pdev->dev, status);
 149 }
 150
 151 /* The device specifies that the management vector can either be the first irq
 152  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
 153  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
 154  * the management vector is first.
 155  *
 156  * gve arranges the msix vectors so that the management vector is last.
 157  */
 158 #define GVE_NTFY_BLK_BASE_MSIX_IDX      0
 159 int gve_adminq_configure_device_resources(struct gve_priv *priv,
 160                                           dma_addr_t counter_array_bus_addr,
 161                                           u32 num_counters,
 162                                           dma_addr_t db_array_bus_addr,
 163                                           u32 num_ntfy_blks)
 164 {
 165         union gve_adminq_command cmd;
 166
 167         memset(&cmd, 0, sizeof(cmd));
 168         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
 169         cmd.configure_device_resources =
 170                 (struct gve_adminq_configure_device_resources) {
 171                 .counter_array = cpu_to_be64(counter_array_bus_addr),
 172                 .num_counters = cpu_to_be32(num_counters),
 173                 .irq_db_addr = cpu_to_be64(db_array_bus_addr),
 174                 .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
 175                 .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
 176                 .ntfy_blk_msix_base_idx =
 177                                         cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
 178         };
 179
 180         return gve_adminq_execute_cmd(priv, &cmd);
 181 }
 182
 183 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
 184 {
 185         union gve_adminq_command cmd;
 186
 187         memset(&cmd, 0, sizeof(cmd));
 188         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
 189
 190         return gve_adminq_execute_cmd(priv, &cmd);
 191 }
 192
 193 int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
 194 {
 195         struct gve_tx_ring *tx = &priv->tx[queue_index];
 196         union gve_adminq_command cmd;
 197
 198         memset(&cmd, 0, sizeof(cmd));
 199         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
 200         cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
 201                 .queue_id = cpu_to_be32(queue_index),
 202                 .reserved = 0,
 203                 .queue_resources_addr = cpu_to_be64(tx->q_resources_bus),
 204                 .tx_ring_addr = cpu_to_be64(tx->bus),
 205                 .queue_page_list_id = cpu_to_be32(tx->tx_fifo.qpl->id),
 206                 .ntfy_id = cpu_to_be32(tx->ntfy_id),
 207         };
 208
 209         return gve_adminq_execute_cmd(priv, &cmd);
 210 }
 211
 212 int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
 213 {
 214         struct gve_rx_ring *rx = &priv->rx[queue_index];
 215         union gve_adminq_command cmd;
 216
 217         memset(&cmd, 0, sizeof(cmd));
 218         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
 219         cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
 220                 .queue_id = cpu_to_be32(queue_index),
 221                 .index = cpu_to_be32(queue_index),
 222                 .reserved = 0,
 223                 .ntfy_id = cpu_to_be32(rx->ntfy_id),
 224                 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
 225                 .rx_desc_ring_addr = cpu_to_be64(rx->desc.bus),
 226                 .rx_data_ring_addr = cpu_to_be64(rx->data.data_bus),
 227                 .queue_page_list_id = cpu_to_be32(rx->data.qpl->id),
 228         };
 229
 230         return gve_adminq_execute_cmd(priv, &cmd);
 231 }
 232
 233 int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
 234 {
 235         union gve_adminq_command cmd;
 236
 237         memset(&cmd, 0, sizeof(cmd));
 238         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
 239         cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
 240                 .queue_id = cpu_to_be32(queue_index),
 241         };
 242
 243         return gve_adminq_execute_cmd(priv, &cmd);
 244 }
 245
 246 int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
 247 {
 248         union gve_adminq_command cmd;
 249
 250         memset(&cmd, 0, sizeof(cmd));
 251         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
 252         cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
 253                 .queue_id = cpu_to_be32(queue_index),
 254         };
 255
 256         return gve_adminq_execute_cmd(priv, &cmd);
 257 }
 258
 259 int gve_adminq_describe_device(struct gve_priv *priv)
 260 {
 261         struct gve_device_descriptor *descriptor;
 262         union gve_adminq_command cmd;
 263         dma_addr_t descriptor_bus;
 264         int err = 0;
 265         u8 *mac;
 266         u16 mtu;
 267
 268         memset(&cmd, 0, sizeof(cmd));
 269         descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
 270                                         &descriptor_bus, GFP_KERNEL);
 271         if (!descriptor)
 272                 return -ENOMEM;
 273         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
 274         cmd.describe_device.device_descriptor_addr =
 275                                                 cpu_to_be64(descriptor_bus);
 276         cmd.describe_device.device_descriptor_version =
 277                         cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
 278         cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
 279
 280         err = gve_adminq_execute_cmd(priv, &cmd);
 281         if (err)
 282                 goto free_device_descriptor;
 283
 284         priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
 285         if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
 286                 netif_err(priv, drv, priv->dev, "Tx desc count %d too low\n",
 287                           priv->tx_desc_cnt);
 288                 err = -EINVAL;
 289                 goto free_device_descriptor;
 290         }
 291         priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
 292         if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
 293             < PAGE_SIZE ||
 294             priv->rx_desc_cnt * sizeof(priv->rx->data.data_ring[0])
 295             < PAGE_SIZE) {
 296                 netif_err(priv, drv, priv->dev, "Rx desc count %d too low\n",
 297                           priv->rx_desc_cnt);
 298                 err = -EINVAL;
 299                 goto free_device_descriptor;
 300         }
 301         priv->max_registered_pages =
 302                                 be64_to_cpu(descriptor->max_registered_pages);
 303         mtu = be16_to_cpu(descriptor->mtu);
 304         if (mtu < ETH_MIN_MTU) {
 305                 netif_err(priv, drv, priv->dev, "MTU %d below minimum MTU\n",
 306                           mtu);
 307                 err = -EINVAL;
 308                 goto free_device_descriptor;
 309         }
 310         priv->dev->max_mtu = mtu;
 311         priv->num_event_counters = be16_to_cpu(descriptor->counters);
 312         ether_addr_copy(priv->dev->dev_addr, descriptor->mac);
 313         mac = descriptor->mac;
 314         netif_info(priv, drv, priv->dev, "MAC addr: %pM\n", mac);
 315         priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
 316         priv->rx_pages_per_qpl = be16_to_cpu(descriptor->rx_pages_per_qpl);
 317         if (priv->rx_pages_per_qpl < priv->rx_desc_cnt) {
 318                 netif_err(priv, drv, priv->dev, "rx_pages_per_qpl cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
 319                           priv->rx_pages_per_qpl);
 320                 priv->rx_desc_cnt = priv->rx_pages_per_qpl;
 321         }
 322         priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
 323
 324 free_device_descriptor:
 325         dma_free_coherent(&priv->pdev->dev, sizeof(*descriptor), descriptor,
 326                           descriptor_bus);
 327         return err;
 328 }
 329
 330 int gve_adminq_register_page_list(struct gve_priv *priv,
 331                                   struct gve_queue_page_list *qpl)
 332 {
 333         struct device *hdev = &priv->pdev->dev;
 334         u32 num_entries = qpl->num_entries;
 335         u32 size = num_entries * sizeof(qpl->page_buses[0]);
 336         union gve_adminq_command cmd;
 337         dma_addr_t page_list_bus;
 338         __be64 *page_list;
 339         int err;
 340         int i;
 341
 342         memset(&cmd, 0, sizeof(cmd));
 343         page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
 344         if (!page_list)
 345                 return -ENOMEM;
 346
 347         for (i = 0; i < num_entries; i++)
 348                 page_list[i] = cpu_to_be64(qpl->page_buses[i]);
 349
 350         cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
 351         cmd.reg_page_list = (struct gve_adminq_register_page_list) {
 352                 .page_list_id = cpu_to_be32(qpl->id),
 353                 .num_pages = cpu_to_be32(num_entries),
 354                 .page_address_list_addr = cpu_to_be64(page_list_bus),
 355         };
 356
 357         err = gve_adminq_execute_cmd(priv, &cmd);
 358         dma_free_coherent(hdev, size, page_list, page_list_bus);
 359         return err;
 360 }
 361
 362 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
 363 {
 364         union gve_adminq_command cmd;
 365
 366         memset(&cmd, 0, sizeof(cmd));
 367         cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
 368         cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
 369                 .page_list_id = cpu_to_be32(page_list_id),
 370         };
 371
 372         return gve_adminq_execute_cmd(priv, &cmd);
 373 }
 374
 375 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
 376 {
 377         union gve_adminq_command cmd;
 378
 379         memset(&cmd, 0, sizeof(cmd));
 380         cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
 381         cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
 382                 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
 383                 .parameter_value = cpu_to_be64(mtu),
 384         };
 385
 386         return gve_adminq_execute_cmd(priv, &cmd);
 387 }