drivers/net/ethernet/google/gve/gve_adminq.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2021 Google, Inc.
   5  */
   6
   7 #include <linux/etherdevice.h>
   8 #include <linux/pci.h>
   9 #include "gve.h"
  10 #include "gve_adminq.h"
  11 #include "gve_register.h"
  12
  13 #define GVE_MAX_ADMINQ_RELEASE_CHECK    500
  14 #define GVE_ADMINQ_SLEEP_LEN            20
  15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK      100
  16
  17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
  18 "Expected: length=%d, feature_mask=%x.\n" \
  19 "Actual: length=%d, feature_mask=%x.\n"
  20
  21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
  22
  23 static
  24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
  25                                               struct gve_device_option *option)
  26 {
  27         void *option_end, *descriptor_end;
  28
  29         option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
  30         descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
  31
  32         return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
  33 }
  34
  35 static
  36 void gve_parse_device_option(struct gve_priv *priv,
  37                              struct gve_device_descriptor *device_descriptor,
  38                              struct gve_device_option *option,
  39                              struct gve_device_option_gqi_rda **dev_op_gqi_rda,
  40                              struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
  41                              struct gve_device_option_dqo_rda **dev_op_dqo_rda)
  42 {
  43         u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
  44         u16 option_length = be16_to_cpu(option->option_length);
  45         u16 option_id = be16_to_cpu(option->option_id);
  46
  47         /* If the length or feature mask doesn't match, continue without
  48          * enabling the feature.
  49          */
  50         switch (option_id) {
  51         case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
  52                 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
  53                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
  54                         dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  55                                  "Raw Addressing",
  56                                  GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
  57                                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
  58                                  option_length, req_feat_mask);
  59                         break;
  60                 }
  61
  62                 dev_info(&priv->pdev->dev,
  63                          "Gqi raw addressing device option enabled.\n");
  64                 priv->queue_format = GVE_GQI_RDA_FORMAT;
  65                 break;
  66         case GVE_DEV_OPT_ID_GQI_RDA:
  67                 if (option_length < sizeof(**dev_op_gqi_rda) ||
  68                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
  69                         dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  70                                  "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
  71                                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
  72                                  option_length, req_feat_mask);
  73                         break;
  74                 }
  75
  76                 if (option_length > sizeof(**dev_op_gqi_rda)) {
  77                         dev_warn(&priv->pdev->dev,
  78                                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
  79                 }
  80                 *dev_op_gqi_rda = (void *)(option + 1);
  81                 break;
  82         case GVE_DEV_OPT_ID_GQI_QPL:
  83                 if (option_length < sizeof(**dev_op_gqi_qpl) ||
  84                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
  85                         dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  86                                  "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
  87                                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
  88                                  option_length, req_feat_mask);
  89                         break;
  90                 }
  91
  92                 if (option_length > sizeof(**dev_op_gqi_qpl)) {
  93                         dev_warn(&priv->pdev->dev,
  94                                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
  95                 }
  96                 *dev_op_gqi_qpl = (void *)(option + 1);
  97                 break;
  98         case GVE_DEV_OPT_ID_DQO_RDA:
  99                 if (option_length < sizeof(**dev_op_dqo_rda) ||
 100                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
 101                         dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 102                                  "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
 103                                  GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
 104                                  option_length, req_feat_mask);
 105                         break;
 106                 }
 107
 108                 if (option_length > sizeof(**dev_op_dqo_rda)) {
 109                         dev_warn(&priv->pdev->dev,
 110                                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
 111                 }
 112                 *dev_op_dqo_rda = (void *)(option + 1);
 113                 break;
 114         default:
 115                 /* If we don't recognize the option just continue
 116                  * without doing anything.
 117                  */
 118                 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
 119                         option_id);
 120         }
 121 }
 122
 123 /* Process all device options for a given describe device call. */
 124 static int
 125 gve_process_device_options(struct gve_priv *priv,
 126                            struct gve_device_descriptor *descriptor,
 127                            struct gve_device_option_gqi_rda **dev_op_gqi_rda,
 128                            struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
 129                            struct gve_device_option_dqo_rda **dev_op_dqo_rda)
 130 {
 131         const int num_options = be16_to_cpu(descriptor->num_device_options);
 132         struct gve_device_option *dev_opt;
 133         int i;
 134
 135         /* The options struct directly follows the device descriptor. */
 136         dev_opt = (void *)(descriptor + 1);
 137         for (i = 0; i < num_options; i++) {
 138                 struct gve_device_option *next_opt;
 139
 140                 next_opt = gve_get_next_option(descriptor, dev_opt);
 141                 if (!next_opt) {
 142                         dev_err(&priv->dev->dev,
 143                                 "options exceed device_descriptor's total length.\n");
 144                         return -EINVAL;
 145                 }
 146
 147                 gve_parse_device_option(priv, descriptor, dev_opt,
 148                                         dev_op_gqi_rda, dev_op_gqi_qpl,
 149                                         dev_op_dqo_rda);
 150                 dev_opt = next_opt;
 151         }
 152
 153         return 0;
 154 }
 155
 156 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
 157 {
 158         priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
 159                                           &priv->adminq_bus_addr, GFP_KERNEL);
 160         if (unlikely(!priv->adminq))
 161                 return -ENOMEM;
 162
 163         priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
 164         priv->adminq_prod_cnt = 0;
 165         priv->adminq_cmd_fail = 0;
 166         priv->adminq_timeouts = 0;
 167         priv->adminq_describe_device_cnt = 0;
 168         priv->adminq_cfg_device_resources_cnt = 0;
 169         priv->adminq_register_page_list_cnt = 0;
 170         priv->adminq_unregister_page_list_cnt = 0;
 171         priv->adminq_create_tx_queue_cnt = 0;
 172         priv->adminq_create_rx_queue_cnt = 0;
 173         priv->adminq_destroy_tx_queue_cnt = 0;
 174         priv->adminq_destroy_rx_queue_cnt = 0;
 175         priv->adminq_dcfg_device_resources_cnt = 0;
 176         priv->adminq_set_driver_parameter_cnt = 0;
 177         priv->adminq_report_stats_cnt = 0;
 178         priv->adminq_report_link_speed_cnt = 0;
 179         priv->adminq_get_ptype_map_cnt = 0;
 180
 181         /* Setup Admin queue with the device */
 182         iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
 183                     &priv->reg_bar0->adminq_pfn);
 184
 185         gve_set_admin_queue_ok(priv);
 186         return 0;
 187 }
 188
 189 void gve_adminq_release(struct gve_priv *priv)
 190 {
 191         int i = 0;
 192
 193         /* Tell the device the adminq is leaving */
 194         iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
 195         while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
 196                 /* If this is reached the device is unrecoverable and still
 197                  * holding memory. Continue looping to avoid memory corruption,
 198                  * but WARN so it is visible what is going on.
 199                  */
 200                 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
 201                         WARN(1, "Unrecoverable platform error!");
 202                 i++;
 203                 msleep(GVE_ADMINQ_SLEEP_LEN);
 204         }
 205         gve_clear_device_rings_ok(priv);
 206         gve_clear_device_resources_ok(priv);
 207         gve_clear_admin_queue_ok(priv);
 208 }
 209
 210 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
 211 {
 212         if (!gve_get_admin_queue_ok(priv))
 213                 return;
 214         gve_adminq_release(priv);
 215         dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
 216         gve_clear_admin_queue_ok(priv);
 217 }
 218
 219 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
 220 {
 221         iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
 222 }
 223
 224 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
 225 {
 226         int i;
 227
 228         for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
 229                 if (ioread32be(&priv->reg_bar0->adminq_event_counter)
 230                     == prod_cnt)
 231                         return true;
 232                 msleep(GVE_ADMINQ_SLEEP_LEN);
 233         }
 234
 235         return false;
 236 }
 237
 238 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
 239 {
 240         if (status != GVE_ADMINQ_COMMAND_PASSED &&
 241             status != GVE_ADMINQ_COMMAND_UNSET) {
 242                 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
 243                 priv->adminq_cmd_fail++;
 244         }
 245         switch (status) {
 246         case GVE_ADMINQ_COMMAND_PASSED:
 247                 return 0;
 248         case GVE_ADMINQ_COMMAND_UNSET:
 249                 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
 250                 return -EINVAL;
 251         case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
 252         case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
 253         case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
 254         case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
 255         case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
 256                 return -EAGAIN;
 257         case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
 258         case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
 259         case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
 260         case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
 261         case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
 262         case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
 263                 return -EINVAL;
 264         case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
 265                 return -ETIME;
 266         case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
 267         case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
 268                 return -EACCES;
 269         case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
 270                 return -ENOMEM;
 271         case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
 272                 return -ENOTSUPP;
 273         default:
 274                 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
 275                 return -EINVAL;
 276         }
 277 }
 278
 279 /* Flushes all AQ commands currently queued and waits for them to complete.
 280  * If there are failures, it will return the first error.
 281  */
 282 static int gve_adminq_kick_and_wait(struct gve_priv *priv)
 283 {
 284         u32 tail, head;
 285         int i;
 286
 287         tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 288         head = priv->adminq_prod_cnt;
 289
 290         gve_adminq_kick_cmd(priv, head);
 291         if (!gve_adminq_wait_for_cmd(priv, head)) {
 292                 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
 293                 priv->adminq_timeouts++;
 294                 return -ENOTRECOVERABLE;
 295         }
 296
 297         for (i = tail; i < head; i++) {
 298                 union gve_adminq_command *cmd;
 299                 u32 status, err;
 300
 301                 cmd = &priv->adminq[i & priv->adminq_mask];
 302                 status = be32_to_cpu(READ_ONCE(cmd->status));
 303                 err = gve_adminq_parse_err(priv, status);
 304                 if (err)
 305                         // Return the first error if we failed.
 306                         return err;
 307         }
 308
 309         return 0;
 310 }
 311
 312 /* This function is not threadsafe - the caller is responsible for any
 313  * necessary locks.
 314  */
 315 static int gve_adminq_issue_cmd(struct gve_priv *priv,
 316                                 union gve_adminq_command *cmd_orig)
 317 {
 318         union gve_adminq_command *cmd;
 319         u32 opcode;
 320         u32 tail;
 321
 322         tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 323
 324         // Check if next command will overflow the buffer.
 325         if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
 326             (tail & priv->adminq_mask)) {
 327                 int err;
 328
 329                 // Flush existing commands to make room.
 330                 err = gve_adminq_kick_and_wait(priv);
 331                 if (err)
 332                         return err;
 333
 334                 // Retry.
 335                 tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 336                 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
 337                     (tail & priv->adminq_mask)) {
 338                         // This should never happen. We just flushed the
 339                         // command queue so there should be enough space.
 340                         return -ENOMEM;
 341                 }
 342         }
 343
 344         cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
 345         priv->adminq_prod_cnt++;
 346
 347         memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
 348         opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
 349
 350         switch (opcode) {
 351         case GVE_ADMINQ_DESCRIBE_DEVICE:
 352                 priv->adminq_describe_device_cnt++;
 353                 break;
 354         case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
 355                 priv->adminq_cfg_device_resources_cnt++;
 356                 break;
 357         case GVE_ADMINQ_REGISTER_PAGE_LIST:
 358                 priv->adminq_register_page_list_cnt++;
 359                 break;
 360         case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
 361                 priv->adminq_unregister_page_list_cnt++;
 362                 break;
 363         case GVE_ADMINQ_CREATE_TX_QUEUE:
 364                 priv->adminq_create_tx_queue_cnt++;
 365                 break;
 366         case GVE_ADMINQ_CREATE_RX_QUEUE:
 367                 priv->adminq_create_rx_queue_cnt++;
 368                 break;
 369         case GVE_ADMINQ_DESTROY_TX_QUEUE:
 370                 priv->adminq_destroy_tx_queue_cnt++;
 371                 break;
 372         case GVE_ADMINQ_DESTROY_RX_QUEUE:
 373                 priv->adminq_destroy_rx_queue_cnt++;
 374                 break;
 375         case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
 376                 priv->adminq_dcfg_device_resources_cnt++;
 377                 break;
 378         case GVE_ADMINQ_SET_DRIVER_PARAMETER:
 379                 priv->adminq_set_driver_parameter_cnt++;
 380                 break;
 381         case GVE_ADMINQ_REPORT_STATS:
 382                 priv->adminq_report_stats_cnt++;
 383                 break;
 384         case GVE_ADMINQ_REPORT_LINK_SPEED:
 385                 priv->adminq_report_link_speed_cnt++;
 386                 break;
 387         case GVE_ADMINQ_GET_PTYPE_MAP:
 388                 priv->adminq_get_ptype_map_cnt++;
 389                 break;
 390         default:
 391                 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
 392         }
 393
 394         return 0;
 395 }
 396
 397 /* This function is not threadsafe - the caller is responsible for any
 398  * necessary locks.
 399  * The caller is also responsible for making sure there are no commands
 400  * waiting to be executed.
 401  */
 402 static int gve_adminq_execute_cmd(struct gve_priv *priv,
 403                                   union gve_adminq_command *cmd_orig)
 404 {
 405         u32 tail, head;
 406         int err;
 407
 408         tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 409         head = priv->adminq_prod_cnt;
 410         if (tail != head)
 411                 // This is not a valid path
 412                 return -EINVAL;
 413
 414         err = gve_adminq_issue_cmd(priv, cmd_orig);
 415         if (err)
 416                 return err;
 417
 418         return gve_adminq_kick_and_wait(priv);
 419 }
 420
 421 /* The device specifies that the management vector can either be the first irq
 422  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
 423  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
 424  * the management vector is first.
 425  *
 426  * gve arranges the msix vectors so that the management vector is last.
 427  */
 428 #define GVE_NTFY_BLK_BASE_MSIX_IDX      0
 429 int gve_adminq_configure_device_resources(struct gve_priv *priv,
 430                                           dma_addr_t counter_array_bus_addr,
 431                                           u32 num_counters,
 432                                           dma_addr_t db_array_bus_addr,
 433                                           u32 num_ntfy_blks)
 434 {
 435         union gve_adminq_command cmd;
 436
 437         memset(&cmd, 0, sizeof(cmd));
 438         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
 439         cmd.configure_device_resources =
 440                 (struct gve_adminq_configure_device_resources) {
 441                 .counter_array = cpu_to_be64(counter_array_bus_addr),
 442                 .num_counters = cpu_to_be32(num_counters),
 443                 .irq_db_addr = cpu_to_be64(db_array_bus_addr),
 444                 .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
 445                 .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
 446                 .ntfy_blk_msix_base_idx =
 447                                         cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
 448                 .queue_format = priv->queue_format,
 449         };
 450
 451         return gve_adminq_execute_cmd(priv, &cmd);
 452 }
 453
 454 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
 455 {
 456         union gve_adminq_command cmd;
 457
 458         memset(&cmd, 0, sizeof(cmd));
 459         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
 460
 461         return gve_adminq_execute_cmd(priv, &cmd);
 462 }
 463
 464 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
 465 {
 466         struct gve_tx_ring *tx = &priv->tx[queue_index];
 467         union gve_adminq_command cmd;
 468
 469         memset(&cmd, 0, sizeof(cmd));
 470         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
 471         cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
 472                 .queue_id = cpu_to_be32(queue_index),
 473                 .queue_resources_addr =
 474                         cpu_to_be64(tx->q_resources_bus),
 475                 .tx_ring_addr = cpu_to_be64(tx->bus),
 476                 .ntfy_id = cpu_to_be32(tx->ntfy_id),
 477         };
 478
 479         if (gve_is_gqi(priv)) {
 480                 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
 481                         GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
 482
 483                 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 484         } else {
 485                 cmd.create_tx_queue.tx_ring_size =
 486                         cpu_to_be16(priv->tx_desc_cnt);
 487                 cmd.create_tx_queue.tx_comp_ring_addr =
 488                         cpu_to_be64(tx->complq_bus_dqo);
 489                 cmd.create_tx_queue.tx_comp_ring_size =
 490                         cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries);
 491         }
 492
 493         return gve_adminq_issue_cmd(priv, &cmd);
 494 }
 495
 496 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
 497 {
 498         int err;
 499         int i;
 500
 501         for (i = 0; i < num_queues; i++) {
 502                 err = gve_adminq_create_tx_queue(priv, i);
 503                 if (err)
 504                         return err;
 505         }
 506
 507         return gve_adminq_kick_and_wait(priv);
 508 }
 509
 510 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
 511 {
 512         struct gve_rx_ring *rx = &priv->rx[queue_index];
 513         union gve_adminq_command cmd;
 514
 515         memset(&cmd, 0, sizeof(cmd));
 516         cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
 517         cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
 518                 .queue_id = cpu_to_be32(queue_index),
 519                 .ntfy_id = cpu_to_be32(rx->ntfy_id),
 520                 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
 521         };
 522
 523         if (gve_is_gqi(priv)) {
 524                 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
 525                         GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
 526
 527                 cmd.create_rx_queue.rx_desc_ring_addr =
 528                         cpu_to_be64(rx->desc.bus),
 529                 cmd.create_rx_queue.rx_data_ring_addr =
 530                         cpu_to_be64(rx->data.data_bus),
 531                 cmd.create_rx_queue.index = cpu_to_be32(queue_index);
 532                 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 533         } else {
 534                 cmd.create_rx_queue.rx_ring_size =
 535                         cpu_to_be16(priv->rx_desc_cnt);
 536                 cmd.create_rx_queue.rx_desc_ring_addr =
 537                         cpu_to_be64(rx->dqo.complq.bus);
 538                 cmd.create_rx_queue.rx_data_ring_addr =
 539                         cpu_to_be64(rx->dqo.bufq.bus);
 540                 cmd.create_rx_queue.packet_buffer_size =
 541                         cpu_to_be16(priv->data_buffer_size_dqo);
 542                 cmd.create_rx_queue.rx_buff_ring_size =
 543                         cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries);
 544                 cmd.create_rx_queue.enable_rsc =
 545                         !!(priv->dev->features & NETIF_F_LRO);
 546         }
 547
 548         return gve_adminq_issue_cmd(priv, &cmd);
 549 }
 550
 551 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
 552 {
 553         int err;
 554         int i;
 555
 556         for (i = 0; i < num_queues; i++) {
 557                 err = gve_adminq_create_rx_queue(priv, i);
 558                 if (err)
 559                         return err;
 560         }
 561
 562         return gve_adminq_kick_and_wait(priv);
 563 }
 564
 565 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
 566 {
 567         union gve_adminq_command cmd;
 568         int err;
 569
 570         memset(&cmd, 0, sizeof(cmd));
 571         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
 572         cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
 573                 .queue_id = cpu_to_be32(queue_index),
 574         };
 575
 576         err = gve_adminq_issue_cmd(priv, &cmd);
 577         if (err)
 578                 return err;
 579
 580         return 0;
 581 }
 582
 583 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
 584 {
 585         int err;
 586         int i;
 587
 588         for (i = 0; i < num_queues; i++) {
 589                 err = gve_adminq_destroy_tx_queue(priv, i);
 590                 if (err)
 591                         return err;
 592         }
 593
 594         return gve_adminq_kick_and_wait(priv);
 595 }
 596
 597 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
 598 {
 599         union gve_adminq_command cmd;
 600         int err;
 601
 602         memset(&cmd, 0, sizeof(cmd));
 603         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
 604         cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
 605                 .queue_id = cpu_to_be32(queue_index),
 606         };
 607
 608         err = gve_adminq_issue_cmd(priv, &cmd);
 609         if (err)
 610                 return err;
 611
 612         return 0;
 613 }
 614
 615 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
 616 {
 617         int err;
 618         int i;
 619
 620         for (i = 0; i < num_queues; i++) {
 621                 err = gve_adminq_destroy_rx_queue(priv, i);
 622                 if (err)
 623                         return err;
 624         }
 625
 626         return gve_adminq_kick_and_wait(priv);
 627 }
 628
 629 static int gve_set_desc_cnt(struct gve_priv *priv,
 630                             struct gve_device_descriptor *descriptor)
 631 {
 632         priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
 633         if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
 634                 dev_err(&priv->pdev->dev, "Tx desc count %d too low\n",
 635                         priv->tx_desc_cnt);
 636                 return -EINVAL;
 637         }
 638         priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
 639         if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
 640             < PAGE_SIZE) {
 641                 dev_err(&priv->pdev->dev, "Rx desc count %d too low\n",
 642                         priv->rx_desc_cnt);
 643                 return -EINVAL;
 644         }
 645         return 0;
 646 }
 647
 648 static int
 649 gve_set_desc_cnt_dqo(struct gve_priv *priv,
 650                      const struct gve_device_descriptor *descriptor,
 651                      const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
 652 {
 653         priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
 654         priv->options_dqo_rda.tx_comp_ring_entries =
 655                 be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
 656         priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
 657         priv->options_dqo_rda.rx_buff_ring_entries =
 658                 be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
 659
 660         return 0;
 661 }
 662
 663 int gve_adminq_describe_device(struct gve_priv *priv)
 664 {
 665         struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
 666         struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
 667         struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
 668         struct gve_device_descriptor *descriptor;
 669         union gve_adminq_command cmd;
 670         dma_addr_t descriptor_bus;
 671         int err = 0;
 672         u8 *mac;
 673         u16 mtu;
 674
 675         memset(&cmd, 0, sizeof(cmd));
 676         descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
 677                                         &descriptor_bus, GFP_KERNEL);
 678         if (!descriptor)
 679                 return -ENOMEM;
 680         cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
 681         cmd.describe_device.device_descriptor_addr =
 682                                                 cpu_to_be64(descriptor_bus);
 683         cmd.describe_device.device_descriptor_version =
 684                         cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
 685         cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
 686
 687         err = gve_adminq_execute_cmd(priv, &cmd);
 688         if (err)
 689                 goto free_device_descriptor;
 690
 691         err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
 692                                          &dev_op_gqi_qpl, &dev_op_dqo_rda);
 693         if (err)
 694                 goto free_device_descriptor;
 695
 696         /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
 697          * is not set to GqiRda, choose the queue format in a priority order:
 698          * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default.
 699          */
 700         if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
 701                 dev_info(&priv->pdev->dev,
 702                          "Driver is running with GQI RDA queue format.\n");
 703         } else if (dev_op_dqo_rda) {
 704                 priv->queue_format = GVE_DQO_RDA_FORMAT;
 705                 dev_info(&priv->pdev->dev,
 706                          "Driver is running with DQO RDA queue format.\n");
 707         } else if (dev_op_gqi_rda) {
 708                 priv->queue_format = GVE_GQI_RDA_FORMAT;
 709                 dev_info(&priv->pdev->dev,
 710                          "Driver is running with GQI RDA queue format.\n");
 711         } else {
 712                 priv->queue_format = GVE_GQI_QPL_FORMAT;
 713                 dev_info(&priv->pdev->dev,
 714                          "Driver is running with GQI QPL queue format.\n");
 715         }
 716         if (gve_is_gqi(priv)) {
 717                 err = gve_set_desc_cnt(priv, descriptor);
 718         } else {
 719                 /* DQO supports LRO. */
 720                 priv->dev->hw_features |= NETIF_F_LRO;
 721                 err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda);
 722         }
 723         if (err)
 724                 goto free_device_descriptor;
 725
 726         priv->max_registered_pages =
 727                                 be64_to_cpu(descriptor->max_registered_pages);
 728         mtu = be16_to_cpu(descriptor->mtu);
 729         if (mtu < ETH_MIN_MTU) {
 730                 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
 731                 err = -EINVAL;
 732                 goto free_device_descriptor;
 733         }
 734         priv->dev->max_mtu = mtu;
 735         priv->num_event_counters = be16_to_cpu(descriptor->counters);
 736         eth_hw_addr_set(priv->dev, descriptor->mac);
 737         mac = descriptor->mac;
 738         dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
 739         priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
 740         priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl);
 741
 742         if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) {
 743                 dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
 744                         priv->rx_data_slot_cnt);
 745                 priv->rx_desc_cnt = priv->rx_data_slot_cnt;
 746         }
 747         priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
 748
 749 free_device_descriptor:
 750         dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
 751                           descriptor_bus);
 752         return err;
 753 }
 754
 755 int gve_adminq_register_page_list(struct gve_priv *priv,
 756                                   struct gve_queue_page_list *qpl)
 757 {
 758         struct device *hdev = &priv->pdev->dev;
 759         u32 num_entries = qpl->num_entries;
 760         u32 size = num_entries * sizeof(qpl->page_buses[0]);
 761         union gve_adminq_command cmd;
 762         dma_addr_t page_list_bus;
 763         __be64 *page_list;
 764         int err;
 765         int i;
 766
 767         memset(&cmd, 0, sizeof(cmd));
 768         page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
 769         if (!page_list)
 770                 return -ENOMEM;
 771
 772         for (i = 0; i < num_entries; i++)
 773                 page_list[i] = cpu_to_be64(qpl->page_buses[i]);
 774
 775         cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
 776         cmd.reg_page_list = (struct gve_adminq_register_page_list) {
 777                 .page_list_id = cpu_to_be32(qpl->id),
 778                 .num_pages = cpu_to_be32(num_entries),
 779                 .page_address_list_addr = cpu_to_be64(page_list_bus),
 780         };
 781
 782         err = gve_adminq_execute_cmd(priv, &cmd);
 783         dma_free_coherent(hdev, size, page_list, page_list_bus);
 784         return err;
 785 }
 786
 787 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
 788 {
 789         union gve_adminq_command cmd;
 790
 791         memset(&cmd, 0, sizeof(cmd));
 792         cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
 793         cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
 794                 .page_list_id = cpu_to_be32(page_list_id),
 795         };
 796
 797         return gve_adminq_execute_cmd(priv, &cmd);
 798 }
 799
 800 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
 801 {
 802         union gve_adminq_command cmd;
 803
 804         memset(&cmd, 0, sizeof(cmd));
 805         cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
 806         cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
 807                 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
 808                 .parameter_value = cpu_to_be64(mtu),
 809         };
 810
 811         return gve_adminq_execute_cmd(priv, &cmd);
 812 }
 813
 814 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
 815                             dma_addr_t stats_report_addr, u64 interval)
 816 {
 817         union gve_adminq_command cmd;
 818
 819         memset(&cmd, 0, sizeof(cmd));
 820         cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
 821         cmd.report_stats = (struct gve_adminq_report_stats) {
 822                 .stats_report_len = cpu_to_be64(stats_report_len),
 823                 .stats_report_addr = cpu_to_be64(stats_report_addr),
 824                 .interval = cpu_to_be64(interval),
 825         };
 826
 827         return gve_adminq_execute_cmd(priv, &cmd);
 828 }
 829
 830 int gve_adminq_report_link_speed(struct gve_priv *priv)
 831 {
 832         union gve_adminq_command gvnic_cmd;
 833         dma_addr_t link_speed_region_bus;
 834         __be64 *link_speed_region;
 835         int err;
 836
 837         link_speed_region =
 838                 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
 839                                    &link_speed_region_bus, GFP_KERNEL);
 840
 841         if (!link_speed_region)
 842                 return -ENOMEM;
 843
 844         memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
 845         gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
 846         gvnic_cmd.report_link_speed.link_speed_address =
 847                 cpu_to_be64(link_speed_region_bus);
 848
 849         err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
 850
 851         priv->link_speed = be64_to_cpu(*link_speed_region);
 852         dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
 853                           link_speed_region_bus);
 854         return err;
 855 }
 856
 857 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
 858                                  struct gve_ptype_lut *ptype_lut)
 859 {
 860         struct gve_ptype_map *ptype_map;
 861         union gve_adminq_command cmd;
 862         dma_addr_t ptype_map_bus;
 863         int err = 0;
 864         int i;
 865
 866         memset(&cmd, 0, sizeof(cmd));
 867         ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
 868                                        &ptype_map_bus, GFP_KERNEL);
 869         if (!ptype_map)
 870                 return -ENOMEM;
 871
 872         cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
 873         cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
 874                 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
 875                 .ptype_map_addr = cpu_to_be64(ptype_map_bus),
 876         };
 877
 878         err = gve_adminq_execute_cmd(priv, &cmd);
 879         if (err)
 880                 goto err;
 881
 882         /* Populate ptype_lut. */
 883         for (i = 0; i < GVE_NUM_PTYPES; i++) {
 884                 ptype_lut->ptypes[i].l3_type =
 885                         ptype_map->ptypes[i].l3_type;
 886                 ptype_lut->ptypes[i].l4_type =
 887                         ptype_map->ptypes[i].l4_type;
 888         }
 889 err:
 890         dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
 891                           ptype_map_bus);
 892         return err;
 893 }