drivers/hv/channel.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2009, Microsoft Corporation.
   4  *
   5  * Authors:
   6  *   Haiyang Zhang <haiyangz@microsoft.com>
   7  *   Hank Janssen  <hjanssen@microsoft.com>
   8  */
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/kernel.h>
  12 #include <linux/sched.h>
  13 #include <linux/wait.h>
  14 #include <linux/mm.h>
  15 #include <linux/slab.h>
  16 #include <linux/module.h>
  17 #include <linux/hyperv.h>
  18 #include <linux/uio.h>
  19 #include <linux/interrupt.h>
  20 #include <asm/page.h>
  21 #include <asm/mshyperv.h>
  22
  23 #include "hyperv_vmbus.h"
  24
  25 /*
  26  * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses
  27  *
  28  * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does.
  29  *
  30  * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header
  31  * (because of the alignment requirement), however, the hypervisor only
  32  * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a
  33  * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a
  34  * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the
  35  * total size that the guest uses minus twice of the gap size.
  36  */
  37 static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size)
  38 {
  39         switch (type) {
  40         case HV_GPADL_BUFFER:
  41                 return size;
  42         case HV_GPADL_RING:
  43                 /* The size of a ringbuffer must be page-aligned */
  44                 BUG_ON(size % PAGE_SIZE);
  45                 /*
  46                  * Two things to notice here:
  47                  * 1) We're processing two ring buffers as a unit
  48                  * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in
  49                  * the first guest-size page of each of the two ring buffers.
  50                  * So we effectively subtract out two guest-size pages, and add
  51                  * back two Hyper-V size pages.
  52                  */
  53                 return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
  54         }
  55         BUG();
  56         return 0;
  57 }
  58
  59 /*
  60  * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of
  61  *                                 HV_HYP_PAGE) in a ring gpadl based on the
  62  *                                 offset in the guest
  63  *
  64  * @offset: the offset (in bytes) where the send ringbuffer starts in the
  65  *               virtual address space of the guest
  66  */
  67 static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset)
  68 {
  69
  70         /*
  71          * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the
  72          * header (because of the alignment requirement), however, the
  73          * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header,
  74          * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap.
  75          *
  76          * And to calculate the effective send offset in gpadl, we need to
  77          * substract this gap.
  78          */
  79         return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT;
  80 }
  81
  82 /*
  83  * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in
  84  *                  the gpadl
  85  *
  86  * @type: the type of the gpadl
  87  * @kbuffer: the pointer to the gpadl in the guest
  88  * @size: the total size (in bytes) of the gpadl
  89  * @send_offset: the offset (in bytes) where the send ringbuffer starts in the
  90  *               virtual address space of the guest
  91  * @i: the index
  92  */
  93 static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer,
  94                                  u32 size, u32 send_offset, int i)
  95 {
  96         int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset);
  97         unsigned long delta = 0UL;
  98
  99         switch (type) {
 100         case HV_GPADL_BUFFER:
 101                 break;
 102         case HV_GPADL_RING:
 103                 if (i == 0)
 104                         delta = 0;
 105                 else if (i <= send_idx)
 106                         delta = PAGE_SIZE - HV_HYP_PAGE_SIZE;
 107                 else
 108                         delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
 109                 break;
 110         default:
 111                 BUG();
 112                 break;
 113         }
 114
 115         return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i));
 116 }
 117
 118 /*
 119  * vmbus_setevent- Trigger an event notification on the specified
 120  * channel.
 121  */
 122 void vmbus_setevent(struct vmbus_channel *channel)
 123 {
 124         struct hv_monitor_page *monitorpage;
 125
 126         trace_vmbus_setevent(channel);
 127
 128         /*
 129          * For channels marked as in "low latency" mode
 130          * bypass the monitor page mechanism.
 131          */
 132         if (channel->offermsg.monitor_allocated && !channel->low_latency) {
 133                 vmbus_send_interrupt(channel->offermsg.child_relid);
 134
 135                 /* Get the child to parent monitor page */
 136                 monitorpage = vmbus_connection.monitor_pages[1];
 137
 138                 sync_set_bit(channel->monitor_bit,
 139                         (unsigned long *)&monitorpage->trigger_group
 140                                         [channel->monitor_grp].pending);
 141
 142         } else {
 143                 vmbus_set_event(channel);
 144         }
 145 }
 146 EXPORT_SYMBOL_GPL(vmbus_setevent);
 147
 148 /* vmbus_free_ring - drop mapping of ring buffer */
 149 void vmbus_free_ring(struct vmbus_channel *channel)
 150 {
 151         hv_ringbuffer_cleanup(&channel->outbound);
 152         hv_ringbuffer_cleanup(&channel->inbound);
 153
 154         if (channel->ringbuffer_page) {
 155                 __free_pages(channel->ringbuffer_page,
 156                              get_order(channel->ringbuffer_pagecount
 157                                        << PAGE_SHIFT));
 158                 channel->ringbuffer_page = NULL;
 159         }
 160 }
 161 EXPORT_SYMBOL_GPL(vmbus_free_ring);
 162
 163 /* vmbus_alloc_ring - allocate and map pages for ring buffer */
 164 int vmbus_alloc_ring(struct vmbus_channel *newchannel,
 165                      u32 send_size, u32 recv_size)
 166 {
 167         struct page *page;
 168         int order;
 169
 170         if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE)
 171                 return -EINVAL;
 172
 173         /* Allocate the ring buffer */
 174         order = get_order(send_size + recv_size);
 175         page = alloc_pages_node(cpu_to_node(newchannel->target_cpu),
 176                                 GFP_KERNEL|__GFP_ZERO, order);
 177
 178         if (!page)
 179                 page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order);
 180
 181         if (!page)
 182                 return -ENOMEM;
 183
 184         newchannel->ringbuffer_page = page;
 185         newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT;
 186         newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT;
 187
 188         return 0;
 189 }
 190 EXPORT_SYMBOL_GPL(vmbus_alloc_ring);
 191
 192 /* Used for Hyper-V Socket: a guest client's connect() to the host */
 193 int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id,
 194                                   const guid_t *shv_host_servie_id)
 195 {
 196         struct vmbus_channel_tl_connect_request conn_msg;
 197         int ret;
 198
 199         memset(&conn_msg, 0, sizeof(conn_msg));
 200         conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST;
 201         conn_msg.guest_endpoint_id = *shv_guest_servie_id;
 202         conn_msg.host_service_id = *shv_host_servie_id;
 203
 204         ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true);
 205
 206         trace_vmbus_send_tl_connect_request(&conn_msg, ret);
 207
 208         return ret;
 209 }
 210 EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request);
 211
 212 static int send_modifychannel_without_ack(struct vmbus_channel *channel, u32 target_vp)
 213 {
 214         struct vmbus_channel_modifychannel msg;
 215         int ret;
 216
 217         memset(&msg, 0, sizeof(msg));
 218         msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL;
 219         msg.child_relid = channel->offermsg.child_relid;
 220         msg.target_vp = target_vp;
 221
 222         ret = vmbus_post_msg(&msg, sizeof(msg), true);
 223         trace_vmbus_send_modifychannel(&msg, ret);
 224
 225         return ret;
 226 }
 227
 228 static int send_modifychannel_with_ack(struct vmbus_channel *channel, u32 target_vp)
 229 {
 230         struct vmbus_channel_modifychannel *msg;
 231         struct vmbus_channel_msginfo *info;
 232         unsigned long flags;
 233         int ret;
 234
 235         info = kzalloc(sizeof(struct vmbus_channel_msginfo) +
 236                                 sizeof(struct vmbus_channel_modifychannel),
 237                        GFP_KERNEL);
 238         if (!info)
 239                 return -ENOMEM;
 240
 241         init_completion(&info->waitevent);
 242         info->waiting_channel = channel;
 243
 244         msg = (struct vmbus_channel_modifychannel *)info->msg;
 245         msg->header.msgtype = CHANNELMSG_MODIFYCHANNEL;
 246         msg->child_relid = channel->offermsg.child_relid;
 247         msg->target_vp = target_vp;
 248
 249         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 250         list_add_tail(&info->msglistentry, &vmbus_connection.chn_msg_list);
 251         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 252
 253         ret = vmbus_post_msg(msg, sizeof(*msg), true);
 254         trace_vmbus_send_modifychannel(msg, ret);
 255         if (ret != 0) {
 256                 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 257                 list_del(&info->msglistentry);
 258                 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 259                 goto free_info;
 260         }
 261
 262         /*
 263          * Release channel_mutex; otherwise, vmbus_onoffer_rescind() could block on
 264          * the mutex and be unable to signal the completion.
 265          *
 266          * See the caller target_cpu_store() for information about the usage of the
 267          * mutex.
 268          */
 269         mutex_unlock(&vmbus_connection.channel_mutex);
 270         wait_for_completion(&info->waitevent);
 271         mutex_lock(&vmbus_connection.channel_mutex);
 272
 273         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 274         list_del(&info->msglistentry);
 275         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 276
 277         if (info->response.modify_response.status)
 278                 ret = -EAGAIN;
 279
 280 free_info:
 281         kfree(info);
 282         return ret;
 283 }
 284
 285 /*
 286  * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt.
 287  *
 288  * CHANNELMSG_MODIFYCHANNEL messages are aynchronous.  When VMbus version 5.3
 289  * or later is negotiated, Hyper-V always sends an ACK in response to such a
 290  * message.  For VMbus version 5.2 and earlier, it never sends an ACK.  With-
 291  * out an ACK, we can not know when the host will stop interrupting the "old"
 292  * vCPU and start interrupting the "new" vCPU for the given channel.
 293  *
 294  * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version
 295  * VERSION_WIN10_V4_1.
 296  */
 297 int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp)
 298 {
 299         if (vmbus_proto_version >= VERSION_WIN10_V5_3)
 300                 return send_modifychannel_with_ack(channel, target_vp);
 301         return send_modifychannel_without_ack(channel, target_vp);
 302 }
 303 EXPORT_SYMBOL_GPL(vmbus_send_modifychannel);
 304
 305 /*
 306  * create_gpadl_header - Creates a gpadl for the specified buffer
 307  */
 308 static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer,
 309                                u32 size, u32 send_offset,
 310                                struct vmbus_channel_msginfo **msginfo)
 311 {
 312         int i;
 313         int pagecount;
 314         struct vmbus_channel_gpadl_header *gpadl_header;
 315         struct vmbus_channel_gpadl_body *gpadl_body;
 316         struct vmbus_channel_msginfo *msgheader;
 317         struct vmbus_channel_msginfo *msgbody = NULL;
 318         u32 msgsize;
 319
 320         int pfnsum, pfncount, pfnleft, pfncurr, pfnsize;
 321
 322         pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT;
 323
 324         /* do we need a gpadl body msg */
 325         pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 326                   sizeof(struct vmbus_channel_gpadl_header) -
 327                   sizeof(struct gpa_range);
 328         pfncount = pfnsize / sizeof(u64);
 329
 330         if (pagecount > pfncount) {
 331                 /* we need a gpadl body */
 332                 /* fill in the header */
 333                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 334                           sizeof(struct vmbus_channel_gpadl_header) +
 335                           sizeof(struct gpa_range) + pfncount * sizeof(u64);
 336                 msgheader =  kzalloc(msgsize, GFP_KERNEL);
 337                 if (!msgheader)
 338                         goto nomem;
 339
 340                 INIT_LIST_HEAD(&msgheader->submsglist);
 341                 msgheader->msgsize = msgsize;
 342
 343                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 344                         msgheader->msg;
 345                 gpadl_header->rangecount = 1;
 346                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 347                                          pagecount * sizeof(u64);
 348                 gpadl_header->range[0].byte_offset = 0;
 349                 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
 350                 for (i = 0; i < pfncount; i++)
 351                         gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
 352                                 type, kbuffer, size, send_offset, i);
 353                 *msginfo = msgheader;
 354
 355                 pfnsum = pfncount;
 356                 pfnleft = pagecount - pfncount;
 357
 358                 /* how many pfns can we fit */
 359                 pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 360                           sizeof(struct vmbus_channel_gpadl_body);
 361                 pfncount = pfnsize / sizeof(u64);
 362
 363                 /* fill in the body */
 364                 while (pfnleft) {
 365                         if (pfnleft > pfncount)
 366                                 pfncurr = pfncount;
 367                         else
 368                                 pfncurr = pfnleft;
 369
 370                         msgsize = sizeof(struct vmbus_channel_msginfo) +
 371                                   sizeof(struct vmbus_channel_gpadl_body) +
 372                                   pfncurr * sizeof(u64);
 373                         msgbody = kzalloc(msgsize, GFP_KERNEL);
 374
 375                         if (!msgbody) {
 376                                 struct vmbus_channel_msginfo *pos = NULL;
 377                                 struct vmbus_channel_msginfo *tmp = NULL;
 378                                 /*
 379                                  * Free up all the allocated messages.
 380                                  */
 381                                 list_for_each_entry_safe(pos, tmp,
 382                                         &msgheader->submsglist,
 383                                         msglistentry) {
 384
 385                                         list_del(&pos->msglistentry);
 386                                         kfree(pos);
 387                                 }
 388
 389                                 goto nomem;
 390                         }
 391
 392                         msgbody->msgsize = msgsize;
 393                         gpadl_body =
 394                                 (struct vmbus_channel_gpadl_body *)msgbody->msg;
 395
 396                         /*
 397                          * Gpadl is u32 and we are using a pointer which could
 398                          * be 64-bit
 399                          * This is governed by the guest/host protocol and
 400                          * so the hypervisor guarantees that this is ok.
 401                          */
 402                         for (i = 0; i < pfncurr; i++)
 403                                 gpadl_body->pfn[i] = hv_gpadl_hvpfn(type,
 404                                         kbuffer, size, send_offset, pfnsum + i);
 405
 406                         /* add to msg header */
 407                         list_add_tail(&msgbody->msglistentry,
 408                                       &msgheader->submsglist);
 409                         pfnsum += pfncurr;
 410                         pfnleft -= pfncurr;
 411                 }
 412         } else {
 413                 /* everything fits in a header */
 414                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 415                           sizeof(struct vmbus_channel_gpadl_header) +
 416                           sizeof(struct gpa_range) + pagecount * sizeof(u64);
 417                 msgheader = kzalloc(msgsize, GFP_KERNEL);
 418                 if (msgheader == NULL)
 419                         goto nomem;
 420
 421                 INIT_LIST_HEAD(&msgheader->submsglist);
 422                 msgheader->msgsize = msgsize;
 423
 424                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 425                         msgheader->msg;
 426                 gpadl_header->rangecount = 1;
 427                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 428                                          pagecount * sizeof(u64);
 429                 gpadl_header->range[0].byte_offset = 0;
 430                 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
 431                 for (i = 0; i < pagecount; i++)
 432                         gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
 433                                 type, kbuffer, size, send_offset, i);
 434
 435                 *msginfo = msgheader;
 436         }
 437
 438         return 0;
 439 nomem:
 440         kfree(msgheader);
 441         kfree(msgbody);
 442         return -ENOMEM;
 443 }
 444
 445 /*
 446  * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer
 447  *
 448  * @channel: a channel
 449  * @type: the type of the corresponding GPADL, only meaningful for the guest.
 450  * @kbuffer: from kmalloc or vmalloc
 451  * @size: page-size multiple
 452  * @send_offset: the offset (in bytes) where the send ring buffer starts,
 453  *              should be 0 for BUFFER type gpadl
 454  * @gpadl_handle: some funky thing
 455  */
 456 static int __vmbus_establish_gpadl(struct vmbus_channel *channel,
 457                                    enum hv_gpadl_type type, void *kbuffer,
 458                                    u32 size, u32 send_offset,
 459                                    u32 *gpadl_handle)
 460 {
 461         struct vmbus_channel_gpadl_header *gpadlmsg;
 462         struct vmbus_channel_gpadl_body *gpadl_body;
 463         struct vmbus_channel_msginfo *msginfo = NULL;
 464         struct vmbus_channel_msginfo *submsginfo, *tmp;
 465         struct list_head *curr;
 466         u32 next_gpadl_handle;
 467         unsigned long flags;
 468         int ret = 0;
 469
 470         next_gpadl_handle =
 471                 (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
 472
 473         ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo);
 474         if (ret)
 475                 return ret;
 476
 477         init_completion(&msginfo->waitevent);
 478         msginfo->waiting_channel = channel;
 479
 480         gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg;
 481         gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER;
 482         gpadlmsg->child_relid = channel->offermsg.child_relid;
 483         gpadlmsg->gpadl = next_gpadl_handle;
 484
 485
 486         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 487         list_add_tail(&msginfo->msglistentry,
 488                       &vmbus_connection.chn_msg_list);
 489
 490         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 491
 492         if (channel->rescind) {
 493                 ret = -ENODEV;
 494                 goto cleanup;
 495         }
 496
 497         ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize -
 498                              sizeof(*msginfo), true);
 499
 500         trace_vmbus_establish_gpadl_header(gpadlmsg, ret);
 501
 502         if (ret != 0)
 503                 goto cleanup;
 504
 505         list_for_each(curr, &msginfo->submsglist) {
 506                 submsginfo = (struct vmbus_channel_msginfo *)curr;
 507                 gpadl_body =
 508                         (struct vmbus_channel_gpadl_body *)submsginfo->msg;
 509
 510                 gpadl_body->header.msgtype =
 511                         CHANNELMSG_GPADL_BODY;
 512                 gpadl_body->gpadl = next_gpadl_handle;
 513
 514                 ret = vmbus_post_msg(gpadl_body,
 515                                      submsginfo->msgsize - sizeof(*submsginfo),
 516                                      true);
 517
 518                 trace_vmbus_establish_gpadl_body(gpadl_body, ret);
 519
 520                 if (ret != 0)
 521                         goto cleanup;
 522
 523         }
 524         wait_for_completion(&msginfo->waitevent);
 525
 526         if (msginfo->response.gpadl_created.creation_status != 0) {
 527                 pr_err("Failed to establish GPADL: err = 0x%x\n",
 528                        msginfo->response.gpadl_created.creation_status);
 529
 530                 ret = -EDQUOT;
 531                 goto cleanup;
 532         }
 533
 534         if (channel->rescind) {
 535                 ret = -ENODEV;
 536                 goto cleanup;
 537         }
 538
 539         /* At this point, we received the gpadl created msg */
 540         *gpadl_handle = gpadlmsg->gpadl;
 541
 542 cleanup:
 543         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 544         list_del(&msginfo->msglistentry);
 545         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 546         list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist,
 547                                  msglistentry) {
 548                 kfree(submsginfo);
 549         }
 550
 551         kfree(msginfo);
 552         return ret;
 553 }
 554
 555 /*
 556  * vmbus_establish_gpadl - Establish a GPADL for the specified buffer
 557  *
 558  * @channel: a channel
 559  * @kbuffer: from kmalloc or vmalloc
 560  * @size: page-size multiple
 561  * @gpadl_handle: some funky thing
 562  */
 563 int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 564                           u32 size, u32 *gpadl_handle)
 565 {
 566         return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size,
 567                                        0U, gpadl_handle);
 568 }
 569 EXPORT_SYMBOL_GPL(vmbus_establish_gpadl);
 570
 571 /**
 572  * request_arr_init - Allocates memory for the requestor array. Each slot
 573  * keeps track of the next available slot in the array. Initially, each
 574  * slot points to the next one (as in a Linked List). The last slot
 575  * does not point to anything, so its value is U64_MAX by default.
 576  * @size The size of the array
 577  */
 578 static u64 *request_arr_init(u32 size)
 579 {
 580         int i;
 581         u64 *req_arr;
 582
 583         req_arr = kcalloc(size, sizeof(u64), GFP_KERNEL);
 584         if (!req_arr)
 585                 return NULL;
 586
 587         for (i = 0; i < size - 1; i++)
 588                 req_arr[i] = i + 1;
 589
 590         /* Last slot (no more available slots) */
 591         req_arr[i] = U64_MAX;
 592
 593         return req_arr;
 594 }
 595
 596 /*
 597  * vmbus_alloc_requestor - Initializes @rqstor's fields.
 598  * Index 0 is the first free slot
 599  * @size: Size of the requestor array
 600  */
 601 static int vmbus_alloc_requestor(struct vmbus_requestor *rqstor, u32 size)
 602 {
 603         u64 *rqst_arr;
 604         unsigned long *bitmap;
 605
 606         rqst_arr = request_arr_init(size);
 607         if (!rqst_arr)
 608                 return -ENOMEM;
 609
 610         bitmap = bitmap_zalloc(size, GFP_KERNEL);
 611         if (!bitmap) {
 612                 kfree(rqst_arr);
 613                 return -ENOMEM;
 614         }
 615
 616         rqstor->req_arr = rqst_arr;
 617         rqstor->req_bitmap = bitmap;
 618         rqstor->size = size;
 619         rqstor->next_request_id = 0;
 620         spin_lock_init(&rqstor->req_lock);
 621
 622         return 0;
 623 }
 624
 625 /*
 626  * vmbus_free_requestor - Frees memory allocated for @rqstor
 627  * @rqstor: Pointer to the requestor struct
 628  */
 629 static void vmbus_free_requestor(struct vmbus_requestor *rqstor)
 630 {
 631         kfree(rqstor->req_arr);
 632         bitmap_free(rqstor->req_bitmap);
 633 }
 634
 635 static int __vmbus_open(struct vmbus_channel *newchannel,
 636                        void *userdata, u32 userdatalen,
 637                        void (*onchannelcallback)(void *context), void *context)
 638 {
 639         struct vmbus_channel_open_channel *open_msg;
 640         struct vmbus_channel_msginfo *open_info = NULL;
 641         struct page *page = newchannel->ringbuffer_page;
 642         u32 send_pages, recv_pages;
 643         unsigned long flags;
 644         int err;
 645
 646         if (userdatalen > MAX_USER_DEFINED_BYTES)
 647                 return -EINVAL;
 648
 649         send_pages = newchannel->ringbuffer_send_offset;
 650         recv_pages = newchannel->ringbuffer_pagecount - send_pages;
 651
 652         if (newchannel->state != CHANNEL_OPEN_STATE)
 653                 return -EINVAL;
 654
 655         /* Create and init requestor */
 656         if (newchannel->rqstor_size) {
 657                 if (vmbus_alloc_requestor(&newchannel->requestor, newchannel->rqstor_size))
 658                         return -ENOMEM;
 659         }
 660
 661         newchannel->state = CHANNEL_OPENING_STATE;
 662         newchannel->onchannel_callback = onchannelcallback;
 663         newchannel->channel_callback_context = context;
 664
 665         if (!newchannel->max_pkt_size)
 666                 newchannel->max_pkt_size = VMBUS_DEFAULT_MAX_PKT_SIZE;
 667
 668         err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages, 0);
 669         if (err)
 670                 goto error_clean_ring;
 671
 672         err = hv_ringbuffer_init(&newchannel->inbound, &page[send_pages],
 673                                  recv_pages, newchannel->max_pkt_size);
 674         if (err)
 675                 goto error_clean_ring;
 676
 677         /* Establish the gpadl for the ring buffer */
 678         newchannel->ringbuffer_gpadlhandle = 0;
 679
 680         err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING,
 681                                       page_address(newchannel->ringbuffer_page),
 682                                       (send_pages + recv_pages) << PAGE_SHIFT,
 683                                       newchannel->ringbuffer_send_offset << PAGE_SHIFT,
 684                                       &newchannel->ringbuffer_gpadlhandle);
 685         if (err)
 686                 goto error_clean_ring;
 687
 688         /* Create and init the channel open message */
 689         open_info = kzalloc(sizeof(*open_info) +
 690                            sizeof(struct vmbus_channel_open_channel),
 691                            GFP_KERNEL);
 692         if (!open_info) {
 693                 err = -ENOMEM;
 694                 goto error_free_gpadl;
 695         }
 696
 697         init_completion(&open_info->waitevent);
 698         open_info->waiting_channel = newchannel;
 699
 700         open_msg = (struct vmbus_channel_open_channel *)open_info->msg;
 701         open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL;
 702         open_msg->openid = newchannel->offermsg.child_relid;
 703         open_msg->child_relid = newchannel->offermsg.child_relid;
 704         open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle;
 705         /*
 706          * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and
 707          * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so
 708          * here we calculate it into HV_HYP_PAGE.
 709          */
 710         open_msg->downstream_ringbuffer_pageoffset =
 711                 hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT);
 712         open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu);
 713
 714         if (userdatalen)
 715                 memcpy(open_msg->userdata, userdata, userdatalen);
 716
 717         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 718         list_add_tail(&open_info->msglistentry,
 719                       &vmbus_connection.chn_msg_list);
 720         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 721
 722         if (newchannel->rescind) {
 723                 err = -ENODEV;
 724                 goto error_clean_msglist;
 725         }
 726
 727         err = vmbus_post_msg(open_msg,
 728                              sizeof(struct vmbus_channel_open_channel), true);
 729
 730         trace_vmbus_open(open_msg, err);
 731
 732         if (err != 0)
 733                 goto error_clean_msglist;
 734
 735         wait_for_completion(&open_info->waitevent);
 736
 737         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 738         list_del(&open_info->msglistentry);
 739         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 740
 741         if (newchannel->rescind) {
 742                 err = -ENODEV;
 743                 goto error_free_info;
 744         }
 745
 746         if (open_info->response.open_result.status) {
 747                 err = -EAGAIN;
 748                 goto error_free_info;
 749         }
 750
 751         newchannel->state = CHANNEL_OPENED_STATE;
 752         kfree(open_info);
 753         return 0;
 754
 755 error_clean_msglist:
 756         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 757         list_del(&open_info->msglistentry);
 758         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 759 error_free_info:
 760         kfree(open_info);
 761 error_free_gpadl:
 762         vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle);
 763         newchannel->ringbuffer_gpadlhandle = 0;
 764 error_clean_ring:
 765         hv_ringbuffer_cleanup(&newchannel->outbound);
 766         hv_ringbuffer_cleanup(&newchannel->inbound);
 767         vmbus_free_requestor(&newchannel->requestor);
 768         newchannel->state = CHANNEL_OPEN_STATE;
 769         return err;
 770 }
 771
 772 /*
 773  * vmbus_connect_ring - Open the channel but reuse ring buffer
 774  */
 775 int vmbus_connect_ring(struct vmbus_channel *newchannel,
 776                        void (*onchannelcallback)(void *context), void *context)
 777 {
 778         return  __vmbus_open(newchannel, NULL, 0, onchannelcallback, context);
 779 }
 780 EXPORT_SYMBOL_GPL(vmbus_connect_ring);
 781
 782 /*
 783  * vmbus_open - Open the specified channel.
 784  */
 785 int vmbus_open(struct vmbus_channel *newchannel,
 786                u32 send_ringbuffer_size, u32 recv_ringbuffer_size,
 787                void *userdata, u32 userdatalen,
 788                void (*onchannelcallback)(void *context), void *context)
 789 {
 790         int err;
 791
 792         err = vmbus_alloc_ring(newchannel, send_ringbuffer_size,
 793                                recv_ringbuffer_size);
 794         if (err)
 795                 return err;
 796
 797         err = __vmbus_open(newchannel, userdata, userdatalen,
 798                            onchannelcallback, context);
 799         if (err)
 800                 vmbus_free_ring(newchannel);
 801
 802         return err;
 803 }
 804 EXPORT_SYMBOL_GPL(vmbus_open);
 805
 806 /*
 807  * vmbus_teardown_gpadl -Teardown the specified GPADL handle
 808  */
 809 int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle)
 810 {
 811         struct vmbus_channel_gpadl_teardown *msg;
 812         struct vmbus_channel_msginfo *info;
 813         unsigned long flags;
 814         int ret;
 815
 816         info = kzalloc(sizeof(*info) +
 817                        sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL);
 818         if (!info)
 819                 return -ENOMEM;
 820
 821         init_completion(&info->waitevent);
 822         info->waiting_channel = channel;
 823
 824         msg = (struct vmbus_channel_gpadl_teardown *)info->msg;
 825
 826         msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN;
 827         msg->child_relid = channel->offermsg.child_relid;
 828         msg->gpadl = gpadl_handle;
 829
 830         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 831         list_add_tail(&info->msglistentry,
 832                       &vmbus_connection.chn_msg_list);
 833         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 834
 835         if (channel->rescind)
 836                 goto post_msg_err;
 837
 838         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown),
 839                              true);
 840
 841         trace_vmbus_teardown_gpadl(msg, ret);
 842
 843         if (ret)
 844                 goto post_msg_err;
 845
 846         wait_for_completion(&info->waitevent);
 847
 848 post_msg_err:
 849         /*
 850          * If the channel has been rescinded;
 851          * we will be awakened by the rescind
 852          * handler; set the error code to zero so we don't leak memory.
 853          */
 854         if (channel->rescind)
 855                 ret = 0;
 856
 857         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 858         list_del(&info->msglistentry);
 859         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 860
 861         kfree(info);
 862         return ret;
 863 }
 864 EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl);
 865
 866 void vmbus_reset_channel_cb(struct vmbus_channel *channel)
 867 {
 868         unsigned long flags;
 869
 870         /*
 871          * vmbus_on_event(), running in the per-channel tasklet, can race
 872          * with vmbus_close_internal() in the case of SMP guest, e.g., when
 873          * the former is accessing channel->inbound.ring_buffer, the latter
 874          * could be freeing the ring_buffer pages, so here we must stop it
 875          * first.
 876          *
 877          * vmbus_chan_sched() might call the netvsc driver callback function
 878          * that ends up scheduling NAPI work that accesses the ring buffer.
 879          * At this point, we have to ensure that any such work is completed
 880          * and that the channel ring buffer is no longer being accessed, cf.
 881          * the calls to napi_disable() in netvsc_device_remove().
 882          */
 883         tasklet_disable(&channel->callback_event);
 884
 885         /* See the inline comments in vmbus_chan_sched(). */
 886         spin_lock_irqsave(&channel->sched_lock, flags);
 887         channel->onchannel_callback = NULL;
 888         spin_unlock_irqrestore(&channel->sched_lock, flags);
 889
 890         channel->sc_creation_callback = NULL;
 891
 892         /* Re-enable tasklet for use on re-open */
 893         tasklet_enable(&channel->callback_event);
 894 }
 895
 896 static int vmbus_close_internal(struct vmbus_channel *channel)
 897 {
 898         struct vmbus_channel_close_channel *msg;
 899         int ret;
 900
 901         vmbus_reset_channel_cb(channel);
 902
 903         /*
 904          * In case a device driver's probe() fails (e.g.,
 905          * util_probe() -> vmbus_open() returns -ENOMEM) and the device is
 906          * rescinded later (e.g., we dynamically disable an Integrated Service
 907          * in Hyper-V Manager), the driver's remove() invokes vmbus_close():
 908          * here we should skip most of the below cleanup work.
 909          */
 910         if (channel->state != CHANNEL_OPENED_STATE)
 911                 return -EINVAL;
 912
 913         channel->state = CHANNEL_OPEN_STATE;
 914
 915         /* Send a closing message */
 916
 917         msg = &channel->close_msg.msg;
 918
 919         msg->header.msgtype = CHANNELMSG_CLOSECHANNEL;
 920         msg->child_relid = channel->offermsg.child_relid;
 921
 922         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel),
 923                              true);
 924
 925         trace_vmbus_close_internal(msg, ret);
 926
 927         if (ret) {
 928                 pr_err("Close failed: close post msg return is %d\n", ret);
 929                 /*
 930                  * If we failed to post the close msg,
 931                  * it is perhaps better to leak memory.
 932                  */
 933         }
 934
 935         /* Tear down the gpadl for the channel's ring buffer */
 936         else if (channel->ringbuffer_gpadlhandle) {
 937                 ret = vmbus_teardown_gpadl(channel,
 938                                            channel->ringbuffer_gpadlhandle);
 939                 if (ret) {
 940                         pr_err("Close failed: teardown gpadl return %d\n", ret);
 941                         /*
 942                          * If we failed to teardown gpadl,
 943                          * it is perhaps better to leak memory.
 944                          */
 945                 }
 946
 947                 channel->ringbuffer_gpadlhandle = 0;
 948         }
 949
 950         if (!ret)
 951                 vmbus_free_requestor(&channel->requestor);
 952
 953         return ret;
 954 }
 955
 956 /* disconnect ring - close all channels */
 957 int vmbus_disconnect_ring(struct vmbus_channel *channel)
 958 {
 959         struct vmbus_channel *cur_channel, *tmp;
 960         int ret;
 961
 962         if (channel->primary_channel != NULL)
 963                 return -EINVAL;
 964
 965         list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) {
 966                 if (cur_channel->rescind)
 967                         wait_for_completion(&cur_channel->rescind_event);
 968
 969                 mutex_lock(&vmbus_connection.channel_mutex);
 970                 if (vmbus_close_internal(cur_channel) == 0) {
 971                         vmbus_free_ring(cur_channel);
 972
 973                         if (cur_channel->rescind)
 974                                 hv_process_channel_removal(cur_channel);
 975                 }
 976                 mutex_unlock(&vmbus_connection.channel_mutex);
 977         }
 978
 979         /*
 980          * Now close the primary.
 981          */
 982         mutex_lock(&vmbus_connection.channel_mutex);
 983         ret = vmbus_close_internal(channel);
 984         mutex_unlock(&vmbus_connection.channel_mutex);
 985
 986         return ret;
 987 }
 988 EXPORT_SYMBOL_GPL(vmbus_disconnect_ring);
 989
 990 /*
 991  * vmbus_close - Close the specified channel
 992  */
 993 void vmbus_close(struct vmbus_channel *channel)
 994 {
 995         if (vmbus_disconnect_ring(channel) == 0)
 996                 vmbus_free_ring(channel);
 997 }
 998 EXPORT_SYMBOL_GPL(vmbus_close);
 999
1000 /**
1001  * vmbus_sendpacket() - Send the specified buffer on the given channel
1002  * @channel: Pointer to vmbus_channel structure
1003  * @buffer: Pointer to the buffer you want to send the data from.
1004  * @bufferlen: Maximum size of what the buffer holds.
1005  * @requestid: Identifier of the request
1006  * @type: Type of packet that is being sent e.g. negotiate, time
1007  *        packet etc.
1008  * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED
1009  *
1010  * Sends data in @buffer directly to Hyper-V via the vmbus.
1011  * This will send the data unparsed to Hyper-V.
1012  *
1013  * Mainly used by Hyper-V drivers.
1014  */
1015 int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
1016                            u32 bufferlen, u64 requestid,
1017                            enum vmbus_packet_type type, u32 flags)
1018 {
1019         struct vmpacket_descriptor desc;
1020         u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen;
1021         u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1022         struct kvec bufferlist[3];
1023         u64 aligned_data = 0;
1024         int num_vecs = ((bufferlen != 0) ? 3 : 1);
1025
1026
1027         /* Setup the descriptor */
1028         desc.type = type; /* VmbusPacketTypeDataInBand; */
1029         desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */
1030         /* in 8-bytes granularity */
1031         desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3;
1032         desc.len8 = (u16)(packetlen_aligned >> 3);
1033         desc.trans_id = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1034
1035         bufferlist[0].iov_base = &desc;
1036         bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor);
1037         bufferlist[1].iov_base = buffer;
1038         bufferlist[1].iov_len = bufferlen;
1039         bufferlist[2].iov_base = &aligned_data;
1040         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1041
1042         return hv_ringbuffer_write(channel, bufferlist, num_vecs, requestid);
1043 }
1044 EXPORT_SYMBOL(vmbus_sendpacket);
1045
1046 /*
1047  * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
1048  * packets using a GPADL Direct packet type. This interface allows you
1049  * to control notifying the host. This will be useful for sending
1050  * batched data. Also the sender can control the send flags
1051  * explicitly.
1052  */
1053 int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
1054                                 struct hv_page_buffer pagebuffers[],
1055                                 u32 pagecount, void *buffer, u32 bufferlen,
1056                                 u64 requestid)
1057 {
1058         int i;
1059         struct vmbus_channel_packet_page_buffer desc;
1060         u32 descsize;
1061         u32 packetlen;
1062         u32 packetlen_aligned;
1063         struct kvec bufferlist[3];
1064         u64 aligned_data = 0;
1065
1066         if (pagecount > MAX_PAGE_BUFFER_COUNT)
1067                 return -EINVAL;
1068
1069         /*
1070          * Adjust the size down since vmbus_channel_packet_page_buffer is the
1071          * largest size we support
1072          */
1073         descsize = sizeof(struct vmbus_channel_packet_page_buffer) -
1074                           ((MAX_PAGE_BUFFER_COUNT - pagecount) *
1075                           sizeof(struct hv_page_buffer));
1076         packetlen = descsize + bufferlen;
1077         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1078
1079         /* Setup the descriptor */
1080         desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
1081         desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
1082         desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
1083         desc.length8 = (u16)(packetlen_aligned >> 3);
1084         desc.transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1085         desc.reserved = 0;
1086         desc.rangecount = pagecount;
1087
1088         for (i = 0; i < pagecount; i++) {
1089                 desc.range[i].len = pagebuffers[i].len;
1090                 desc.range[i].offset = pagebuffers[i].offset;
1091                 desc.range[i].pfn        = pagebuffers[i].pfn;
1092         }
1093
1094         bufferlist[0].iov_base = &desc;
1095         bufferlist[0].iov_len = descsize;
1096         bufferlist[1].iov_base = buffer;
1097         bufferlist[1].iov_len = bufferlen;
1098         bufferlist[2].iov_base = &aligned_data;
1099         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1100
1101         return hv_ringbuffer_write(channel, bufferlist, 3, requestid);
1102 }
1103 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer);
1104
1105 /*
1106  * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet
1107  * using a GPADL Direct packet type.
1108  * The buffer includes the vmbus descriptor.
1109  */
1110 int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
1111                               struct vmbus_packet_mpb_array *desc,
1112                               u32 desc_size,
1113                               void *buffer, u32 bufferlen, u64 requestid)
1114 {
1115         u32 packetlen;
1116         u32 packetlen_aligned;
1117         struct kvec bufferlist[3];
1118         u64 aligned_data = 0;
1119
1120         packetlen = desc_size + bufferlen;
1121         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1122
1123         /* Setup the descriptor */
1124         desc->type = VM_PKT_DATA_USING_GPA_DIRECT;
1125         desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
1126         desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */
1127         desc->length8 = (u16)(packetlen_aligned >> 3);
1128         desc->transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1129         desc->reserved = 0;
1130         desc->rangecount = 1;
1131
1132         bufferlist[0].iov_base = desc;
1133         bufferlist[0].iov_len = desc_size;
1134         bufferlist[1].iov_base = buffer;
1135         bufferlist[1].iov_len = bufferlen;
1136         bufferlist[2].iov_base = &aligned_data;
1137         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1138
1139         return hv_ringbuffer_write(channel, bufferlist, 3, requestid);
1140 }
1141 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
1142
1143 /**
1144  * __vmbus_recvpacket() - Retrieve the user packet on the specified channel
1145  * @channel: Pointer to vmbus_channel structure
1146  * @buffer: Pointer to the buffer you want to receive the data into.
1147  * @bufferlen: Maximum size of what the buffer can hold.
1148  * @buffer_actual_len: The actual size of the data after it was received.
1149  * @requestid: Identifier of the request
1150  * @raw: true means keep the vmpacket_descriptor header in the received data.
1151  *
1152  * Receives directly from the hyper-v vmbus and puts the data it received
1153  * into Buffer. This will receive the data unparsed from hyper-v.
1154  *
1155  * Mainly used by Hyper-V drivers.
1156  */
1157 static inline int
1158 __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1159                    u32 bufferlen, u32 *buffer_actual_len, u64 *requestid,
1160                    bool raw)
1161 {
1162         return hv_ringbuffer_read(channel, buffer, bufferlen,
1163                                   buffer_actual_len, requestid, raw);
1164
1165 }
1166
1167 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1168                      u32 bufferlen, u32 *buffer_actual_len,
1169                      u64 *requestid)
1170 {
1171         return __vmbus_recvpacket(channel, buffer, bufferlen,
1172                                   buffer_actual_len, requestid, false);
1173 }
1174 EXPORT_SYMBOL(vmbus_recvpacket);
1175
1176 /*
1177  * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel
1178  */
1179 int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer,
1180                               u32 bufferlen, u32 *buffer_actual_len,
1181                               u64 *requestid)
1182 {
1183         return __vmbus_recvpacket(channel, buffer, bufferlen,
1184                                   buffer_actual_len, requestid, true);
1185 }
1186 EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw);
1187
1188 /*
1189  * vmbus_next_request_id - Returns a new request id. It is also
1190  * the index at which the guest memory address is stored.
1191  * Uses a spin lock to avoid race conditions.
1192  * @channel: Pointer to the VMbus channel struct
1193  * @rqst_add: Guest memory address to be stored in the array
1194  */
1195 u64 vmbus_next_request_id(struct vmbus_channel *channel, u64 rqst_addr)
1196 {
1197         struct vmbus_requestor *rqstor = &channel->requestor;
1198         unsigned long flags;
1199         u64 current_id;
1200
1201         /* Check rqstor has been initialized */
1202         if (!channel->rqstor_size)
1203                 return VMBUS_NO_RQSTOR;
1204
1205         spin_lock_irqsave(&rqstor->req_lock, flags);
1206         current_id = rqstor->next_request_id;
1207
1208         /* Requestor array is full */
1209         if (current_id >= rqstor->size) {
1210                 spin_unlock_irqrestore(&rqstor->req_lock, flags);
1211                 return VMBUS_RQST_ERROR;
1212         }
1213
1214         rqstor->next_request_id = rqstor->req_arr[current_id];
1215         rqstor->req_arr[current_id] = rqst_addr;
1216
1217         /* The already held spin lock provides atomicity */
1218         bitmap_set(rqstor->req_bitmap, current_id, 1);
1219
1220         spin_unlock_irqrestore(&rqstor->req_lock, flags);
1221
1222         /*
1223          * Cannot return an ID of 0, which is reserved for an unsolicited
1224          * message from Hyper-V.
1225          */
1226         return current_id + 1;
1227 }
1228 EXPORT_SYMBOL_GPL(vmbus_next_request_id);
1229
1230 /*
1231  * vmbus_request_addr - Returns the memory address stored at @trans_id
1232  * in @rqstor. Uses a spin lock to avoid race conditions.
1233  * @channel: Pointer to the VMbus channel struct
1234  * @trans_id: Request id sent back from Hyper-V. Becomes the requestor's
1235  * next request id.
1236  */
1237 u64 vmbus_request_addr(struct vmbus_channel *channel, u64 trans_id)
1238 {
1239         struct vmbus_requestor *rqstor = &channel->requestor;
1240         unsigned long flags;
1241         u64 req_addr;
1242
1243         /* Check rqstor has been initialized */
1244         if (!channel->rqstor_size)
1245                 return VMBUS_NO_RQSTOR;
1246
1247         /* Hyper-V can send an unsolicited message with ID of 0 */
1248         if (!trans_id)
1249                 return trans_id;
1250
1251         spin_lock_irqsave(&rqstor->req_lock, flags);
1252
1253         /* Data corresponding to trans_id is stored at trans_id - 1 */
1254         trans_id--;
1255
1256         /* Invalid trans_id */
1257         if (trans_id >= rqstor->size || !test_bit(trans_id, rqstor->req_bitmap)) {
1258                 spin_unlock_irqrestore(&rqstor->req_lock, flags);
1259                 return VMBUS_RQST_ERROR;
1260         }
1261
1262         req_addr = rqstor->req_arr[trans_id];
1263         rqstor->req_arr[trans_id] = rqstor->next_request_id;
1264         rqstor->next_request_id = trans_id;
1265
1266         /* The already held spin lock provides atomicity */
1267         bitmap_clear(rqstor->req_bitmap, trans_id, 1);
1268
1269         spin_unlock_irqrestore(&rqstor->req_lock, flags);
1270         return req_addr;
1271 }
1272 EXPORT_SYMBOL_GPL(vmbus_request_addr);