drivers/hv/channel.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2009, Microsoft Corporation.
   4  *
   5  * Authors:
   6  *   Haiyang Zhang <haiyangz@microsoft.com>
   7  *   Hank Janssen  <hjanssen@microsoft.com>
   8  */
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/kernel.h>
  12 #include <linux/sched.h>
  13 #include <linux/wait.h>
  14 #include <linux/mm.h>
  15 #include <linux/slab.h>
  16 #include <linux/module.h>
  17 #include <linux/hyperv.h>
  18 #include <linux/uio.h>
  19 #include <linux/interrupt.h>
  20 #include <asm/page.h>
  21 #include <asm/mshyperv.h>
  22
  23 #include "hyperv_vmbus.h"
  24
  25 /*
  26  * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses
  27  *
  28  * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does.
  29  *
  30  * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header
  31  * (because of the alignment requirement), however, the hypervisor only
  32  * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a
  33  * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a
  34  * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the
  35  * total size that the guest uses minus twice of the gap size.
  36  */
  37 static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size)
  38 {
  39         switch (type) {
  40         case HV_GPADL_BUFFER:
  41                 return size;
  42         case HV_GPADL_RING:
  43                 /* The size of a ringbuffer must be page-aligned */
  44                 BUG_ON(size % PAGE_SIZE);
  45                 /*
  46                  * Two things to notice here:
  47                  * 1) We're processing two ring buffers as a unit
  48                  * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in
  49                  * the first guest-size page of each of the two ring buffers.
  50                  * So we effectively subtract out two guest-size pages, and add
  51                  * back two Hyper-V size pages.
  52                  */
  53                 return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
  54         }
  55         BUG();
  56         return 0;
  57 }
  58
  59 /*
  60  * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of
  61  *                                 HV_HYP_PAGE) in a ring gpadl based on the
  62  *                                 offset in the guest
  63  *
  64  * @offset: the offset (in bytes) where the send ringbuffer starts in the
  65  *               virtual address space of the guest
  66  */
  67 static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset)
  68 {
  69
  70         /*
  71          * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the
  72          * header (because of the alignment requirement), however, the
  73          * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header,
  74          * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap.
  75          *
  76          * And to calculate the effective send offset in gpadl, we need to
  77          * substract this gap.
  78          */
  79         return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT;
  80 }
  81
  82 /*
  83  * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in
  84  *                  the gpadl
  85  *
  86  * @type: the type of the gpadl
  87  * @kbuffer: the pointer to the gpadl in the guest
  88  * @size: the total size (in bytes) of the gpadl
  89  * @send_offset: the offset (in bytes) where the send ringbuffer starts in the
  90  *               virtual address space of the guest
  91  * @i: the index
  92  */
  93 static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer,
  94                                  u32 size, u32 send_offset, int i)
  95 {
  96         int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset);
  97         unsigned long delta = 0UL;
  98
  99         switch (type) {
 100         case HV_GPADL_BUFFER:
 101                 break;
 102         case HV_GPADL_RING:
 103                 if (i == 0)
 104                         delta = 0;
 105                 else if (i <= send_idx)
 106                         delta = PAGE_SIZE - HV_HYP_PAGE_SIZE;
 107                 else
 108                         delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
 109                 break;
 110         default:
 111                 BUG();
 112                 break;
 113         }
 114
 115         return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i));
 116 }
 117
 118 /*
 119  * vmbus_setevent- Trigger an event notification on the specified
 120  * channel.
 121  */
 122 void vmbus_setevent(struct vmbus_channel *channel)
 123 {
 124         struct hv_monitor_page *monitorpage;
 125
 126         trace_vmbus_setevent(channel);
 127
 128         /*
 129          * For channels marked as in "low latency" mode
 130          * bypass the monitor page mechanism.
 131          */
 132         if (channel->offermsg.monitor_allocated && !channel->low_latency) {
 133                 vmbus_send_interrupt(channel->offermsg.child_relid);
 134
 135                 /* Get the child to parent monitor page */
 136                 monitorpage = vmbus_connection.monitor_pages[1];
 137
 138                 sync_set_bit(channel->monitor_bit,
 139                         (unsigned long *)&monitorpage->trigger_group
 140                                         [channel->monitor_grp].pending);
 141
 142         } else {
 143                 vmbus_set_event(channel);
 144         }
 145 }
 146 EXPORT_SYMBOL_GPL(vmbus_setevent);
 147
 148 /* vmbus_free_ring - drop mapping of ring buffer */
 149 void vmbus_free_ring(struct vmbus_channel *channel)
 150 {
 151         hv_ringbuffer_cleanup(&channel->outbound);
 152         hv_ringbuffer_cleanup(&channel->inbound);
 153
 154         if (channel->ringbuffer_page) {
 155                 __free_pages(channel->ringbuffer_page,
 156                              get_order(channel->ringbuffer_pagecount
 157                                        << PAGE_SHIFT));
 158                 channel->ringbuffer_page = NULL;
 159         }
 160 }
 161 EXPORT_SYMBOL_GPL(vmbus_free_ring);
 162
 163 /* vmbus_alloc_ring - allocate and map pages for ring buffer */
 164 int vmbus_alloc_ring(struct vmbus_channel *newchannel,
 165                      u32 send_size, u32 recv_size)
 166 {
 167         struct page *page;
 168         int order;
 169
 170         if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE)
 171                 return -EINVAL;
 172
 173         /* Allocate the ring buffer */
 174         order = get_order(send_size + recv_size);
 175         page = alloc_pages_node(cpu_to_node(newchannel->target_cpu),
 176                                 GFP_KERNEL|__GFP_ZERO, order);
 177
 178         if (!page)
 179                 page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order);
 180
 181         if (!page)
 182                 return -ENOMEM;
 183
 184         newchannel->ringbuffer_page = page;
 185         newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT;
 186         newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT;
 187
 188         return 0;
 189 }
 190 EXPORT_SYMBOL_GPL(vmbus_alloc_ring);
 191
 192 /* Used for Hyper-V Socket: a guest client's connect() to the host */
 193 int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id,
 194                                   const guid_t *shv_host_servie_id)
 195 {
 196         struct vmbus_channel_tl_connect_request conn_msg;
 197         int ret;
 198
 199         memset(&conn_msg, 0, sizeof(conn_msg));
 200         conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST;
 201         conn_msg.guest_endpoint_id = *shv_guest_servie_id;
 202         conn_msg.host_service_id = *shv_host_servie_id;
 203
 204         ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true);
 205
 206         trace_vmbus_send_tl_connect_request(&conn_msg, ret);
 207
 208         return ret;
 209 }
 210 EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request);
 211
 212 static int send_modifychannel_without_ack(struct vmbus_channel *channel, u32 target_vp)
 213 {
 214         struct vmbus_channel_modifychannel msg;
 215         int ret;
 216
 217         memset(&msg, 0, sizeof(msg));
 218         msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL;
 219         msg.child_relid = channel->offermsg.child_relid;
 220         msg.target_vp = target_vp;
 221
 222         ret = vmbus_post_msg(&msg, sizeof(msg), true);
 223         trace_vmbus_send_modifychannel(&msg, ret);
 224
 225         return ret;
 226 }
 227
 228 static int send_modifychannel_with_ack(struct vmbus_channel *channel, u32 target_vp)
 229 {
 230         struct vmbus_channel_modifychannel *msg;
 231         struct vmbus_channel_msginfo *info;
 232         unsigned long flags;
 233         int ret;
 234
 235         info = kzalloc(sizeof(struct vmbus_channel_msginfo) +
 236                                 sizeof(struct vmbus_channel_modifychannel),
 237                        GFP_KERNEL);
 238         if (!info)
 239                 return -ENOMEM;
 240
 241         init_completion(&info->waitevent);
 242         info->waiting_channel = channel;
 243
 244         msg = (struct vmbus_channel_modifychannel *)info->msg;
 245         msg->header.msgtype = CHANNELMSG_MODIFYCHANNEL;
 246         msg->child_relid = channel->offermsg.child_relid;
 247         msg->target_vp = target_vp;
 248
 249         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 250         list_add_tail(&info->msglistentry, &vmbus_connection.chn_msg_list);
 251         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 252
 253         ret = vmbus_post_msg(msg, sizeof(*msg), true);
 254         trace_vmbus_send_modifychannel(msg, ret);
 255         if (ret != 0) {
 256                 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 257                 list_del(&info->msglistentry);
 258                 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 259                 goto free_info;
 260         }
 261
 262         /*
 263          * Release channel_mutex; otherwise, vmbus_onoffer_rescind() could block on
 264          * the mutex and be unable to signal the completion.
 265          *
 266          * See the caller target_cpu_store() for information about the usage of the
 267          * mutex.
 268          */
 269         mutex_unlock(&vmbus_connection.channel_mutex);
 270         wait_for_completion(&info->waitevent);
 271         mutex_lock(&vmbus_connection.channel_mutex);
 272
 273         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 274         list_del(&info->msglistentry);
 275         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 276
 277         if (info->response.modify_response.status)
 278                 ret = -EAGAIN;
 279
 280 free_info:
 281         kfree(info);
 282         return ret;
 283 }
 284
 285 /*
 286  * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt.
 287  *
 288  * CHANNELMSG_MODIFYCHANNEL messages are aynchronous.  When VMbus version 5.3
 289  * or later is negotiated, Hyper-V always sends an ACK in response to such a
 290  * message.  For VMbus version 5.2 and earlier, it never sends an ACK.  With-
 291  * out an ACK, we can not know when the host will stop interrupting the "old"
 292  * vCPU and start interrupting the "new" vCPU for the given channel.
 293  *
 294  * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version
 295  * VERSION_WIN10_V4_1.
 296  */
 297 int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp)
 298 {
 299         if (vmbus_proto_version >= VERSION_WIN10_V5_3)
 300                 return send_modifychannel_with_ack(channel, target_vp);
 301         return send_modifychannel_without_ack(channel, target_vp);
 302 }
 303 EXPORT_SYMBOL_GPL(vmbus_send_modifychannel);
 304
 305 /*
 306  * create_gpadl_header - Creates a gpadl for the specified buffer
 307  */
 308 static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer,
 309                                u32 size, u32 send_offset,
 310                                struct vmbus_channel_msginfo **msginfo)
 311 {
 312         int i;
 313         int pagecount;
 314         struct vmbus_channel_gpadl_header *gpadl_header;
 315         struct vmbus_channel_gpadl_body *gpadl_body;
 316         struct vmbus_channel_msginfo *msgheader;
 317         struct vmbus_channel_msginfo *msgbody = NULL;
 318         u32 msgsize;
 319
 320         int pfnsum, pfncount, pfnleft, pfncurr, pfnsize;
 321
 322         pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT;
 323
 324         /* do we need a gpadl body msg */
 325         pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 326                   sizeof(struct vmbus_channel_gpadl_header) -
 327                   sizeof(struct gpa_range);
 328         pfncount = pfnsize / sizeof(u64);
 329
 330         if (pagecount > pfncount) {
 331                 /* we need a gpadl body */
 332                 /* fill in the header */
 333                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 334                           sizeof(struct vmbus_channel_gpadl_header) +
 335                           sizeof(struct gpa_range) + pfncount * sizeof(u64);
 336                 msgheader =  kzalloc(msgsize, GFP_KERNEL);
 337                 if (!msgheader)
 338                         goto nomem;
 339
 340                 INIT_LIST_HEAD(&msgheader->submsglist);
 341                 msgheader->msgsize = msgsize;
 342
 343                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 344                         msgheader->msg;
 345                 gpadl_header->rangecount = 1;
 346                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 347                                          pagecount * sizeof(u64);
 348                 gpadl_header->range[0].byte_offset = 0;
 349                 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
 350                 for (i = 0; i < pfncount; i++)
 351                         gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
 352                                 type, kbuffer, size, send_offset, i);
 353                 *msginfo = msgheader;
 354
 355                 pfnsum = pfncount;
 356                 pfnleft = pagecount - pfncount;
 357
 358                 /* how many pfns can we fit */
 359                 pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 360                           sizeof(struct vmbus_channel_gpadl_body);
 361                 pfncount = pfnsize / sizeof(u64);
 362
 363                 /* fill in the body */
 364                 while (pfnleft) {
 365                         if (pfnleft > pfncount)
 366                                 pfncurr = pfncount;
 367                         else
 368                                 pfncurr = pfnleft;
 369
 370                         msgsize = sizeof(struct vmbus_channel_msginfo) +
 371                                   sizeof(struct vmbus_channel_gpadl_body) +
 372                                   pfncurr * sizeof(u64);
 373                         msgbody = kzalloc(msgsize, GFP_KERNEL);
 374
 375                         if (!msgbody) {
 376                                 struct vmbus_channel_msginfo *pos = NULL;
 377                                 struct vmbus_channel_msginfo *tmp = NULL;
 378                                 /*
 379                                  * Free up all the allocated messages.
 380                                  */
 381                                 list_for_each_entry_safe(pos, tmp,
 382                                         &msgheader->submsglist,
 383                                         msglistentry) {
 384
 385                                         list_del(&pos->msglistentry);
 386                                         kfree(pos);
 387                                 }
 388
 389                                 goto nomem;
 390                         }
 391
 392                         msgbody->msgsize = msgsize;
 393                         gpadl_body =
 394                                 (struct vmbus_channel_gpadl_body *)msgbody->msg;
 395
 396                         /*
 397                          * Gpadl is u32 and we are using a pointer which could
 398                          * be 64-bit
 399                          * This is governed by the guest/host protocol and
 400                          * so the hypervisor guarantees that this is ok.
 401                          */
 402                         for (i = 0; i < pfncurr; i++)
 403                                 gpadl_body->pfn[i] = hv_gpadl_hvpfn(type,
 404                                         kbuffer, size, send_offset, pfnsum + i);
 405
 406                         /* add to msg header */
 407                         list_add_tail(&msgbody->msglistentry,
 408                                       &msgheader->submsglist);
 409                         pfnsum += pfncurr;
 410                         pfnleft -= pfncurr;
 411                 }
 412         } else {
 413                 /* everything fits in a header */
 414                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 415                           sizeof(struct vmbus_channel_gpadl_header) +
 416                           sizeof(struct gpa_range) + pagecount * sizeof(u64);
 417                 msgheader = kzalloc(msgsize, GFP_KERNEL);
 418                 if (msgheader == NULL)
 419                         goto nomem;
 420
 421                 INIT_LIST_HEAD(&msgheader->submsglist);
 422                 msgheader->msgsize = msgsize;
 423
 424                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 425                         msgheader->msg;
 426                 gpadl_header->rangecount = 1;
 427                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 428                                          pagecount * sizeof(u64);
 429                 gpadl_header->range[0].byte_offset = 0;
 430                 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
 431                 for (i = 0; i < pagecount; i++)
 432                         gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
 433                                 type, kbuffer, size, send_offset, i);
 434
 435                 *msginfo = msgheader;
 436         }
 437
 438         return 0;
 439 nomem:
 440         kfree(msgheader);
 441         kfree(msgbody);
 442         return -ENOMEM;
 443 }
 444
 445 /*
 446  * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer
 447  *
 448  * @channel: a channel
 449  * @type: the type of the corresponding GPADL, only meaningful for the guest.
 450  * @kbuffer: from kmalloc or vmalloc
 451  * @size: page-size multiple
 452  * @send_offset: the offset (in bytes) where the send ring buffer starts,
 453  *              should be 0 for BUFFER type gpadl
 454  * @gpadl_handle: some funky thing
 455  */
 456 static int __vmbus_establish_gpadl(struct vmbus_channel *channel,
 457                                    enum hv_gpadl_type type, void *kbuffer,
 458                                    u32 size, u32 send_offset,
 459                                    u32 *gpadl_handle)
 460 {
 461         struct vmbus_channel_gpadl_header *gpadlmsg;
 462         struct vmbus_channel_gpadl_body *gpadl_body;
 463         struct vmbus_channel_msginfo *msginfo = NULL;
 464         struct vmbus_channel_msginfo *submsginfo, *tmp;
 465         struct list_head *curr;
 466         u32 next_gpadl_handle;
 467         unsigned long flags;
 468         int ret = 0;
 469
 470         next_gpadl_handle =
 471                 (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
 472
 473         ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo);
 474         if (ret)
 475                 return ret;
 476
 477         init_completion(&msginfo->waitevent);
 478         msginfo->waiting_channel = channel;
 479
 480         gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg;
 481         gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER;
 482         gpadlmsg->child_relid = channel->offermsg.child_relid;
 483         gpadlmsg->gpadl = next_gpadl_handle;
 484
 485
 486         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 487         list_add_tail(&msginfo->msglistentry,
 488                       &vmbus_connection.chn_msg_list);
 489
 490         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 491
 492         if (channel->rescind) {
 493                 ret = -ENODEV;
 494                 goto cleanup;
 495         }
 496
 497         ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize -
 498                              sizeof(*msginfo), true);
 499
 500         trace_vmbus_establish_gpadl_header(gpadlmsg, ret);
 501
 502         if (ret != 0)
 503                 goto cleanup;
 504
 505         list_for_each(curr, &msginfo->submsglist) {
 506                 submsginfo = (struct vmbus_channel_msginfo *)curr;
 507                 gpadl_body =
 508                         (struct vmbus_channel_gpadl_body *)submsginfo->msg;
 509
 510                 gpadl_body->header.msgtype =
 511                         CHANNELMSG_GPADL_BODY;
 512                 gpadl_body->gpadl = next_gpadl_handle;
 513
 514                 ret = vmbus_post_msg(gpadl_body,
 515                                      submsginfo->msgsize - sizeof(*submsginfo),
 516                                      true);
 517
 518                 trace_vmbus_establish_gpadl_body(gpadl_body, ret);
 519
 520                 if (ret != 0)
 521                         goto cleanup;
 522
 523         }
 524         wait_for_completion(&msginfo->waitevent);
 525
 526         if (msginfo->response.gpadl_created.creation_status != 0) {
 527                 pr_err("Failed to establish GPADL: err = 0x%x\n",
 528                        msginfo->response.gpadl_created.creation_status);
 529
 530                 ret = -EDQUOT;
 531                 goto cleanup;
 532         }
 533
 534         if (channel->rescind) {
 535                 ret = -ENODEV;
 536                 goto cleanup;
 537         }
 538
 539         /* At this point, we received the gpadl created msg */
 540         *gpadl_handle = gpadlmsg->gpadl;
 541
 542 cleanup:
 543         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 544         list_del(&msginfo->msglistentry);
 545         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 546         list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist,
 547                                  msglistentry) {
 548                 kfree(submsginfo);
 549         }
 550
 551         kfree(msginfo);
 552         return ret;
 553 }
 554
 555 /*
 556  * vmbus_establish_gpadl - Establish a GPADL for the specified buffer
 557  *
 558  * @channel: a channel
 559  * @kbuffer: from kmalloc or vmalloc
 560  * @size: page-size multiple
 561  * @gpadl_handle: some funky thing
 562  */
 563 int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 564                           u32 size, u32 *gpadl_handle)
 565 {
 566         return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size,
 567                                        0U, gpadl_handle);
 568 }
 569 EXPORT_SYMBOL_GPL(vmbus_establish_gpadl);
 570
 571 /**
 572  * request_arr_init - Allocates memory for the requestor array. Each slot
 573  * keeps track of the next available slot in the array. Initially, each
 574  * slot points to the next one (as in a Linked List). The last slot
 575  * does not point to anything, so its value is U64_MAX by default.
 576  * @size The size of the array
 577  */
 578 static u64 *request_arr_init(u32 size)
 579 {
 580         int i;
 581         u64 *req_arr;
 582
 583         req_arr = kcalloc(size, sizeof(u64), GFP_KERNEL);
 584         if (!req_arr)
 585                 return NULL;
 586
 587         for (i = 0; i < size - 1; i++)
 588                 req_arr[i] = i + 1;
 589
 590         /* Last slot (no more available slots) */
 591         req_arr[i] = U64_MAX;
 592
 593         return req_arr;
 594 }
 595
 596 /*
 597  * vmbus_alloc_requestor - Initializes @rqstor's fields.
 598  * Index 0 is the first free slot
 599  * @size: Size of the requestor array
 600  */
 601 static int vmbus_alloc_requestor(struct vmbus_requestor *rqstor, u32 size)
 602 {
 603         u64 *rqst_arr;
 604         unsigned long *bitmap;
 605
 606         rqst_arr = request_arr_init(size);
 607         if (!rqst_arr)
 608                 return -ENOMEM;
 609
 610         bitmap = bitmap_zalloc(size, GFP_KERNEL);
 611         if (!bitmap) {
 612                 kfree(rqst_arr);
 613                 return -ENOMEM;
 614         }
 615
 616         rqstor->req_arr = rqst_arr;
 617         rqstor->req_bitmap = bitmap;
 618         rqstor->size = size;
 619         rqstor->next_request_id = 0;
 620         spin_lock_init(&rqstor->req_lock);
 621
 622         return 0;
 623 }
 624
 625 /*
 626  * vmbus_free_requestor - Frees memory allocated for @rqstor
 627  * @rqstor: Pointer to the requestor struct
 628  */
 629 static void vmbus_free_requestor(struct vmbus_requestor *rqstor)
 630 {
 631         kfree(rqstor->req_arr);
 632         bitmap_free(rqstor->req_bitmap);
 633 }
 634
 635 static int __vmbus_open(struct vmbus_channel *newchannel,
 636                        void *userdata, u32 userdatalen,
 637                        void (*onchannelcallback)(void *context), void *context)
 638 {
 639         struct vmbus_channel_open_channel *open_msg;
 640         struct vmbus_channel_msginfo *open_info = NULL;
 641         struct page *page = newchannel->ringbuffer_page;
 642         u32 send_pages, recv_pages;
 643         unsigned long flags;
 644         int err;
 645
 646         if (userdatalen > MAX_USER_DEFINED_BYTES)
 647                 return -EINVAL;
 648
 649         send_pages = newchannel->ringbuffer_send_offset;
 650         recv_pages = newchannel->ringbuffer_pagecount - send_pages;
 651
 652         if (newchannel->state != CHANNEL_OPEN_STATE)
 653                 return -EINVAL;
 654
 655         /* Create and init requestor */
 656         if (newchannel->rqstor_size) {
 657                 if (vmbus_alloc_requestor(&newchannel->requestor, newchannel->rqstor_size))
 658                         return -ENOMEM;
 659         }
 660
 661         newchannel->state = CHANNEL_OPENING_STATE;
 662         newchannel->onchannel_callback = onchannelcallback;
 663         newchannel->channel_callback_context = context;
 664
 665         err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages);
 666         if (err)
 667                 goto error_clean_ring;
 668
 669         err = hv_ringbuffer_init(&newchannel->inbound,
 670                                  &page[send_pages], recv_pages);
 671         if (err)
 672                 goto error_clean_ring;
 673
 674         /* Establish the gpadl for the ring buffer */
 675         newchannel->ringbuffer_gpadlhandle = 0;
 676
 677         err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING,
 678                                       page_address(newchannel->ringbuffer_page),
 679                                       (send_pages + recv_pages) << PAGE_SHIFT,
 680                                       newchannel->ringbuffer_send_offset << PAGE_SHIFT,
 681                                       &newchannel->ringbuffer_gpadlhandle);
 682         if (err)
 683                 goto error_clean_ring;
 684
 685         /* Create and init the channel open message */
 686         open_info = kzalloc(sizeof(*open_info) +
 687                            sizeof(struct vmbus_channel_open_channel),
 688                            GFP_KERNEL);
 689         if (!open_info) {
 690                 err = -ENOMEM;
 691                 goto error_free_gpadl;
 692         }
 693
 694         init_completion(&open_info->waitevent);
 695         open_info->waiting_channel = newchannel;
 696
 697         open_msg = (struct vmbus_channel_open_channel *)open_info->msg;
 698         open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL;
 699         open_msg->openid = newchannel->offermsg.child_relid;
 700         open_msg->child_relid = newchannel->offermsg.child_relid;
 701         open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle;
 702         /*
 703          * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and
 704          * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so
 705          * here we calculate it into HV_HYP_PAGE.
 706          */
 707         open_msg->downstream_ringbuffer_pageoffset =
 708                 hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT);
 709         open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu);
 710
 711         if (userdatalen)
 712                 memcpy(open_msg->userdata, userdata, userdatalen);
 713
 714         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 715         list_add_tail(&open_info->msglistentry,
 716                       &vmbus_connection.chn_msg_list);
 717         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 718
 719         if (newchannel->rescind) {
 720                 err = -ENODEV;
 721                 goto error_clean_msglist;
 722         }
 723
 724         err = vmbus_post_msg(open_msg,
 725                              sizeof(struct vmbus_channel_open_channel), true);
 726
 727         trace_vmbus_open(open_msg, err);
 728
 729         if (err != 0)
 730                 goto error_clean_msglist;
 731
 732         wait_for_completion(&open_info->waitevent);
 733
 734         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 735         list_del(&open_info->msglistentry);
 736         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 737
 738         if (newchannel->rescind) {
 739                 err = -ENODEV;
 740                 goto error_free_info;
 741         }
 742
 743         if (open_info->response.open_result.status) {
 744                 err = -EAGAIN;
 745                 goto error_free_info;
 746         }
 747
 748         newchannel->state = CHANNEL_OPENED_STATE;
 749         kfree(open_info);
 750         return 0;
 751
 752 error_clean_msglist:
 753         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 754         list_del(&open_info->msglistentry);
 755         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 756 error_free_info:
 757         kfree(open_info);
 758 error_free_gpadl:
 759         vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle);
 760         newchannel->ringbuffer_gpadlhandle = 0;
 761 error_clean_ring:
 762         hv_ringbuffer_cleanup(&newchannel->outbound);
 763         hv_ringbuffer_cleanup(&newchannel->inbound);
 764         vmbus_free_requestor(&newchannel->requestor);
 765         newchannel->state = CHANNEL_OPEN_STATE;
 766         return err;
 767 }
 768
 769 /*
 770  * vmbus_connect_ring - Open the channel but reuse ring buffer
 771  */
 772 int vmbus_connect_ring(struct vmbus_channel *newchannel,
 773                        void (*onchannelcallback)(void *context), void *context)
 774 {
 775         return  __vmbus_open(newchannel, NULL, 0, onchannelcallback, context);
 776 }
 777 EXPORT_SYMBOL_GPL(vmbus_connect_ring);
 778
 779 /*
 780  * vmbus_open - Open the specified channel.
 781  */
 782 int vmbus_open(struct vmbus_channel *newchannel,
 783                u32 send_ringbuffer_size, u32 recv_ringbuffer_size,
 784                void *userdata, u32 userdatalen,
 785                void (*onchannelcallback)(void *context), void *context)
 786 {
 787         int err;
 788
 789         err = vmbus_alloc_ring(newchannel, send_ringbuffer_size,
 790                                recv_ringbuffer_size);
 791         if (err)
 792                 return err;
 793
 794         err = __vmbus_open(newchannel, userdata, userdatalen,
 795                            onchannelcallback, context);
 796         if (err)
 797                 vmbus_free_ring(newchannel);
 798
 799         return err;
 800 }
 801 EXPORT_SYMBOL_GPL(vmbus_open);
 802
 803 /*
 804  * vmbus_teardown_gpadl -Teardown the specified GPADL handle
 805  */
 806 int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle)
 807 {
 808         struct vmbus_channel_gpadl_teardown *msg;
 809         struct vmbus_channel_msginfo *info;
 810         unsigned long flags;
 811         int ret;
 812
 813         info = kzalloc(sizeof(*info) +
 814                        sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL);
 815         if (!info)
 816                 return -ENOMEM;
 817
 818         init_completion(&info->waitevent);
 819         info->waiting_channel = channel;
 820
 821         msg = (struct vmbus_channel_gpadl_teardown *)info->msg;
 822
 823         msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN;
 824         msg->child_relid = channel->offermsg.child_relid;
 825         msg->gpadl = gpadl_handle;
 826
 827         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 828         list_add_tail(&info->msglistentry,
 829                       &vmbus_connection.chn_msg_list);
 830         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 831
 832         if (channel->rescind)
 833                 goto post_msg_err;
 834
 835         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown),
 836                              true);
 837
 838         trace_vmbus_teardown_gpadl(msg, ret);
 839
 840         if (ret)
 841                 goto post_msg_err;
 842
 843         wait_for_completion(&info->waitevent);
 844
 845 post_msg_err:
 846         /*
 847          * If the channel has been rescinded;
 848          * we will be awakened by the rescind
 849          * handler; set the error code to zero so we don't leak memory.
 850          */
 851         if (channel->rescind)
 852                 ret = 0;
 853
 854         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 855         list_del(&info->msglistentry);
 856         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 857
 858         kfree(info);
 859         return ret;
 860 }
 861 EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl);
 862
 863 void vmbus_reset_channel_cb(struct vmbus_channel *channel)
 864 {
 865         unsigned long flags;
 866
 867         /*
 868          * vmbus_on_event(), running in the per-channel tasklet, can race
 869          * with vmbus_close_internal() in the case of SMP guest, e.g., when
 870          * the former is accessing channel->inbound.ring_buffer, the latter
 871          * could be freeing the ring_buffer pages, so here we must stop it
 872          * first.
 873          *
 874          * vmbus_chan_sched() might call the netvsc driver callback function
 875          * that ends up scheduling NAPI work that accesses the ring buffer.
 876          * At this point, we have to ensure that any such work is completed
 877          * and that the channel ring buffer is no longer being accessed, cf.
 878          * the calls to napi_disable() in netvsc_device_remove().
 879          */
 880         tasklet_disable(&channel->callback_event);
 881
 882         /* See the inline comments in vmbus_chan_sched(). */
 883         spin_lock_irqsave(&channel->sched_lock, flags);
 884         channel->onchannel_callback = NULL;
 885         spin_unlock_irqrestore(&channel->sched_lock, flags);
 886
 887         channel->sc_creation_callback = NULL;
 888
 889         /* Re-enable tasklet for use on re-open */
 890         tasklet_enable(&channel->callback_event);
 891 }
 892
 893 static int vmbus_close_internal(struct vmbus_channel *channel)
 894 {
 895         struct vmbus_channel_close_channel *msg;
 896         int ret;
 897
 898         vmbus_reset_channel_cb(channel);
 899
 900         /*
 901          * In case a device driver's probe() fails (e.g.,
 902          * util_probe() -> vmbus_open() returns -ENOMEM) and the device is
 903          * rescinded later (e.g., we dynamically disable an Integrated Service
 904          * in Hyper-V Manager), the driver's remove() invokes vmbus_close():
 905          * here we should skip most of the below cleanup work.
 906          */
 907         if (channel->state != CHANNEL_OPENED_STATE)
 908                 return -EINVAL;
 909
 910         channel->state = CHANNEL_OPEN_STATE;
 911
 912         /* Send a closing message */
 913
 914         msg = &channel->close_msg.msg;
 915
 916         msg->header.msgtype = CHANNELMSG_CLOSECHANNEL;
 917         msg->child_relid = channel->offermsg.child_relid;
 918
 919         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel),
 920                              true);
 921
 922         trace_vmbus_close_internal(msg, ret);
 923
 924         if (ret) {
 925                 pr_err("Close failed: close post msg return is %d\n", ret);
 926                 /*
 927                  * If we failed to post the close msg,
 928                  * it is perhaps better to leak memory.
 929                  */
 930         }
 931
 932         /* Tear down the gpadl for the channel's ring buffer */
 933         else if (channel->ringbuffer_gpadlhandle) {
 934                 ret = vmbus_teardown_gpadl(channel,
 935                                            channel->ringbuffer_gpadlhandle);
 936                 if (ret) {
 937                         pr_err("Close failed: teardown gpadl return %d\n", ret);
 938                         /*
 939                          * If we failed to teardown gpadl,
 940                          * it is perhaps better to leak memory.
 941                          */
 942                 }
 943
 944                 channel->ringbuffer_gpadlhandle = 0;
 945         }
 946
 947         if (!ret)
 948                 vmbus_free_requestor(&channel->requestor);
 949
 950         return ret;
 951 }
 952
 953 /* disconnect ring - close all channels */
 954 int vmbus_disconnect_ring(struct vmbus_channel *channel)
 955 {
 956         struct vmbus_channel *cur_channel, *tmp;
 957         int ret;
 958
 959         if (channel->primary_channel != NULL)
 960                 return -EINVAL;
 961
 962         list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) {
 963                 if (cur_channel->rescind)
 964                         wait_for_completion(&cur_channel->rescind_event);
 965
 966                 mutex_lock(&vmbus_connection.channel_mutex);
 967                 if (vmbus_close_internal(cur_channel) == 0) {
 968                         vmbus_free_ring(cur_channel);
 969
 970                         if (cur_channel->rescind)
 971                                 hv_process_channel_removal(cur_channel);
 972                 }
 973                 mutex_unlock(&vmbus_connection.channel_mutex);
 974         }
 975
 976         /*
 977          * Now close the primary.
 978          */
 979         mutex_lock(&vmbus_connection.channel_mutex);
 980         ret = vmbus_close_internal(channel);
 981         mutex_unlock(&vmbus_connection.channel_mutex);
 982
 983         return ret;
 984 }
 985 EXPORT_SYMBOL_GPL(vmbus_disconnect_ring);
 986
 987 /*
 988  * vmbus_close - Close the specified channel
 989  */
 990 void vmbus_close(struct vmbus_channel *channel)
 991 {
 992         if (vmbus_disconnect_ring(channel) == 0)
 993                 vmbus_free_ring(channel);
 994 }
 995 EXPORT_SYMBOL_GPL(vmbus_close);
 996
 997 /**
 998  * vmbus_sendpacket() - Send the specified buffer on the given channel
 999  * @channel: Pointer to vmbus_channel structure
1000  * @buffer: Pointer to the buffer you want to send the data from.
1001  * @bufferlen: Maximum size of what the buffer holds.
1002  * @requestid: Identifier of the request
1003  * @type: Type of packet that is being sent e.g. negotiate, time
1004  *        packet etc.
1005  * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED
1006  *
1007  * Sends data in @buffer directly to Hyper-V via the vmbus.
1008  * This will send the data unparsed to Hyper-V.
1009  *
1010  * Mainly used by Hyper-V drivers.
1011  */
1012 int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
1013                            u32 bufferlen, u64 requestid,
1014                            enum vmbus_packet_type type, u32 flags)
1015 {
1016         struct vmpacket_descriptor desc;
1017         u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen;
1018         u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1019         struct kvec bufferlist[3];
1020         u64 aligned_data = 0;
1021         int num_vecs = ((bufferlen != 0) ? 3 : 1);
1022
1023
1024         /* Setup the descriptor */
1025         desc.type = type; /* VmbusPacketTypeDataInBand; */
1026         desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */
1027         /* in 8-bytes granularity */
1028         desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3;
1029         desc.len8 = (u16)(packetlen_aligned >> 3);
1030         desc.trans_id = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1031
1032         bufferlist[0].iov_base = &desc;
1033         bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor);
1034         bufferlist[1].iov_base = buffer;
1035         bufferlist[1].iov_len = bufferlen;
1036         bufferlist[2].iov_base = &aligned_data;
1037         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1038
1039         return hv_ringbuffer_write(channel, bufferlist, num_vecs, requestid);
1040 }
1041 EXPORT_SYMBOL(vmbus_sendpacket);
1042
1043 /*
1044  * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
1045  * packets using a GPADL Direct packet type. This interface allows you
1046  * to control notifying the host. This will be useful for sending
1047  * batched data. Also the sender can control the send flags
1048  * explicitly.
1049  */
1050 int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
1051                                 struct hv_page_buffer pagebuffers[],
1052                                 u32 pagecount, void *buffer, u32 bufferlen,
1053                                 u64 requestid)
1054 {
1055         int i;
1056         struct vmbus_channel_packet_page_buffer desc;
1057         u32 descsize;
1058         u32 packetlen;
1059         u32 packetlen_aligned;
1060         struct kvec bufferlist[3];
1061         u64 aligned_data = 0;
1062
1063         if (pagecount > MAX_PAGE_BUFFER_COUNT)
1064                 return -EINVAL;
1065
1066         /*
1067          * Adjust the size down since vmbus_channel_packet_page_buffer is the
1068          * largest size we support
1069          */
1070         descsize = sizeof(struct vmbus_channel_packet_page_buffer) -
1071                           ((MAX_PAGE_BUFFER_COUNT - pagecount) *
1072                           sizeof(struct hv_page_buffer));
1073         packetlen = descsize + bufferlen;
1074         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1075
1076         /* Setup the descriptor */
1077         desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
1078         desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
1079         desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
1080         desc.length8 = (u16)(packetlen_aligned >> 3);
1081         desc.transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1082         desc.reserved = 0;
1083         desc.rangecount = pagecount;
1084
1085         for (i = 0; i < pagecount; i++) {
1086                 desc.range[i].len = pagebuffers[i].len;
1087                 desc.range[i].offset = pagebuffers[i].offset;
1088                 desc.range[i].pfn        = pagebuffers[i].pfn;
1089         }
1090
1091         bufferlist[0].iov_base = &desc;
1092         bufferlist[0].iov_len = descsize;
1093         bufferlist[1].iov_base = buffer;
1094         bufferlist[1].iov_len = bufferlen;
1095         bufferlist[2].iov_base = &aligned_data;
1096         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1097
1098         return hv_ringbuffer_write(channel, bufferlist, 3, requestid);
1099 }
1100 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer);
1101
1102 /*
1103  * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet
1104  * using a GPADL Direct packet type.
1105  * The buffer includes the vmbus descriptor.
1106  */
1107 int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
1108                               struct vmbus_packet_mpb_array *desc,
1109                               u32 desc_size,
1110                               void *buffer, u32 bufferlen, u64 requestid)
1111 {
1112         u32 packetlen;
1113         u32 packetlen_aligned;
1114         struct kvec bufferlist[3];
1115         u64 aligned_data = 0;
1116
1117         packetlen = desc_size + bufferlen;
1118         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
1119
1120         /* Setup the descriptor */
1121         desc->type = VM_PKT_DATA_USING_GPA_DIRECT;
1122         desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
1123         desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */
1124         desc->length8 = (u16)(packetlen_aligned >> 3);
1125         desc->transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */
1126         desc->reserved = 0;
1127         desc->rangecount = 1;
1128
1129         bufferlist[0].iov_base = desc;
1130         bufferlist[0].iov_len = desc_size;
1131         bufferlist[1].iov_base = buffer;
1132         bufferlist[1].iov_len = bufferlen;
1133         bufferlist[2].iov_base = &aligned_data;
1134         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
1135
1136         return hv_ringbuffer_write(channel, bufferlist, 3, requestid);
1137 }
1138 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
1139
1140 /**
1141  * __vmbus_recvpacket() - Retrieve the user packet on the specified channel
1142  * @channel: Pointer to vmbus_channel structure
1143  * @buffer: Pointer to the buffer you want to receive the data into.
1144  * @bufferlen: Maximum size of what the buffer can hold.
1145  * @buffer_actual_len: The actual size of the data after it was received.
1146  * @requestid: Identifier of the request
1147  * @raw: true means keep the vmpacket_descriptor header in the received data.
1148  *
1149  * Receives directly from the hyper-v vmbus and puts the data it received
1150  * into Buffer. This will receive the data unparsed from hyper-v.
1151  *
1152  * Mainly used by Hyper-V drivers.
1153  */
1154 static inline int
1155 __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1156                    u32 bufferlen, u32 *buffer_actual_len, u64 *requestid,
1157                    bool raw)
1158 {
1159         return hv_ringbuffer_read(channel, buffer, bufferlen,
1160                                   buffer_actual_len, requestid, raw);
1161
1162 }
1163
1164 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1165                      u32 bufferlen, u32 *buffer_actual_len,
1166                      u64 *requestid)
1167 {
1168         return __vmbus_recvpacket(channel, buffer, bufferlen,
1169                                   buffer_actual_len, requestid, false);
1170 }
1171 EXPORT_SYMBOL(vmbus_recvpacket);
1172
1173 /*
1174  * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel
1175  */
1176 int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer,
1177                               u32 bufferlen, u32 *buffer_actual_len,
1178                               u64 *requestid)
1179 {
1180         return __vmbus_recvpacket(channel, buffer, bufferlen,
1181                                   buffer_actual_len, requestid, true);
1182 }
1183 EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw);
1184
1185 /*
1186  * vmbus_next_request_id - Returns a new request id. It is also
1187  * the index at which the guest memory address is stored.
1188  * Uses a spin lock to avoid race conditions.
1189  * @rqstor: Pointer to the requestor struct
1190  * @rqst_add: Guest memory address to be stored in the array
1191  */
1192 u64 vmbus_next_request_id(struct vmbus_requestor *rqstor, u64 rqst_addr)
1193 {
1194         unsigned long flags;
1195         u64 current_id;
1196         const struct vmbus_channel *channel =
1197                 container_of(rqstor, const struct vmbus_channel, requestor);
1198
1199         /* Check rqstor has been initialized */
1200         if (!channel->rqstor_size)
1201                 return VMBUS_NO_RQSTOR;
1202
1203         spin_lock_irqsave(&rqstor->req_lock, flags);
1204         current_id = rqstor->next_request_id;
1205
1206         /* Requestor array is full */
1207         if (current_id >= rqstor->size) {
1208                 spin_unlock_irqrestore(&rqstor->req_lock, flags);
1209                 return VMBUS_RQST_ERROR;
1210         }
1211
1212         rqstor->next_request_id = rqstor->req_arr[current_id];
1213         rqstor->req_arr[current_id] = rqst_addr;
1214
1215         /* The already held spin lock provides atomicity */
1216         bitmap_set(rqstor->req_bitmap, current_id, 1);
1217
1218         spin_unlock_irqrestore(&rqstor->req_lock, flags);
1219
1220         /*
1221          * Cannot return an ID of 0, which is reserved for an unsolicited
1222          * message from Hyper-V.
1223          */
1224         return current_id + 1;
1225 }
1226 EXPORT_SYMBOL_GPL(vmbus_next_request_id);
1227
1228 /*
1229  * vmbus_request_addr - Returns the memory address stored at @trans_id
1230  * in @rqstor. Uses a spin lock to avoid race conditions.
1231  * @rqstor: Pointer to the requestor struct
1232  * @trans_id: Request id sent back from Hyper-V. Becomes the requestor's
1233  * next request id.
1234  */
1235 u64 vmbus_request_addr(struct vmbus_requestor *rqstor, u64 trans_id)
1236 {
1237         unsigned long flags;
1238         u64 req_addr;
1239         const struct vmbus_channel *channel =
1240                 container_of(rqstor, const struct vmbus_channel, requestor);
1241
1242         /* Check rqstor has been initialized */
1243         if (!channel->rqstor_size)
1244                 return VMBUS_NO_RQSTOR;
1245
1246         /* Hyper-V can send an unsolicited message with ID of 0 */
1247         if (!trans_id)
1248                 return trans_id;
1249
1250         spin_lock_irqsave(&rqstor->req_lock, flags);
1251
1252         /* Data corresponding to trans_id is stored at trans_id - 1 */
1253         trans_id--;
1254
1255         /* Invalid trans_id */
1256         if (trans_id >= rqstor->size || !test_bit(trans_id, rqstor->req_bitmap)) {
1257                 spin_unlock_irqrestore(&rqstor->req_lock, flags);
1258                 return VMBUS_RQST_ERROR;
1259         }
1260
1261         req_addr = rqstor->req_arr[trans_id];
1262         rqstor->req_arr[trans_id] = rqstor->next_request_id;
1263         rqstor->next_request_id = trans_id;
1264
1265         /* The already held spin lock provides atomicity */
1266         bitmap_clear(rqstor->req_bitmap, trans_id, 1);
1267
1268         spin_unlock_irqrestore(&rqstor->req_lock, flags);
1269         return req_addr;
1270 }
1271 EXPORT_SYMBOL_GPL(vmbus_request_addr);