arch/x86/kvm/svm/sev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM-SEV support
   6  *
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  */
   9
  10 #include <linux/kvm_types.h>
  11 #include <linux/kvm_host.h>
  12 #include <linux/kernel.h>
  13 #include <linux/highmem.h>
  14 #include <linux/psp-sev.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/swap.h>
  17 #include <linux/processor.h>
  18
  19 #include "x86.h"
  20 #include "svm.h"
  21
  22 static int sev_flush_asids(void);
  23 static DECLARE_RWSEM(sev_deactivate_lock);
  24 static DEFINE_MUTEX(sev_bitmap_lock);
  25 unsigned int max_sev_asid;
  26 static unsigned int min_sev_asid;
  27 static unsigned long *sev_asid_bitmap;
  28 static unsigned long *sev_reclaim_asid_bitmap;
  29 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
  30
  31 struct enc_region {
  32         struct list_head list;
  33         unsigned long npages;
  34         struct page **pages;
  35         unsigned long uaddr;
  36         unsigned long size;
  37 };
  38
  39 static int sev_flush_asids(void)
  40 {
  41         int ret, error = 0;
  42
  43         /*
  44          * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
  45          * so it must be guarded.
  46          */
  47         down_write(&sev_deactivate_lock);
  48
  49         wbinvd_on_all_cpus();
  50         ret = sev_guest_df_flush(&error);
  51
  52         up_write(&sev_deactivate_lock);
  53
  54         if (ret)
  55                 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error);
  56
  57         return ret;
  58 }
  59
  60 /* Must be called with the sev_bitmap_lock held */
  61 static bool __sev_recycle_asids(void)
  62 {
  63         int pos;
  64
  65         /* Check if there are any ASIDs to reclaim before performing a flush */
  66         pos = find_next_bit(sev_reclaim_asid_bitmap,
  67                             max_sev_asid, min_sev_asid - 1);
  68         if (pos >= max_sev_asid)
  69                 return false;
  70
  71         if (sev_flush_asids())
  72                 return false;
  73
  74         bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
  75                    max_sev_asid);
  76         bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
  77
  78         return true;
  79 }
  80
  81 static int sev_asid_new(void)
  82 {
  83         bool retry = true;
  84         int pos;
  85
  86         mutex_lock(&sev_bitmap_lock);
  87
  88         /*
  89          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
  90          */
  91 again:
  92         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
  93         if (pos >= max_sev_asid) {
  94                 if (retry && __sev_recycle_asids()) {
  95                         retry = false;
  96                         goto again;
  97                 }
  98                 mutex_unlock(&sev_bitmap_lock);
  99                 return -EBUSY;
 100         }
 101
 102         __set_bit(pos, sev_asid_bitmap);
 103
 104         mutex_unlock(&sev_bitmap_lock);
 105
 106         return pos + 1;
 107 }
 108
 109 static int sev_get_asid(struct kvm *kvm)
 110 {
 111         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 112
 113         return sev->asid;
 114 }
 115
 116 static void sev_asid_free(int asid)
 117 {
 118         struct svm_cpu_data *sd;
 119         int cpu, pos;
 120
 121         mutex_lock(&sev_bitmap_lock);
 122
 123         pos = asid - 1;
 124         __set_bit(pos, sev_reclaim_asid_bitmap);
 125
 126         for_each_possible_cpu(cpu) {
 127                 sd = per_cpu(svm_data, cpu);
 128                 sd->sev_vmcbs[pos] = NULL;
 129         }
 130
 131         mutex_unlock(&sev_bitmap_lock);
 132 }
 133
 134 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
 135 {
 136         struct sev_data_decommission *decommission;
 137         struct sev_data_deactivate *data;
 138
 139         if (!handle)
 140                 return;
 141
 142         data = kzalloc(sizeof(*data), GFP_KERNEL);
 143         if (!data)
 144                 return;
 145
 146         /* deactivate handle */
 147         data->handle = handle;
 148
 149         /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
 150         down_read(&sev_deactivate_lock);
 151         sev_guest_deactivate(data, NULL);
 152         up_read(&sev_deactivate_lock);
 153
 154         kfree(data);
 155
 156         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
 157         if (!decommission)
 158                 return;
 159
 160         /* decommission handle */
 161         decommission->handle = handle;
 162         sev_guest_decommission(decommission, NULL);
 163
 164         kfree(decommission);
 165 }
 166
 167 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
 168 {
 169         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 170         int asid, ret;
 171
 172         ret = -EBUSY;
 173         if (unlikely(sev->active))
 174                 return ret;
 175
 176         asid = sev_asid_new();
 177         if (asid < 0)
 178                 return ret;
 179
 180         ret = sev_platform_init(&argp->error);
 181         if (ret)
 182                 goto e_free;
 183
 184         sev->active = true;
 185         sev->asid = asid;
 186         INIT_LIST_HEAD(&sev->regions_list);
 187
 188         return 0;
 189
 190 e_free:
 191         sev_asid_free(asid);
 192         return ret;
 193 }
 194
 195 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
 196 {
 197         struct sev_data_activate *data;
 198         int asid = sev_get_asid(kvm);
 199         int ret;
 200
 201         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 202         if (!data)
 203                 return -ENOMEM;
 204
 205         /* activate ASID on the given handle */
 206         data->handle = handle;
 207         data->asid   = asid;
 208         ret = sev_guest_activate(data, error);
 209         kfree(data);
 210
 211         return ret;
 212 }
 213
 214 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
 215 {
 216         struct fd f;
 217         int ret;
 218
 219         f = fdget(fd);
 220         if (!f.file)
 221                 return -EBADF;
 222
 223         ret = sev_issue_cmd_external_user(f.file, id, data, error);
 224
 225         fdput(f);
 226         return ret;
 227 }
 228
 229 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
 230 {
 231         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 232
 233         return __sev_issue_cmd(sev->fd, id, data, error);
 234 }
 235
 236 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
 237 {
 238         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 239         struct sev_data_launch_start *start;
 240         struct kvm_sev_launch_start params;
 241         void *dh_blob, *session_blob;
 242         int *error = &argp->error;
 243         int ret;
 244
 245         if (!sev_guest(kvm))
 246                 return -ENOTTY;
 247
 248         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
 249                 return -EFAULT;
 250
 251         start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
 252         if (!start)
 253                 return -ENOMEM;
 254
 255         dh_blob = NULL;
 256         if (params.dh_uaddr) {
 257                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
 258                 if (IS_ERR(dh_blob)) {
 259                         ret = PTR_ERR(dh_blob);
 260                         goto e_free;
 261                 }
 262
 263                 start->dh_cert_address = __sme_set(__pa(dh_blob));
 264                 start->dh_cert_len = params.dh_len;
 265         }
 266
 267         session_blob = NULL;
 268         if (params.session_uaddr) {
 269                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
 270                 if (IS_ERR(session_blob)) {
 271                         ret = PTR_ERR(session_blob);
 272                         goto e_free_dh;
 273                 }
 274
 275                 start->session_address = __sme_set(__pa(session_blob));
 276                 start->session_len = params.session_len;
 277         }
 278
 279         start->handle = params.handle;
 280         start->policy = params.policy;
 281
 282         /* create memory encryption context */
 283         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
 284         if (ret)
 285                 goto e_free_session;
 286
 287         /* Bind ASID to this guest */
 288         ret = sev_bind_asid(kvm, start->handle, error);
 289         if (ret)
 290                 goto e_free_session;
 291
 292         /* return handle to userspace */
 293         params.handle = start->handle;
 294         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
 295                 sev_unbind_asid(kvm, start->handle);
 296                 ret = -EFAULT;
 297                 goto e_free_session;
 298         }
 299
 300         sev->handle = start->handle;
 301         sev->fd = argp->sev_fd;
 302
 303 e_free_session:
 304         kfree(session_blob);
 305 e_free_dh:
 306         kfree(dh_blob);
 307 e_free:
 308         kfree(start);
 309         return ret;
 310 }
 311
 312 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 313                                     unsigned long ulen, unsigned long *n,
 314                                     int write)
 315 {
 316         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 317         unsigned long npages, size;
 318         int npinned;
 319         unsigned long locked, lock_limit;
 320         struct page **pages;
 321         unsigned long first, last;
 322         int ret;
 323
 324         if (ulen == 0 || uaddr + ulen < uaddr)
 325                 return ERR_PTR(-EINVAL);
 326
 327         /* Calculate number of pages. */
 328         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
 329         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
 330         npages = (last - first + 1);
 331
 332         locked = sev->pages_locked + npages;
 333         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 334         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
 335                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
 336                 return ERR_PTR(-ENOMEM);
 337         }
 338
 339         if (WARN_ON_ONCE(npages > INT_MAX))
 340                 return ERR_PTR(-EINVAL);
 341
 342         /* Avoid using vmalloc for smaller buffers. */
 343         size = npages * sizeof(struct page *);
 344         if (size > PAGE_SIZE)
 345                 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 346         else
 347                 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
 348
 349         if (!pages)
 350                 return ERR_PTR(-ENOMEM);
 351
 352         /* Pin the user virtual address. */
 353         npinned = pin_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
 354         if (npinned != npages) {
 355                 pr_err("SEV: Failure locking %lu pages.\n", npages);
 356                 ret = -ENOMEM;
 357                 goto err;
 358         }
 359
 360         *n = npages;
 361         sev->pages_locked = locked;
 362
 363         return pages;
 364
 365 err:
 366         if (npinned > 0)
 367                 unpin_user_pages(pages, npinned);
 368
 369         kvfree(pages);
 370         return ERR_PTR(ret);
 371 }
 372
 373 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
 374                              unsigned long npages)
 375 {
 376         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 377
 378         unpin_user_pages(pages, npages);
 379         kvfree(pages);
 380         sev->pages_locked -= npages;
 381 }
 382
 383 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
 384 {
 385         uint8_t *page_virtual;
 386         unsigned long i;
 387
 388         if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 ||
 389             pages == NULL)
 390                 return;
 391
 392         for (i = 0; i < npages; i++) {
 393                 page_virtual = kmap_atomic(pages[i]);
 394                 clflush_cache_range(page_virtual, PAGE_SIZE);
 395                 kunmap_atomic(page_virtual);
 396         }
 397 }
 398
 399 static unsigned long get_num_contig_pages(unsigned long idx,
 400                                 struct page **inpages, unsigned long npages)
 401 {
 402         unsigned long paddr, next_paddr;
 403         unsigned long i = idx + 1, pages = 1;
 404
 405         /* find the number of contiguous pages starting from idx */
 406         paddr = __sme_page_pa(inpages[idx]);
 407         while (i < npages) {
 408                 next_paddr = __sme_page_pa(inpages[i++]);
 409                 if ((paddr + PAGE_SIZE) == next_paddr) {
 410                         pages++;
 411                         paddr = next_paddr;
 412                         continue;
 413                 }
 414                 break;
 415         }
 416
 417         return pages;
 418 }
 419
 420 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 421 {
 422         unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
 423         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 424         struct kvm_sev_launch_update_data params;
 425         struct sev_data_launch_update_data *data;
 426         struct page **inpages;
 427         int ret;
 428
 429         if (!sev_guest(kvm))
 430                 return -ENOTTY;
 431
 432         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
 433                 return -EFAULT;
 434
 435         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 436         if (!data)
 437                 return -ENOMEM;
 438
 439         vaddr = params.uaddr;
 440         size = params.len;
 441         vaddr_end = vaddr + size;
 442
 443         /* Lock the user memory. */
 444         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
 445         if (IS_ERR(inpages)) {
 446                 ret = PTR_ERR(inpages);
 447                 goto e_free;
 448         }
 449
 450         /*
 451          * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in
 452          * place; the cache may contain the data that was written unencrypted.
 453          */
 454         sev_clflush_pages(inpages, npages);
 455
 456         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
 457                 int offset, len;
 458
 459                 /*
 460                  * If the user buffer is not page-aligned, calculate the offset
 461                  * within the page.
 462                  */
 463                 offset = vaddr & (PAGE_SIZE - 1);
 464
 465                 /* Calculate the number of pages that can be encrypted in one go. */
 466                 pages = get_num_contig_pages(i, inpages, npages);
 467
 468                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
 469
 470                 data->handle = sev->handle;
 471                 data->len = len;
 472                 data->address = __sme_page_pa(inpages[i]) + offset;
 473                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
 474                 if (ret)
 475                         goto e_unpin;
 476
 477                 size -= len;
 478                 next_vaddr = vaddr + len;
 479         }
 480
 481 e_unpin:
 482         /* content of memory is updated, mark pages dirty */
 483         for (i = 0; i < npages; i++) {
 484                 set_page_dirty_lock(inpages[i]);
 485                 mark_page_accessed(inpages[i]);
 486         }
 487         /* unlock the user pages */
 488         sev_unpin_memory(kvm, inpages, npages);
 489 e_free:
 490         kfree(data);
 491         return ret;
 492 }
 493
 494 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
 495 {
 496         void __user *measure = (void __user *)(uintptr_t)argp->data;
 497         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 498         struct sev_data_launch_measure *data;
 499         struct kvm_sev_launch_measure params;
 500         void __user *p = NULL;
 501         void *blob = NULL;
 502         int ret;
 503
 504         if (!sev_guest(kvm))
 505                 return -ENOTTY;
 506
 507         if (copy_from_user(&params, measure, sizeof(params)))
 508                 return -EFAULT;
 509
 510         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 511         if (!data)
 512                 return -ENOMEM;
 513
 514         /* User wants to query the blob length */
 515         if (!params.len)
 516                 goto cmd;
 517
 518         p = (void __user *)(uintptr_t)params.uaddr;
 519         if (p) {
 520                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
 521                         ret = -EINVAL;
 522                         goto e_free;
 523                 }
 524
 525                 ret = -ENOMEM;
 526                 blob = kmalloc(params.len, GFP_KERNEL);
 527                 if (!blob)
 528                         goto e_free;
 529
 530                 data->address = __psp_pa(blob);
 531                 data->len = params.len;
 532         }
 533
 534 cmd:
 535         data->handle = sev->handle;
 536         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
 537
 538         /*
 539          * If we query the session length, FW responded with expected data.
 540          */
 541         if (!params.len)
 542                 goto done;
 543
 544         if (ret)
 545                 goto e_free_blob;
 546
 547         if (blob) {
 548                 if (copy_to_user(p, blob, params.len))
 549                         ret = -EFAULT;
 550         }
 551
 552 done:
 553         params.len = data->len;
 554         if (copy_to_user(measure, &params, sizeof(params)))
 555                 ret = -EFAULT;
 556 e_free_blob:
 557         kfree(blob);
 558 e_free:
 559         kfree(data);
 560         return ret;
 561 }
 562
 563 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
 564 {
 565         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 566         struct sev_data_launch_finish *data;
 567         int ret;
 568
 569         if (!sev_guest(kvm))
 570                 return -ENOTTY;
 571
 572         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 573         if (!data)
 574                 return -ENOMEM;
 575
 576         data->handle = sev->handle;
 577         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
 578
 579         kfree(data);
 580         return ret;
 581 }
 582
 583 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
 584 {
 585         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 586         struct kvm_sev_guest_status params;
 587         struct sev_data_guest_status *data;
 588         int ret;
 589
 590         if (!sev_guest(kvm))
 591                 return -ENOTTY;
 592
 593         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 594         if (!data)
 595                 return -ENOMEM;
 596
 597         data->handle = sev->handle;
 598         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
 599         if (ret)
 600                 goto e_free;
 601
 602         params.policy = data->policy;
 603         params.state = data->state;
 604         params.handle = data->handle;
 605
 606         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
 607                 ret = -EFAULT;
 608 e_free:
 609         kfree(data);
 610         return ret;
 611 }
 612
 613 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
 614                                unsigned long dst, int size,
 615                                int *error, bool enc)
 616 {
 617         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 618         struct sev_data_dbg *data;
 619         int ret;
 620
 621         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 622         if (!data)
 623                 return -ENOMEM;
 624
 625         data->handle = sev->handle;
 626         data->dst_addr = dst;
 627         data->src_addr = src;
 628         data->len = size;
 629
 630         ret = sev_issue_cmd(kvm,
 631                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
 632                             data, error);
 633         kfree(data);
 634         return ret;
 635 }
 636
 637 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
 638                              unsigned long dst_paddr, int sz, int *err)
 639 {
 640         int offset;
 641
 642         /*
 643          * Its safe to read more than we are asked, caller should ensure that
 644          * destination has enough space.
 645          */
 646         src_paddr = round_down(src_paddr, 16);
 647         offset = src_paddr & 15;
 648         sz = round_up(sz + offset, 16);
 649
 650         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
 651 }
 652
 653 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
 654                                   unsigned long __user dst_uaddr,
 655                                   unsigned long dst_paddr,
 656                                   int size, int *err)
 657 {
 658         struct page *tpage = NULL;
 659         int ret, offset;
 660
 661         /* if inputs are not 16-byte then use intermediate buffer */
 662         if (!IS_ALIGNED(dst_paddr, 16) ||
 663             !IS_ALIGNED(paddr,     16) ||
 664             !IS_ALIGNED(size,      16)) {
 665                 tpage = (void *)alloc_page(GFP_KERNEL);
 666                 if (!tpage)
 667                         return -ENOMEM;
 668
 669                 dst_paddr = __sme_page_pa(tpage);
 670         }
 671
 672         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
 673         if (ret)
 674                 goto e_free;
 675
 676         if (tpage) {
 677                 offset = paddr & 15;
 678                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
 679                                  page_address(tpage) + offset, size))
 680                         ret = -EFAULT;
 681         }
 682
 683 e_free:
 684         if (tpage)
 685                 __free_page(tpage);
 686
 687         return ret;
 688 }
 689
 690 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
 691                                   unsigned long __user vaddr,
 692                                   unsigned long dst_paddr,
 693                                   unsigned long __user dst_vaddr,
 694                                   int size, int *error)
 695 {
 696         struct page *src_tpage = NULL;
 697         struct page *dst_tpage = NULL;
 698         int ret, len = size;
 699
 700         /* If source buffer is not aligned then use an intermediate buffer */
 701         if (!IS_ALIGNED(vaddr, 16)) {
 702                 src_tpage = alloc_page(GFP_KERNEL);
 703                 if (!src_tpage)
 704                         return -ENOMEM;
 705
 706                 if (copy_from_user(page_address(src_tpage),
 707                                 (void __user *)(uintptr_t)vaddr, size)) {
 708                         __free_page(src_tpage);
 709                         return -EFAULT;
 710                 }
 711
 712                 paddr = __sme_page_pa(src_tpage);
 713         }
 714
 715         /*
 716          *  If destination buffer or length is not aligned then do read-modify-write:
 717          *   - decrypt destination in an intermediate buffer
 718          *   - copy the source buffer in an intermediate buffer
 719          *   - use the intermediate buffer as source buffer
 720          */
 721         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
 722                 int dst_offset;
 723
 724                 dst_tpage = alloc_page(GFP_KERNEL);
 725                 if (!dst_tpage) {
 726                         ret = -ENOMEM;
 727                         goto e_free;
 728                 }
 729
 730                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
 731                                         __sme_page_pa(dst_tpage), size, error);
 732                 if (ret)
 733                         goto e_free;
 734
 735                 /*
 736                  *  If source is kernel buffer then use memcpy() otherwise
 737                  *  copy_from_user().
 738                  */
 739                 dst_offset = dst_paddr & 15;
 740
 741                 if (src_tpage)
 742                         memcpy(page_address(dst_tpage) + dst_offset,
 743                                page_address(src_tpage), size);
 744                 else {
 745                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
 746                                            (void __user *)(uintptr_t)vaddr, size)) {
 747                                 ret = -EFAULT;
 748                                 goto e_free;
 749                         }
 750                 }
 751
 752                 paddr = __sme_page_pa(dst_tpage);
 753                 dst_paddr = round_down(dst_paddr, 16);
 754                 len = round_up(size, 16);
 755         }
 756
 757         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
 758
 759 e_free:
 760         if (src_tpage)
 761                 __free_page(src_tpage);
 762         if (dst_tpage)
 763                 __free_page(dst_tpage);
 764         return ret;
 765 }
 766
 767 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
 768 {
 769         unsigned long vaddr, vaddr_end, next_vaddr;
 770         unsigned long dst_vaddr;
 771         struct page **src_p, **dst_p;
 772         struct kvm_sev_dbg debug;
 773         unsigned long n;
 774         unsigned int size;
 775         int ret;
 776
 777         if (!sev_guest(kvm))
 778                 return -ENOTTY;
 779
 780         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
 781                 return -EFAULT;
 782
 783         if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
 784                 return -EINVAL;
 785         if (!debug.dst_uaddr)
 786                 return -EINVAL;
 787
 788         vaddr = debug.src_uaddr;
 789         size = debug.len;
 790         vaddr_end = vaddr + size;
 791         dst_vaddr = debug.dst_uaddr;
 792
 793         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
 794                 int len, s_off, d_off;
 795
 796                 /* lock userspace source and destination page */
 797                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
 798                 if (IS_ERR(src_p))
 799                         return PTR_ERR(src_p);
 800
 801                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
 802                 if (IS_ERR(dst_p)) {
 803                         sev_unpin_memory(kvm, src_p, n);
 804                         return PTR_ERR(dst_p);
 805                 }
 806
 807                 /*
 808                  * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify
 809                  * the pages; flush the destination too so that future accesses do not
 810                  * see stale data.
 811                  */
 812                 sev_clflush_pages(src_p, 1);
 813                 sev_clflush_pages(dst_p, 1);
 814
 815                 /*
 816                  * Since user buffer may not be page aligned, calculate the
 817                  * offset within the page.
 818                  */
 819                 s_off = vaddr & ~PAGE_MASK;
 820                 d_off = dst_vaddr & ~PAGE_MASK;
 821                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
 822
 823                 if (dec)
 824                         ret = __sev_dbg_decrypt_user(kvm,
 825                                                      __sme_page_pa(src_p[0]) + s_off,
 826                                                      dst_vaddr,
 827                                                      __sme_page_pa(dst_p[0]) + d_off,
 828                                                      len, &argp->error);
 829                 else
 830                         ret = __sev_dbg_encrypt_user(kvm,
 831                                                      __sme_page_pa(src_p[0]) + s_off,
 832                                                      vaddr,
 833                                                      __sme_page_pa(dst_p[0]) + d_off,
 834                                                      dst_vaddr,
 835                                                      len, &argp->error);
 836
 837                 sev_unpin_memory(kvm, src_p, n);
 838                 sev_unpin_memory(kvm, dst_p, n);
 839
 840                 if (ret)
 841                         goto err;
 842
 843                 next_vaddr = vaddr + len;
 844                 dst_vaddr = dst_vaddr + len;
 845                 size -= len;
 846         }
 847 err:
 848         return ret;
 849 }
 850
 851 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
 852 {
 853         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 854         struct sev_data_launch_secret *data;
 855         struct kvm_sev_launch_secret params;
 856         struct page **pages;
 857         void *blob, *hdr;
 858         unsigned long n, i;
 859         int ret, offset;
 860
 861         if (!sev_guest(kvm))
 862                 return -ENOTTY;
 863
 864         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
 865                 return -EFAULT;
 866
 867         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
 868         if (IS_ERR(pages))
 869                 return PTR_ERR(pages);
 870
 871         /*
 872          * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in
 873          * place; the cache may contain the data that was written unencrypted.
 874          */
 875         sev_clflush_pages(pages, n);
 876
 877         /*
 878          * The secret must be copied into contiguous memory region, lets verify
 879          * that userspace memory pages are contiguous before we issue command.
 880          */
 881         if (get_num_contig_pages(0, pages, n) != n) {
 882                 ret = -EINVAL;
 883                 goto e_unpin_memory;
 884         }
 885
 886         ret = -ENOMEM;
 887         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
 888         if (!data)
 889                 goto e_unpin_memory;
 890
 891         offset = params.guest_uaddr & (PAGE_SIZE - 1);
 892         data->guest_address = __sme_page_pa(pages[0]) + offset;
 893         data->guest_len = params.guest_len;
 894
 895         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
 896         if (IS_ERR(blob)) {
 897                 ret = PTR_ERR(blob);
 898                 goto e_free;
 899         }
 900
 901         data->trans_address = __psp_pa(blob);
 902         data->trans_len = params.trans_len;
 903
 904         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
 905         if (IS_ERR(hdr)) {
 906                 ret = PTR_ERR(hdr);
 907                 goto e_free_blob;
 908         }
 909         data->hdr_address = __psp_pa(hdr);
 910         data->hdr_len = params.hdr_len;
 911
 912         data->handle = sev->handle;
 913         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
 914
 915         kfree(hdr);
 916
 917 e_free_blob:
 918         kfree(blob);
 919 e_free:
 920         kfree(data);
 921 e_unpin_memory:
 922         /* content of memory is updated, mark pages dirty */
 923         for (i = 0; i < n; i++) {
 924                 set_page_dirty_lock(pages[i]);
 925                 mark_page_accessed(pages[i]);
 926         }
 927         sev_unpin_memory(kvm, pages, n);
 928         return ret;
 929 }
 930
 931 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
 932 {
 933         struct kvm_sev_cmd sev_cmd;
 934         int r;
 935
 936         if (!svm_sev_enabled() || !sev)
 937                 return -ENOTTY;
 938
 939         if (!argp)
 940                 return 0;
 941
 942         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
 943                 return -EFAULT;
 944
 945         mutex_lock(&kvm->lock);
 946
 947         switch (sev_cmd.id) {
 948         case KVM_SEV_INIT:
 949                 r = sev_guest_init(kvm, &sev_cmd);
 950                 break;
 951         case KVM_SEV_LAUNCH_START:
 952                 r = sev_launch_start(kvm, &sev_cmd);
 953                 break;
 954         case KVM_SEV_LAUNCH_UPDATE_DATA:
 955                 r = sev_launch_update_data(kvm, &sev_cmd);
 956                 break;
 957         case KVM_SEV_LAUNCH_MEASURE:
 958                 r = sev_launch_measure(kvm, &sev_cmd);
 959                 break;
 960         case KVM_SEV_LAUNCH_FINISH:
 961                 r = sev_launch_finish(kvm, &sev_cmd);
 962                 break;
 963         case KVM_SEV_GUEST_STATUS:
 964                 r = sev_guest_status(kvm, &sev_cmd);
 965                 break;
 966         case KVM_SEV_DBG_DECRYPT:
 967                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
 968                 break;
 969         case KVM_SEV_DBG_ENCRYPT:
 970                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
 971                 break;
 972         case KVM_SEV_LAUNCH_SECRET:
 973                 r = sev_launch_secret(kvm, &sev_cmd);
 974                 break;
 975         default:
 976                 r = -EINVAL;
 977                 goto out;
 978         }
 979
 980         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
 981                 r = -EFAULT;
 982
 983 out:
 984         mutex_unlock(&kvm->lock);
 985         return r;
 986 }
 987
 988 int svm_register_enc_region(struct kvm *kvm,
 989                             struct kvm_enc_region *range)
 990 {
 991         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 992         struct enc_region *region;
 993         int ret = 0;
 994
 995         if (!sev_guest(kvm))
 996                 return -ENOTTY;
 997
 998         if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
 999                 return -EINVAL;
1000
1001         region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
1002         if (!region)
1003                 return -ENOMEM;
1004
1005         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
1006         if (IS_ERR(region->pages)) {
1007                 ret = PTR_ERR(region->pages);
1008                 goto e_free;
1009         }
1010
1011         /*
1012          * The guest may change the memory encryption attribute from C=0 -> C=1
1013          * or vice versa for this memory range. Lets make sure caches are
1014          * flushed to ensure that guest data gets written into memory with
1015          * correct C-bit.
1016          */
1017         sev_clflush_pages(region->pages, region->npages);
1018
1019         region->uaddr = range->addr;
1020         region->size = range->size;
1021
1022         mutex_lock(&kvm->lock);
1023         list_add_tail(&region->list, &sev->regions_list);
1024         mutex_unlock(&kvm->lock);
1025
1026         return ret;
1027
1028 e_free:
1029         kfree(region);
1030         return ret;
1031 }
1032
1033 static struct enc_region *
1034 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
1035 {
1036         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1037         struct list_head *head = &sev->regions_list;
1038         struct enc_region *i;
1039
1040         list_for_each_entry(i, head, list) {
1041                 if (i->uaddr == range->addr &&
1042                     i->size == range->size)
1043                         return i;
1044         }
1045
1046         return NULL;
1047 }
1048
1049 static void __unregister_enc_region_locked(struct kvm *kvm,
1050                                            struct enc_region *region)
1051 {
1052         sev_unpin_memory(kvm, region->pages, region->npages);
1053         list_del(&region->list);
1054         kfree(region);
1055 }
1056
1057 int svm_unregister_enc_region(struct kvm *kvm,
1058                               struct kvm_enc_region *range)
1059 {
1060         struct enc_region *region;
1061         int ret;
1062
1063         mutex_lock(&kvm->lock);
1064
1065         if (!sev_guest(kvm)) {
1066                 ret = -ENOTTY;
1067                 goto failed;
1068         }
1069
1070         region = find_enc_region(kvm, range);
1071         if (!region) {
1072                 ret = -EINVAL;
1073                 goto failed;
1074         }
1075
1076         /*
1077          * Ensure that all guest tagged cache entries are flushed before
1078          * releasing the pages back to the system for use. CLFLUSH will
1079          * not do this, so issue a WBINVD.
1080          */
1081         wbinvd_on_all_cpus();
1082
1083         __unregister_enc_region_locked(kvm, region);
1084
1085         mutex_unlock(&kvm->lock);
1086         return 0;
1087
1088 failed:
1089         mutex_unlock(&kvm->lock);
1090         return ret;
1091 }
1092
1093 void sev_vm_destroy(struct kvm *kvm)
1094 {
1095         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1096         struct list_head *head = &sev->regions_list;
1097         struct list_head *pos, *q;
1098
1099         if (!sev_guest(kvm))
1100                 return;
1101
1102         mutex_lock(&kvm->lock);
1103
1104         /*
1105          * Ensure that all guest tagged cache entries are flushed before
1106          * releasing the pages back to the system for use. CLFLUSH will
1107          * not do this, so issue a WBINVD.
1108          */
1109         wbinvd_on_all_cpus();
1110
1111         /*
1112          * if userspace was terminated before unregistering the memory regions
1113          * then lets unpin all the registered memory.
1114          */
1115         if (!list_empty(head)) {
1116                 list_for_each_safe(pos, q, head) {
1117                         __unregister_enc_region_locked(kvm,
1118                                 list_entry(pos, struct enc_region, list));
1119                         cond_resched();
1120                 }
1121         }
1122
1123         mutex_unlock(&kvm->lock);
1124
1125         sev_unbind_asid(kvm, sev->handle);
1126         sev_asid_free(sev->asid);
1127 }
1128
1129 void __init sev_hardware_setup(void)
1130 {
1131         unsigned int eax, ebx, ecx, edx;
1132         bool sev_es_supported = false;
1133         bool sev_supported = false;
1134
1135         /* Does the CPU support SEV? */
1136         if (!boot_cpu_has(X86_FEATURE_SEV))
1137                 goto out;
1138
1139         /* Retrieve SEV CPUID information */
1140         cpuid(0x8000001f, &eax, &ebx, &ecx, &edx);
1141
1142         /* Maximum number of encrypted guests supported simultaneously */
1143         max_sev_asid = ecx;
1144
1145         if (!svm_sev_enabled())
1146                 goto out;
1147
1148         /* Minimum ASID value that should be used for SEV guest */
1149         min_sev_asid = edx;
1150
1151         /* Initialize SEV ASID bitmaps */
1152         sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1153         if (!sev_asid_bitmap)
1154                 goto out;
1155
1156         sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1157         if (!sev_reclaim_asid_bitmap)
1158                 goto out;
1159
1160         pr_info("SEV supported: %u ASIDs\n", max_sev_asid - min_sev_asid + 1);
1161         sev_supported = true;
1162
1163         /* SEV-ES support requested? */
1164         if (!sev_es)
1165                 goto out;
1166
1167         /* Does the CPU support SEV-ES? */
1168         if (!boot_cpu_has(X86_FEATURE_SEV_ES))
1169                 goto out;
1170
1171         /* Has the system been allocated ASIDs for SEV-ES? */
1172         if (min_sev_asid == 1)
1173                 goto out;
1174
1175         pr_info("SEV-ES supported: %u ASIDs\n", min_sev_asid - 1);
1176         sev_es_supported = true;
1177
1178 out:
1179         sev = sev_supported;
1180         sev_es = sev_es_supported;
1181 }
1182
1183 void sev_hardware_teardown(void)
1184 {
1185         if (!svm_sev_enabled())
1186                 return;
1187
1188         bitmap_free(sev_asid_bitmap);
1189         bitmap_free(sev_reclaim_asid_bitmap);
1190
1191         sev_flush_asids();
1192 }
1193
1194 /*
1195  * Pages used by hardware to hold guest encrypted state must be flushed before
1196  * returning them to the system.
1197  */
1198 static void sev_flush_guest_memory(struct vcpu_svm *svm, void *va,
1199                                    unsigned long len)
1200 {
1201         /*
1202          * If hardware enforced cache coherency for encrypted mappings of the
1203          * same physical page is supported, nothing to do.
1204          */
1205         if (boot_cpu_has(X86_FEATURE_SME_COHERENT))
1206                 return;
1207
1208         /*
1209          * If the VM Page Flush MSR is supported, use it to flush the page
1210          * (using the page virtual address and the guest ASID).
1211          */
1212         if (boot_cpu_has(X86_FEATURE_VM_PAGE_FLUSH)) {
1213                 struct kvm_sev_info *sev;
1214                 unsigned long va_start;
1215                 u64 start, stop;
1216
1217                 /* Align start and stop to page boundaries. */
1218                 va_start = (unsigned long)va;
1219                 start = (u64)va_start & PAGE_MASK;
1220                 stop = PAGE_ALIGN((u64)va_start + len);
1221
1222                 if (start < stop) {
1223                         sev = &to_kvm_svm(svm->vcpu.kvm)->sev_info;
1224
1225                         while (start < stop) {
1226                                 wrmsrl(MSR_AMD64_VM_PAGE_FLUSH,
1227                                        start | sev->asid);
1228
1229                                 start += PAGE_SIZE;
1230                         }
1231
1232                         return;
1233                 }
1234
1235                 WARN(1, "Address overflow, using WBINVD\n");
1236         }
1237
1238         /*
1239          * Hardware should always have one of the above features,
1240          * but if not, use WBINVD and issue a warning.
1241          */
1242         WARN_ONCE(1, "Using WBINVD to flush guest memory\n");
1243         wbinvd_on_all_cpus();
1244 }
1245
1246 void sev_free_vcpu(struct kvm_vcpu *vcpu)
1247 {
1248         struct vcpu_svm *svm;
1249
1250         if (!sev_es_guest(vcpu->kvm))
1251                 return;
1252
1253         svm = to_svm(vcpu);
1254
1255         if (vcpu->arch.guest_state_protected)
1256                 sev_flush_guest_memory(svm, svm->vmsa, PAGE_SIZE);
1257         __free_page(virt_to_page(svm->vmsa));
1258 }
1259
1260 void pre_sev_run(struct vcpu_svm *svm, int cpu)
1261 {
1262         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1263         int asid = sev_get_asid(svm->vcpu.kvm);
1264
1265         /* Assign the asid allocated with this SEV guest */
1266         svm->asid = asid;
1267
1268         /*
1269          * Flush guest TLB:
1270          *
1271          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
1272          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
1273          */
1274         if (sd->sev_vmcbs[asid] == svm->vmcb &&
1275             svm->vcpu.arch.last_vmentry_cpu == cpu)
1276                 return;
1277
1278         sd->sev_vmcbs[asid] = svm->vmcb;
1279         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
1280         vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1281 }