arch/powerpc/platforms/pseries/cmm.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Collaborative memory management interface.
   4  *
   5  * Copyright (C) 2008 IBM Corporation
   6  * Author(s): Brian King (brking@linux.vnet.ibm.com),
   7  */
   8
   9 #include <linux/ctype.h>
  10 #include <linux/delay.h>
  11 #include <linux/errno.h>
  12 #include <linux/fs.h>
  13 #include <linux/gfp.h>
  14 #include <linux/kthread.h>
  15 #include <linux/module.h>
  16 #include <linux/oom.h>
  17 #include <linux/reboot.h>
  18 #include <linux/sched.h>
  19 #include <linux/stringify.h>
  20 #include <linux/swap.h>
  21 #include <linux/device.h>
  22 #include <linux/mount.h>
  23 #include <linux/pseudo_fs.h>
  24 #include <linux/magic.h>
  25 #include <linux/balloon_compaction.h>
  26 #include <asm/firmware.h>
  27 #include <asm/hvcall.h>
  28 #include <asm/mmu.h>
  29 #include <linux/uaccess.h>
  30 #include <linux/memory.h>
  31 #include <asm/plpar_wrappers.h>
  32
  33 #include "pseries.h"
  34
  35 #define CMM_DRIVER_VERSION      "1.0.0"
  36 #define CMM_DEFAULT_DELAY       1
  37 #define CMM_HOTPLUG_DELAY       5
  38 #define CMM_DEBUG                       0
  39 #define CMM_DISABLE             0
  40 #define CMM_OOM_KB              1024
  41 #define CMM_MIN_MEM_MB          256
  42 #define KB2PAGES(_p)            ((_p)>>(PAGE_SHIFT-10))
  43 #define PAGES2KB(_p)            ((_p)<<(PAGE_SHIFT-10))
  44
  45 #define CMM_MEM_HOTPLUG_PRI     1
  46
  47 static unsigned int delay = CMM_DEFAULT_DELAY;
  48 static unsigned int hotplug_delay = CMM_HOTPLUG_DELAY;
  49 static unsigned int oom_kb = CMM_OOM_KB;
  50 static unsigned int cmm_debug = CMM_DEBUG;
  51 static unsigned int cmm_disabled = CMM_DISABLE;
  52 static unsigned long min_mem_mb = CMM_MIN_MEM_MB;
  53 static bool __read_mostly simulate;
  54 static unsigned long simulate_loan_target_kb;
  55 static struct device cmm_dev;
  56
  57 MODULE_AUTHOR("Brian King <brking@linux.vnet.ibm.com>");
  58 MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager");
  59 MODULE_LICENSE("GPL");
  60 MODULE_VERSION(CMM_DRIVER_VERSION);
  61
  62 module_param_named(delay, delay, uint, 0644);
  63 MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. "
  64                  "[Default=" __stringify(CMM_DEFAULT_DELAY) "]");
  65 module_param_named(hotplug_delay, hotplug_delay, uint, 0644);
  66 MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove "
  67                  "before loaning resumes. "
  68                  "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]");
  69 module_param_named(oom_kb, oom_kb, uint, 0644);
  70 MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. "
  71                  "[Default=" __stringify(CMM_OOM_KB) "]");
  72 module_param_named(min_mem_mb, min_mem_mb, ulong, 0644);
  73 MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. "
  74                  "[Default=" __stringify(CMM_MIN_MEM_MB) "]");
  75 module_param_named(debug, cmm_debug, uint, 0644);
  76 MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. "
  77                  "[Default=" __stringify(CMM_DEBUG) "]");
  78 module_param_named(simulate, simulate, bool, 0444);
  79 MODULE_PARM_DESC(simulate, "Enable simulation mode (no communication with hw).");
  80
  81 #define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); }
  82
  83 static atomic_long_t loaned_pages;
  84 static unsigned long loaned_pages_target;
  85 static unsigned long oom_freed_pages;
  86
  87 static DEFINE_MUTEX(hotplug_mutex);
  88 static int hotplug_occurred; /* protected by the hotplug mutex */
  89
  90 static struct task_struct *cmm_thread_ptr;
  91 static struct balloon_dev_info b_dev_info;
  92
  93 static long plpar_page_set_loaned(struct page *page)
  94 {
  95         const unsigned long vpa = page_to_phys(page);
  96         unsigned long cmo_page_sz = cmo_get_page_size();
  97         long rc = 0;
  98         int i;
  99
 100         if (unlikely(simulate))
 101                 return 0;
 102
 103         for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
 104                 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0);
 105
 106         for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
 107                 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE,
 108                                    vpa + i - cmo_page_sz, 0);
 109
 110         return rc;
 111 }
 112
 113 static long plpar_page_set_active(struct page *page)
 114 {
 115         const unsigned long vpa = page_to_phys(page);
 116         unsigned long cmo_page_sz = cmo_get_page_size();
 117         long rc = 0;
 118         int i;
 119
 120         if (unlikely(simulate))
 121                 return 0;
 122
 123         for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
 124                 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0);
 125
 126         for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
 127                 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED,
 128                                    vpa + i - cmo_page_sz, 0);
 129
 130         return rc;
 131 }
 132
 133 /**
 134  * cmm_alloc_pages - Allocate pages and mark them as loaned
 135  * @nr: number of pages to allocate
 136  *
 137  * Return value:
 138  *      number of pages requested to be allocated which were not
 139  **/
 140 static long cmm_alloc_pages(long nr)
 141 {
 142         struct page *page;
 143         long rc;
 144
 145         cmm_dbg("Begin request for %ld pages\n", nr);
 146
 147         while (nr) {
 148                 /* Exit if a hotplug operation is in progress or occurred */
 149                 if (mutex_trylock(&hotplug_mutex)) {
 150                         if (hotplug_occurred) {
 151                                 mutex_unlock(&hotplug_mutex);
 152                                 break;
 153                         }
 154                         mutex_unlock(&hotplug_mutex);
 155                 } else {
 156                         break;
 157                 }
 158
 159                 page = balloon_page_alloc();
 160                 if (!page)
 161                         break;
 162                 rc = plpar_page_set_loaned(page);
 163                 if (rc) {
 164                         pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc);
 165                         __free_page(page);
 166                         break;
 167                 }
 168
 169                 balloon_page_enqueue(&b_dev_info, page);
 170                 atomic_long_inc(&loaned_pages);
 171                 adjust_managed_page_count(page, -1);
 172                 nr--;
 173         }
 174
 175         cmm_dbg("End request with %ld pages unfulfilled\n", nr);
 176         return nr;
 177 }
 178
 179 /**
 180  * cmm_free_pages - Free pages and mark them as active
 181  * @nr: number of pages to free
 182  *
 183  * Return value:
 184  *      number of pages requested to be freed which were not
 185  **/
 186 static long cmm_free_pages(long nr)
 187 {
 188         struct page *page;
 189
 190         cmm_dbg("Begin free of %ld pages.\n", nr);
 191         while (nr) {
 192                 page = balloon_page_dequeue(&b_dev_info);
 193                 if (!page)
 194                         break;
 195                 plpar_page_set_active(page);
 196                 adjust_managed_page_count(page, 1);
 197                 __free_page(page);
 198                 atomic_long_dec(&loaned_pages);
 199                 nr--;
 200         }
 201         cmm_dbg("End request with %ld pages unfulfilled\n", nr);
 202         return nr;
 203 }
 204
 205 /**
 206  * cmm_oom_notify - OOM notifier
 207  * @self:       notifier block struct
 208  * @dummy:      not used
 209  * @parm:       returned - number of pages freed
 210  *
 211  * Return value:
 212  *      NOTIFY_OK
 213  **/
 214 static int cmm_oom_notify(struct notifier_block *self,
 215                           unsigned long dummy, void *parm)
 216 {
 217         unsigned long *freed = parm;
 218         long nr = KB2PAGES(oom_kb);
 219
 220         cmm_dbg("OOM processing started\n");
 221         nr = cmm_free_pages(nr);
 222         loaned_pages_target = atomic_long_read(&loaned_pages);
 223         *freed += KB2PAGES(oom_kb) - nr;
 224         oom_freed_pages += KB2PAGES(oom_kb) - nr;
 225         cmm_dbg("OOM processing complete\n");
 226         return NOTIFY_OK;
 227 }
 228
 229 /**
 230  * cmm_get_mpp - Read memory performance parameters
 231  *
 232  * Makes hcall to query the current page loan request from the hypervisor.
 233  *
 234  * Return value:
 235  *      nothing
 236  **/
 237 static void cmm_get_mpp(void)
 238 {
 239         const long __loaned_pages = atomic_long_read(&loaned_pages);
 240         const long total_pages = totalram_pages() + __loaned_pages;
 241         int rc;
 242         struct hvcall_mpp_data mpp_data;
 243         signed long active_pages_target, page_loan_request, target;
 244         signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE;
 245
 246         if (likely(!simulate)) {
 247                 rc = h_get_mpp(&mpp_data);
 248                 if (rc != H_SUCCESS)
 249                         return;
 250                 page_loan_request = div_s64((s64)mpp_data.loan_request,
 251                                             PAGE_SIZE);
 252                 target = page_loan_request + __loaned_pages;
 253         } else {
 254                 target = KB2PAGES(simulate_loan_target_kb);
 255                 page_loan_request = target - __loaned_pages;
 256         }
 257
 258         if (target < 0 || total_pages < min_mem_pages)
 259                 target = 0;
 260
 261         if (target > oom_freed_pages)
 262                 target -= oom_freed_pages;
 263         else
 264                 target = 0;
 265
 266         active_pages_target = total_pages - target;
 267
 268         if (min_mem_pages > active_pages_target)
 269                 target = total_pages - min_mem_pages;
 270
 271         if (target < 0)
 272                 target = 0;
 273
 274         loaned_pages_target = target;
 275
 276         cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n",
 277                 page_loan_request, __loaned_pages, loaned_pages_target,
 278                 oom_freed_pages, totalram_pages());
 279 }
 280
 281 static struct notifier_block cmm_oom_nb = {
 282         .notifier_call = cmm_oom_notify
 283 };
 284
 285 /**
 286  * cmm_thread - CMM task thread
 287  * @dummy:      not used
 288  *
 289  * Return value:
 290  *      0
 291  **/
 292 static int cmm_thread(void *dummy)
 293 {
 294         unsigned long timeleft;
 295         long __loaned_pages;
 296
 297         while (1) {
 298                 timeleft = msleep_interruptible(delay * 1000);
 299
 300                 if (kthread_should_stop() || timeleft)
 301                         break;
 302
 303                 if (mutex_trylock(&hotplug_mutex)) {
 304                         if (hotplug_occurred) {
 305                                 hotplug_occurred = 0;
 306                                 mutex_unlock(&hotplug_mutex);
 307                                 cmm_dbg("Hotplug operation has occurred, "
 308                                                 "loaning activity suspended "
 309                                                 "for %d seconds.\n",
 310                                                 hotplug_delay);
 311                                 timeleft = msleep_interruptible(hotplug_delay *
 312                                                 1000);
 313                                 if (kthread_should_stop() || timeleft)
 314                                         break;
 315                                 continue;
 316                         }
 317                         mutex_unlock(&hotplug_mutex);
 318                 } else {
 319                         cmm_dbg("Hotplug operation in progress, activity "
 320                                         "suspended\n");
 321                         continue;
 322                 }
 323
 324                 cmm_get_mpp();
 325
 326                 __loaned_pages = atomic_long_read(&loaned_pages);
 327                 if (loaned_pages_target > __loaned_pages) {
 328                         if (cmm_alloc_pages(loaned_pages_target - __loaned_pages))
 329                                 loaned_pages_target = __loaned_pages;
 330                 } else if (loaned_pages_target < __loaned_pages)
 331                         cmm_free_pages(__loaned_pages - loaned_pages_target);
 332         }
 333         return 0;
 334 }
 335
 336 #define CMM_SHOW(name, format, args...)                 \
 337         static ssize_t show_##name(struct device *dev,  \
 338                                    struct device_attribute *attr,       \
 339                                    char *buf)                   \
 340         {                                                       \
 341                 return sprintf(buf, format, ##args);            \
 342         }                                                       \
 343         static DEVICE_ATTR(name, 0444, show_##name, NULL)
 344
 345 CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(atomic_long_read(&loaned_pages)));
 346 CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target));
 347
 348 static ssize_t show_oom_pages(struct device *dev,
 349                               struct device_attribute *attr, char *buf)
 350 {
 351         return sprintf(buf, "%lu\n", PAGES2KB(oom_freed_pages));
 352 }
 353
 354 static ssize_t store_oom_pages(struct device *dev,
 355                                struct device_attribute *attr,
 356                                const char *buf, size_t count)
 357 {
 358         unsigned long val = simple_strtoul (buf, NULL, 10);
 359
 360         if (!capable(CAP_SYS_ADMIN))
 361                 return -EPERM;
 362         if (val != 0)
 363                 return -EBADMSG;
 364
 365         oom_freed_pages = 0;
 366         return count;
 367 }
 368
 369 static DEVICE_ATTR(oom_freed_kb, 0644,
 370                    show_oom_pages, store_oom_pages);
 371
 372 static struct device_attribute *cmm_attrs[] = {
 373         &dev_attr_loaned_kb,
 374         &dev_attr_loaned_target_kb,
 375         &dev_attr_oom_freed_kb,
 376 };
 377
 378 static DEVICE_ULONG_ATTR(simulate_loan_target_kb, 0644,
 379                          simulate_loan_target_kb);
 380
 381 static struct bus_type cmm_subsys = {
 382         .name = "cmm",
 383         .dev_name = "cmm",
 384 };
 385
 386 static void cmm_release_device(struct device *dev)
 387 {
 388 }
 389
 390 /**
 391  * cmm_sysfs_register - Register with sysfs
 392  *
 393  * Return value:
 394  *      0 on success / other on failure
 395  **/
 396 static int cmm_sysfs_register(struct device *dev)
 397 {
 398         int i, rc;
 399
 400         if ((rc = subsys_system_register(&cmm_subsys, NULL)))
 401                 return rc;
 402
 403         dev->id = 0;
 404         dev->bus = &cmm_subsys;
 405         dev->release = cmm_release_device;
 406
 407         if ((rc = device_register(dev)))
 408                 goto subsys_unregister;
 409
 410         for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) {
 411                 if ((rc = device_create_file(dev, cmm_attrs[i])))
 412                         goto fail;
 413         }
 414
 415         if (!simulate)
 416                 return 0;
 417         rc = device_create_file(dev, &dev_attr_simulate_loan_target_kb.attr);
 418         if (rc)
 419                 goto fail;
 420         return 0;
 421
 422 fail:
 423         while (--i >= 0)
 424                 device_remove_file(dev, cmm_attrs[i]);
 425         device_unregister(dev);
 426 subsys_unregister:
 427         bus_unregister(&cmm_subsys);
 428         return rc;
 429 }
 430
 431 /**
 432  * cmm_unregister_sysfs - Unregister from sysfs
 433  *
 434  **/
 435 static void cmm_unregister_sysfs(struct device *dev)
 436 {
 437         int i;
 438
 439         for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++)
 440                 device_remove_file(dev, cmm_attrs[i]);
 441         device_unregister(dev);
 442         bus_unregister(&cmm_subsys);
 443 }
 444
 445 /**
 446  * cmm_reboot_notifier - Make sure pages are not still marked as "loaned"
 447  *
 448  **/
 449 static int cmm_reboot_notifier(struct notifier_block *nb,
 450                                unsigned long action, void *unused)
 451 {
 452         if (action == SYS_RESTART) {
 453                 if (cmm_thread_ptr)
 454                         kthread_stop(cmm_thread_ptr);
 455                 cmm_thread_ptr = NULL;
 456                 cmm_free_pages(atomic_long_read(&loaned_pages));
 457         }
 458         return NOTIFY_DONE;
 459 }
 460
 461 static struct notifier_block cmm_reboot_nb = {
 462         .notifier_call = cmm_reboot_notifier,
 463 };
 464
 465 /**
 466  * cmm_memory_cb - Handle memory hotplug notifier calls
 467  * @self:       notifier block struct
 468  * @action:     action to take
 469  * @arg:        struct memory_notify data for handler
 470  *
 471  * Return value:
 472  *      NOTIFY_OK or notifier error based on subfunction return value
 473  *
 474  **/
 475 static int cmm_memory_cb(struct notifier_block *self,
 476                         unsigned long action, void *arg)
 477 {
 478         int ret = 0;
 479
 480         switch (action) {
 481         case MEM_GOING_OFFLINE:
 482                 mutex_lock(&hotplug_mutex);
 483                 hotplug_occurred = 1;
 484                 break;
 485         case MEM_OFFLINE:
 486         case MEM_CANCEL_OFFLINE:
 487                 mutex_unlock(&hotplug_mutex);
 488                 cmm_dbg("Memory offline operation complete.\n");
 489                 break;
 490         case MEM_GOING_ONLINE:
 491         case MEM_ONLINE:
 492         case MEM_CANCEL_ONLINE:
 493                 break;
 494         }
 495
 496         return notifier_from_errno(ret);
 497 }
 498
 499 static struct notifier_block cmm_mem_nb = {
 500         .notifier_call = cmm_memory_cb,
 501         .priority = CMM_MEM_HOTPLUG_PRI
 502 };
 503
 504 #ifdef CONFIG_BALLOON_COMPACTION
 505 static struct vfsmount *balloon_mnt;
 506
 507 static int cmm_init_fs_context(struct fs_context *fc)
 508 {
 509         return init_pseudo(fc, PPC_CMM_MAGIC) ? 0 : -ENOMEM;
 510 }
 511
 512 static struct file_system_type balloon_fs = {
 513         .name = "ppc-cmm",
 514         .init_fs_context = cmm_init_fs_context,
 515         .kill_sb = kill_anon_super,
 516 };
 517
 518 static int cmm_migratepage(struct balloon_dev_info *b_dev_info,
 519                            struct page *newpage, struct page *page,
 520                            enum migrate_mode mode)
 521 {
 522         unsigned long flags;
 523
 524         /*
 525          * loan/"inflate" the newpage first.
 526          *
 527          * We might race against the cmm_thread who might discover after our
 528          * loan request that another page is to be unloaned. However, once
 529          * the cmm_thread runs again later, this error will automatically
 530          * be corrected.
 531          */
 532         if (plpar_page_set_loaned(newpage)) {
 533                 /* Unlikely, but possible. Tell the caller not to retry now. */
 534                 pr_err_ratelimited("%s: Cannot set page to loaned.", __func__);
 535                 return -EBUSY;
 536         }
 537
 538         /* balloon page list reference */
 539         get_page(newpage);
 540
 541         /*
 542          * When we migrate a page to a different zone, we have to fixup the
 543          * count of both involved zones as we adjusted the managed page count
 544          * when inflating.
 545          */
 546         if (page_zone(page) != page_zone(newpage)) {
 547                 adjust_managed_page_count(page, 1);
 548                 adjust_managed_page_count(newpage, -1);
 549         }
 550
 551         spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 552         balloon_page_insert(b_dev_info, newpage);
 553         balloon_page_delete(page);
 554         b_dev_info->isolated_pages--;
 555         spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 556
 557         /*
 558          * activate/"deflate" the old page. We ignore any errors just like the
 559          * other callers.
 560          */
 561         plpar_page_set_active(page);
 562
 563         /* balloon page list reference */
 564         put_page(page);
 565
 566         return MIGRATEPAGE_SUCCESS;
 567 }
 568
 569 static int cmm_balloon_compaction_init(void)
 570 {
 571         int rc;
 572
 573         balloon_devinfo_init(&b_dev_info);
 574         b_dev_info.migratepage = cmm_migratepage;
 575
 576         balloon_mnt = kern_mount(&balloon_fs);
 577         if (IS_ERR(balloon_mnt)) {
 578                 rc = PTR_ERR(balloon_mnt);
 579                 balloon_mnt = NULL;
 580                 return rc;
 581         }
 582
 583         b_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
 584         if (IS_ERR(b_dev_info.inode)) {
 585                 rc = PTR_ERR(b_dev_info.inode);
 586                 b_dev_info.inode = NULL;
 587                 kern_unmount(balloon_mnt);
 588                 balloon_mnt = NULL;
 589                 return rc;
 590         }
 591
 592         b_dev_info.inode->i_mapping->a_ops = &balloon_aops;
 593         return 0;
 594 }
 595 static void cmm_balloon_compaction_deinit(void)
 596 {
 597         if (b_dev_info.inode)
 598                 iput(b_dev_info.inode);
 599         b_dev_info.inode = NULL;
 600         kern_unmount(balloon_mnt);
 601         balloon_mnt = NULL;
 602 }
 603 #else /* CONFIG_BALLOON_COMPACTION */
 604 static int cmm_balloon_compaction_init(void)
 605 {
 606         return 0;
 607 }
 608
 609 static void cmm_balloon_compaction_deinit(void)
 610 {
 611 }
 612 #endif /* CONFIG_BALLOON_COMPACTION */
 613
 614 /**
 615  * cmm_init - Module initialization
 616  *
 617  * Return value:
 618  *      0 on success / other on failure
 619  **/
 620 static int cmm_init(void)
 621 {
 622         int rc;
 623
 624         if (!firmware_has_feature(FW_FEATURE_CMO) && !simulate)
 625                 return -EOPNOTSUPP;
 626
 627         rc = cmm_balloon_compaction_init();
 628         if (rc)
 629                 return rc;
 630
 631         rc = register_oom_notifier(&cmm_oom_nb);
 632         if (rc < 0)
 633                 goto out_balloon_compaction;
 634
 635         if ((rc = register_reboot_notifier(&cmm_reboot_nb)))
 636                 goto out_oom_notifier;
 637
 638         if ((rc = cmm_sysfs_register(&cmm_dev)))
 639                 goto out_reboot_notifier;
 640
 641         rc = register_memory_notifier(&cmm_mem_nb);
 642         if (rc)
 643                 goto out_unregister_notifier;
 644
 645         if (cmm_disabled)
 646                 return 0;
 647
 648         cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
 649         if (IS_ERR(cmm_thread_ptr)) {
 650                 rc = PTR_ERR(cmm_thread_ptr);
 651                 goto out_unregister_notifier;
 652         }
 653
 654         return 0;
 655 out_unregister_notifier:
 656         unregister_memory_notifier(&cmm_mem_nb);
 657         cmm_unregister_sysfs(&cmm_dev);
 658 out_reboot_notifier:
 659         unregister_reboot_notifier(&cmm_reboot_nb);
 660 out_oom_notifier:
 661         unregister_oom_notifier(&cmm_oom_nb);
 662 out_balloon_compaction:
 663         cmm_balloon_compaction_deinit();
 664         return rc;
 665 }
 666
 667 /**
 668  * cmm_exit - Module exit
 669  *
 670  * Return value:
 671  *      nothing
 672  **/
 673 static void cmm_exit(void)
 674 {
 675         if (cmm_thread_ptr)
 676                 kthread_stop(cmm_thread_ptr);
 677         unregister_oom_notifier(&cmm_oom_nb);
 678         unregister_reboot_notifier(&cmm_reboot_nb);
 679         unregister_memory_notifier(&cmm_mem_nb);
 680         cmm_free_pages(atomic_long_read(&loaned_pages));
 681         cmm_unregister_sysfs(&cmm_dev);
 682         cmm_balloon_compaction_deinit();
 683 }
 684
 685 /**
 686  * cmm_set_disable - Disable/Enable CMM
 687  *
 688  * Return value:
 689  *      0 on success / other on failure
 690  **/
 691 static int cmm_set_disable(const char *val, const struct kernel_param *kp)
 692 {
 693         int disable = simple_strtoul(val, NULL, 10);
 694
 695         if (disable != 0 && disable != 1)
 696                 return -EINVAL;
 697
 698         if (disable && !cmm_disabled) {
 699                 if (cmm_thread_ptr)
 700                         kthread_stop(cmm_thread_ptr);
 701                 cmm_thread_ptr = NULL;
 702                 cmm_free_pages(atomic_long_read(&loaned_pages));
 703         } else if (!disable && cmm_disabled) {
 704                 cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
 705                 if (IS_ERR(cmm_thread_ptr))
 706                         return PTR_ERR(cmm_thread_ptr);
 707         }
 708
 709         cmm_disabled = disable;
 710         return 0;
 711 }
 712
 713 module_param_call(disable, cmm_set_disable, param_get_uint,
 714                   &cmm_disabled, 0644);
 715 MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. "
 716                  "[Default=" __stringify(CMM_DISABLE) "]");
 717
 718 module_init(cmm_init);
 719 module_exit(cmm_exit);