1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Virtio-mem device driver.
5 * Copyright Red Hat, Inc. 2020
7 * Author(s): David Hildenbrand <david@redhat.com>
10 #include <linux/virtio.h>
11 #include <linux/virtio_mem.h>
12 #include <linux/workqueue.h>
13 #include <linux/slab.h>
14 #include <linux/module.h>
16 #include <linux/memory_hotplug.h>
17 #include <linux/memory.h>
18 #include <linux/hrtimer.h>
19 #include <linux/crash_dump.h>
20 #include <linux/mutex.h>
21 #include <linux/bitmap.h>
22 #include <linux/lockdep.h>
24 #include <acpi/acpi_numa.h>
26 static bool unplug_online = true;
27 module_param(unplug_online, bool, 0644);
28 MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
31 * virtio-mem currently supports the following modes of operation:
33 * * Sub Block Mode (SBM): A Linux memory block spans 1..X subblocks (SB). The
34 * size of a Sub Block (SB) is determined based on the device block size, the
35 * pageblock size, and the maximum allocation granularity of the buddy.
36 * Subblocks within a Linux memory block might either be plugged or unplugged.
37 * Memory is added/removed to Linux MM in Linux memory block granularity.
39 * User space / core MM (auto onlining) is responsible for onlining added
40 * Linux memory blocks - and for selecting a zone. Linux Memory Blocks are
41 * always onlined separately, and all memory within a Linux memory block is
42 * onlined to the same zone - virtio-mem relies on this behavior.
46 * State of a Linux memory block in SBM.
48 enum virtio_mem_sbm_mb_state {
49 /* Unplugged, not added to Linux. Can be reused later. */
50 VIRTIO_MEM_SBM_MB_UNUSED = 0,
51 /* (Partially) plugged, not added to Linux. Error on add_memory(). */
52 VIRTIO_MEM_SBM_MB_PLUGGED,
53 /* Fully plugged, fully added to Linux, offline. */
54 VIRTIO_MEM_SBM_MB_OFFLINE,
55 /* Partially plugged, fully added to Linux, offline. */
56 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
57 /* Fully plugged, fully added to Linux, online. */
58 VIRTIO_MEM_SBM_MB_ONLINE,
59 /* Partially plugged, fully added to Linux, online. */
60 VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL,
61 VIRTIO_MEM_SBM_MB_COUNT
65 struct virtio_device *vdev;
67 /* We might first have to unplug all memory when starting up. */
68 bool unplug_all_required;
70 /* Workqueue that processes the plug/unplug requests. */
71 struct work_struct wq;
73 atomic_t config_changed;
75 /* Virtqueue for guest->host requests. */
78 /* Wait for a host response to a guest request. */
79 wait_queue_head_t host_resp;
81 /* Space for one guest request and the host response. */
82 struct virtio_mem_req req;
83 struct virtio_mem_resp resp;
85 /* The current size of the device. */
86 uint64_t plugged_size;
87 /* The requested size of the device. */
88 uint64_t requested_size;
90 /* The device block size (for communicating with the device). */
91 uint64_t device_block_size;
92 /* The determined node id for all memory of the device. */
94 /* Physical start address of the memory region. */
96 /* Maximum region size in bytes. */
99 /* The subblock size. */
100 uint64_t subblock_size;
101 /* The number of subblocks per memory block. */
102 uint32_t nb_sb_per_mb;
104 /* Id of the first memory block of this device. */
105 unsigned long first_mb_id;
106 /* Id of the last usable memory block of this device. */
107 unsigned long last_usable_mb_id;
108 /* Id of the next memory bock to prepare when needed. */
109 unsigned long next_mb_id;
111 /* The parent resource for all memory added via this device. */
112 struct resource *parent_resource;
114 * Copy of "System RAM (virtio_mem)" to be used for
115 * add_memory_driver_managed().
117 const char *resource_name;
120 * We don't want to add too much memory if it's not getting onlined,
121 * to avoid running OOM. Besides this threshold, we allow to have at
122 * least two offline blocks at a time (whatever is bigger).
124 #define VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD (1024 * 1024 * 1024)
125 atomic64_t offline_size;
126 uint64_t offline_threshold;
129 /* Summary of all memory block states. */
130 unsigned long mb_count[VIRTIO_MEM_SBM_MB_COUNT];
133 * One byte state per memory block. Allocated via vmalloc().
134 * Resized (alloc+copy+free) on demand.
136 * With 128 MiB memory blocks, we have states for 512 GiB of
137 * memory in one 4 KiB page.
142 * Bitmap: one bit per subblock. Allocated similar to
145 * A set bit means the corresponding subblock is plugged,
146 * otherwise it's unblocked.
148 * With 4 MiB subblocks, we manage 128 GiB of memory in one
151 unsigned long *sb_states;
155 * Mutex that protects the sbm.mb_count, sbm.mb_states, and
158 * When this lock is held the pointers can't change, ONLINE and
159 * OFFLINE blocks can't change the state and no subblocks will get
162 struct mutex hotplug_mutex;
165 /* An error occurred we cannot handle - stop processing requests. */
168 /* The driver is being removed. */
169 spinlock_t removal_lock;
172 /* Timer for retrying to plug/unplug memory. */
173 struct hrtimer retry_timer;
174 unsigned int retry_timer_ms;
175 #define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000
176 #define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000
178 /* Memory notifier (online/offline events). */
179 struct notifier_block memory_notifier;
181 /* Next device in the list of virtio-mem devices. */
182 struct list_head next;
186 * We have to share a single online_page callback among all virtio-mem
187 * devices. We use RCU to iterate the list in the callback.
189 static DEFINE_MUTEX(virtio_mem_mutex);
190 static LIST_HEAD(virtio_mem_devices);
192 static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
193 static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
194 unsigned long nr_pages);
195 static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
196 unsigned long nr_pages);
197 static void virtio_mem_retry(struct virtio_mem *vm);
200 * Register a virtio-mem device so it will be considered for the online_page
203 static int register_virtio_mem_device(struct virtio_mem *vm)
207 /* First device registers the callback. */
208 mutex_lock(&virtio_mem_mutex);
209 if (list_empty(&virtio_mem_devices))
210 rc = set_online_page_callback(&virtio_mem_online_page_cb);
212 list_add_rcu(&vm->next, &virtio_mem_devices);
213 mutex_unlock(&virtio_mem_mutex);
219 * Unregister a virtio-mem device so it will no longer be considered for the
220 * online_page callback.
222 static void unregister_virtio_mem_device(struct virtio_mem *vm)
224 /* Last device unregisters the callback. */
225 mutex_lock(&virtio_mem_mutex);
226 list_del_rcu(&vm->next);
227 if (list_empty(&virtio_mem_devices))
228 restore_online_page_callback(&virtio_mem_online_page_cb);
229 mutex_unlock(&virtio_mem_mutex);
235 * Calculate the memory block id of a given address.
237 static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
239 return addr / memory_block_size_bytes();
243 * Calculate the physical start address of a given memory block id.
245 static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
247 return mb_id * memory_block_size_bytes();
251 * Calculate the subblock id of a given address.
253 static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
256 const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
257 const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
259 return (addr - mb_addr) / vm->subblock_size;
263 * Set the state of a memory block, taking care of the state counter.
265 static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm,
266 unsigned long mb_id, uint8_t state)
268 const unsigned long idx = mb_id - vm->first_mb_id;
271 old_state = vm->sbm.mb_states[idx];
272 vm->sbm.mb_states[idx] = state;
274 BUG_ON(vm->sbm.mb_count[old_state] == 0);
275 vm->sbm.mb_count[old_state]--;
276 vm->sbm.mb_count[state]++;
280 * Get the state of a memory block.
282 static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm,
285 const unsigned long idx = mb_id - vm->first_mb_id;
287 return vm->sbm.mb_states[idx];
291 * Prepare the state array for the next memory block.
293 static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm)
295 int old_pages = PFN_UP(vm->next_mb_id - vm->first_mb_id);
296 int new_pages = PFN_UP(vm->next_mb_id - vm->first_mb_id + 1);
299 if (vm->sbm.mb_states && old_pages == new_pages)
302 new_array = vzalloc(new_pages * PAGE_SIZE);
306 mutex_lock(&vm->hotplug_mutex);
307 if (vm->sbm.mb_states)
308 memcpy(new_array, vm->sbm.mb_states, old_pages * PAGE_SIZE);
309 vfree(vm->sbm.mb_states);
310 vm->sbm.mb_states = new_array;
311 mutex_unlock(&vm->hotplug_mutex);
316 #define virtio_mem_sbm_for_each_mb(_vm, _mb_id, _state) \
317 for (_mb_id = _vm->first_mb_id; \
318 _mb_id < _vm->next_mb_id && _vm->sbm.mb_count[_state]; \
320 if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
322 #define virtio_mem_sbm_for_each_mb_rev(_vm, _mb_id, _state) \
323 for (_mb_id = _vm->next_mb_id - 1; \
324 _mb_id >= _vm->first_mb_id && _vm->sbm.mb_count[_state]; \
326 if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
329 * Calculate the bit number in the subblock bitmap for the given subblock
330 * inside the given memory block.
332 static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm,
333 unsigned long mb_id, int sb_id)
335 return (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
339 * Mark all selected subblocks plugged.
341 * Will not modify the state of the memory block.
343 static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm,
344 unsigned long mb_id, int sb_id,
347 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
349 __bitmap_set(vm->sbm.sb_states, bit, count);
353 * Mark all selected subblocks unplugged.
355 * Will not modify the state of the memory block.
357 static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm,
358 unsigned long mb_id, int sb_id,
361 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
363 __bitmap_clear(vm->sbm.sb_states, bit, count);
367 * Test if all selected subblocks are plugged.
369 static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm,
370 unsigned long mb_id, int sb_id,
373 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
376 return test_bit(bit, vm->sbm.sb_states);
378 /* TODO: Helper similar to bitmap_set() */
379 return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >=
384 * Test if all selected subblocks are unplugged.
386 static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm,
387 unsigned long mb_id, int sb_id,
390 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
392 /* TODO: Helper similar to bitmap_set() */
393 return find_next_bit(vm->sbm.sb_states, bit + count, bit) >=
398 * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
401 static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm,
404 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0);
406 return find_next_zero_bit(vm->sbm.sb_states,
407 bit + vm->nb_sb_per_mb, bit) - bit;
411 * Prepare the subblock bitmap for the next memory block.
413 static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm)
415 const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
416 const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
417 const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
418 int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
419 int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
420 unsigned long *new_bitmap, *old_bitmap;
422 if (vm->sbm.sb_states && old_pages == new_pages)
425 new_bitmap = vzalloc(new_pages * PAGE_SIZE);
429 mutex_lock(&vm->hotplug_mutex);
431 memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE);
433 old_bitmap = vm->sbm.sb_states;
434 vm->sbm.sb_states = new_bitmap;
435 mutex_unlock(&vm->hotplug_mutex);
442 * Test if we could add memory without creating too much offline memory -
443 * to avoid running OOM if memory is getting onlined deferred.
445 static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size)
447 if (WARN_ON_ONCE(size > vm->offline_threshold))
450 return atomic64_read(&vm->offline_size) + size <= vm->offline_threshold;
454 * Try to add a memory block to Linux. This will usually only fail
457 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
460 * Will not modify the state of the memory block.
462 static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
464 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
465 const uint64_t size = memory_block_size_bytes();
469 * When force-unloading the driver and we still have memory added to
470 * Linux, the resource name has to stay.
472 if (!vm->resource_name) {
473 vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
475 if (!vm->resource_name)
479 dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
480 /* Memory might get onlined immediately. */
481 atomic64_add(size, &vm->offline_size);
482 rc = add_memory_driver_managed(vm->nid, addr, size, vm->resource_name,
483 MEMHP_MERGE_RESOURCE);
485 atomic64_sub(size, &vm->offline_size);
490 * Try to remove a memory block from Linux. Will only fail if the memory block
493 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
496 * Will not modify the state of the memory block.
498 static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
500 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
501 const uint64_t size = memory_block_size_bytes();
504 dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
505 rc = remove_memory(vm->nid, addr, size);
507 atomic64_sub(size, &vm->offline_size);
509 * We might have freed up memory we can now unplug, retry
510 * immediately instead of waiting.
512 virtio_mem_retry(vm);
518 * Try to offline and remove a memory block from Linux.
520 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
523 * Will not modify the state of the memory block.
525 static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
528 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
529 const uint64_t size = memory_block_size_bytes();
532 dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
534 rc = offline_and_remove_memory(vm->nid, addr, size);
536 atomic64_sub(size, &vm->offline_size);
538 * We might have freed up memory we can now unplug, retry
539 * immediately instead of waiting.
541 virtio_mem_retry(vm);
547 * Trigger the workqueue so the device can perform its magic.
549 static void virtio_mem_retry(struct virtio_mem *vm)
553 spin_lock_irqsave(&vm->removal_lock, flags);
555 queue_work(system_freezable_wq, &vm->wq);
556 spin_unlock_irqrestore(&vm->removal_lock, flags);
559 static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
561 int node = NUMA_NO_NODE;
563 #if defined(CONFIG_ACPI_NUMA)
564 if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
565 node = pxm_to_node(node_id);
571 * Test if a virtio-mem device overlaps with the given range. Can be called
572 * from (notifier) callbacks lockless.
574 static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start,
577 return start < vm->addr + vm->region_size && vm->addr < start + size;
581 * Test if a virtio-mem device contains a given range. Can be called from
582 * (notifier) callbacks lockless.
584 static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start,
587 return start >= vm->addr && start + size <= vm->addr + vm->region_size;
590 static int virtio_mem_notify_going_online(struct virtio_mem *vm,
593 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
594 case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
595 case VIRTIO_MEM_SBM_MB_OFFLINE:
600 dev_warn_ratelimited(&vm->vdev->dev,
601 "memory block onlining denied\n");
605 static void virtio_mem_notify_offline(struct virtio_mem *vm,
608 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
609 case VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL:
610 virtio_mem_sbm_set_mb_state(vm, mb_id,
611 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
613 case VIRTIO_MEM_SBM_MB_ONLINE:
614 virtio_mem_sbm_set_mb_state(vm, mb_id,
615 VIRTIO_MEM_SBM_MB_OFFLINE);
623 static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id)
625 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
626 case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
627 virtio_mem_sbm_set_mb_state(vm, mb_id,
628 VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL);
630 case VIRTIO_MEM_SBM_MB_OFFLINE:
631 virtio_mem_sbm_set_mb_state(vm, mb_id,
632 VIRTIO_MEM_SBM_MB_ONLINE);
640 static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
643 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
647 for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
648 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
650 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
651 sb_id * vm->subblock_size);
652 virtio_mem_fake_offline_going_offline(pfn, nr_pages);
656 static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
659 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
663 for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
664 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
666 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
667 sb_id * vm->subblock_size);
668 virtio_mem_fake_offline_cancel_offline(pfn, nr_pages);
673 * This callback will either be called synchronously from add_memory() or
674 * asynchronously (e.g., triggered via user space). We have to be careful
675 * with locking when calling add_memory().
677 static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
678 unsigned long action, void *arg)
680 struct virtio_mem *vm = container_of(nb, struct virtio_mem,
682 struct memory_notify *mhp = arg;
683 const unsigned long start = PFN_PHYS(mhp->start_pfn);
684 const unsigned long size = PFN_PHYS(mhp->nr_pages);
685 const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
688 if (!virtio_mem_overlaps_range(vm, start, size))
692 * Memory is onlined/offlined in memory block granularity. We cannot
693 * cross virtio-mem device boundaries and memory block boundaries. Bail
694 * out if this ever changes.
696 if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
697 !IS_ALIGNED(start, memory_block_size_bytes())))
701 * Avoid circular locking lockdep warnings. We lock the mutex
702 * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
703 * blocking_notifier_call_chain() has it's own lock, which gets unlocked
704 * between both notifier calls and will bail out. False positive.
709 case MEM_GOING_OFFLINE:
710 mutex_lock(&vm->hotplug_mutex);
712 rc = notifier_from_errno(-EBUSY);
713 mutex_unlock(&vm->hotplug_mutex);
716 vm->hotplug_active = true;
717 virtio_mem_notify_going_offline(vm, mb_id);
719 case MEM_GOING_ONLINE:
720 mutex_lock(&vm->hotplug_mutex);
722 rc = notifier_from_errno(-EBUSY);
723 mutex_unlock(&vm->hotplug_mutex);
726 vm->hotplug_active = true;
727 rc = virtio_mem_notify_going_online(vm, mb_id);
730 virtio_mem_notify_offline(vm, mb_id);
732 atomic64_add(size, &vm->offline_size);
734 * Trigger the workqueue. Now that we have some offline memory,
735 * maybe we can handle pending unplug requests.
738 virtio_mem_retry(vm);
740 vm->hotplug_active = false;
741 mutex_unlock(&vm->hotplug_mutex);
744 virtio_mem_notify_online(vm, mb_id);
746 atomic64_sub(size, &vm->offline_size);
748 * Start adding more memory once we onlined half of our
749 * threshold. Don't trigger if it's possibly due to our actipn
750 * (e.g., us adding memory which gets onlined immediately from
753 if (!atomic_read(&vm->wq_active) &&
754 virtio_mem_could_add_memory(vm, vm->offline_threshold / 2))
755 virtio_mem_retry(vm);
757 vm->hotplug_active = false;
758 mutex_unlock(&vm->hotplug_mutex);
760 case MEM_CANCEL_OFFLINE:
761 if (!vm->hotplug_active)
763 virtio_mem_notify_cancel_offline(vm, mb_id);
764 vm->hotplug_active = false;
765 mutex_unlock(&vm->hotplug_mutex);
767 case MEM_CANCEL_ONLINE:
768 if (!vm->hotplug_active)
770 vm->hotplug_active = false;
771 mutex_unlock(&vm->hotplug_mutex);
783 * Set a range of pages PG_offline. Remember pages that were never onlined
784 * (via generic_online_page()) using PageDirty().
786 static void virtio_mem_set_fake_offline(unsigned long pfn,
787 unsigned long nr_pages, bool onlined)
789 for (; nr_pages--; pfn++) {
790 struct page *page = pfn_to_page(pfn);
792 __SetPageOffline(page);
795 /* FIXME: remove after cleanups */
796 ClearPageReserved(page);
802 * Clear PG_offline from a range of pages. If the pages were never onlined,
803 * (via generic_online_page()), clear PageDirty().
805 static void virtio_mem_clear_fake_offline(unsigned long pfn,
806 unsigned long nr_pages, bool onlined)
808 for (; nr_pages--; pfn++) {
809 struct page *page = pfn_to_page(pfn);
811 __ClearPageOffline(page);
813 ClearPageDirty(page);
818 * Release a range of fake-offline pages to the buddy, effectively
819 * fake-onlining them.
821 static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages)
823 const unsigned long max_nr_pages = MAX_ORDER_NR_PAGES;
827 * We are always called at least with MAX_ORDER_NR_PAGES
828 * granularity/alignment (e.g., the way subblocks work). All pages
829 * inside such a block are alike.
831 for (i = 0; i < nr_pages; i += max_nr_pages) {
832 struct page *page = pfn_to_page(pfn + i);
835 * If the page is PageDirty(), it was kept fake-offline when
836 * onlining the memory block. Otherwise, it was allocated
837 * using alloc_contig_range(). All pages in a subblock are
840 if (PageDirty(page)) {
841 virtio_mem_clear_fake_offline(pfn + i, max_nr_pages,
843 generic_online_page(page, MAX_ORDER - 1);
845 virtio_mem_clear_fake_offline(pfn + i, max_nr_pages,
847 free_contig_range(pfn + i, max_nr_pages);
848 adjust_managed_page_count(page, max_nr_pages);
854 * Try to allocate a range, marking pages fake-offline, effectively
855 * fake-offlining them.
857 static int virtio_mem_fake_offline(unsigned long pfn, unsigned long nr_pages)
859 const bool is_movable = zone_idx(page_zone(pfn_to_page(pfn))) ==
864 * TODO: We want an alloc_contig_range() mode that tries to allocate
865 * harder (e.g., dealing with temporarily pinned pages, PCP), especially
866 * with ZONE_MOVABLE. So for now, retry a couple of times with
867 * ZONE_MOVABLE before giving up - because that zone is supposed to give
870 for (retry_count = 0; retry_count < 5; retry_count++) {
871 rc = alloc_contig_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE,
874 /* whoops, out of memory */
876 else if (rc && !is_movable)
881 virtio_mem_set_fake_offline(pfn, nr_pages, true);
882 adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
890 * Handle fake-offline pages when memory is going offline - such that the
891 * pages can be skipped by mm-core when offlining.
893 static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
894 unsigned long nr_pages)
900 * Drop our reference to the pages so the memory can get offlined
901 * and add the unplugged pages to the managed page counters (so
902 * offlining code can correctly subtract them again).
904 adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
905 /* Drop our reference to the pages so the memory can get offlined. */
906 for (i = 0; i < nr_pages; i++) {
907 page = pfn_to_page(pfn + i);
908 if (WARN_ON(!page_ref_dec_and_test(page)))
909 dump_page(page, "fake-offline page referenced");
914 * Handle fake-offline pages when memory offlining is canceled - to undo
915 * what we did in virtio_mem_fake_offline_going_offline().
917 static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
918 unsigned long nr_pages)
923 * Get the reference we dropped when going offline and subtract the
924 * unplugged pages from the managed page counters.
926 adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
927 for (i = 0; i < nr_pages; i++)
928 page_ref_inc(pfn_to_page(pfn + i));
931 static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
933 const unsigned long addr = page_to_phys(page);
934 const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
935 struct virtio_mem *vm;
939 * We exploit here that subblocks have at least MAX_ORDER_NR_PAGES.
940 * size/alignment and that this callback is is called with such a
941 * size/alignment. So we cannot cross subblocks and therefore
942 * also not memory blocks.
945 list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
946 if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order)))
949 sb_id = virtio_mem_phys_to_sb_id(vm, addr);
951 * If plugged, online the pages, otherwise, set them fake
952 * offline (PageOffline).
954 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
955 generic_online_page(page, order);
957 virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
964 /* not virtio-mem memory, but e.g., a DIMM. online it */
965 generic_online_page(page, order);
968 static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
969 const struct virtio_mem_req *req)
971 struct scatterlist *sgs[2], sg_req, sg_resp;
975 /* don't use the request residing on the stack (vaddr) */
978 /* out: buffer for request */
979 sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
982 /* in: buffer for response */
983 sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
986 rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
990 virtqueue_kick(vm->vq);
992 /* wait for a response */
993 wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
995 return virtio16_to_cpu(vm->vdev, vm->resp.type);
998 static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
1001 const uint64_t nb_vm_blocks = size / vm->device_block_size;
1002 const struct virtio_mem_req req = {
1003 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
1004 .u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
1005 .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
1009 if (atomic_read(&vm->config_changed))
1012 dev_dbg(&vm->vdev->dev, "plugging memory: 0x%llx - 0x%llx\n", addr,
1015 switch (virtio_mem_send_request(vm, &req)) {
1016 case VIRTIO_MEM_RESP_ACK:
1017 vm->plugged_size += size;
1019 case VIRTIO_MEM_RESP_NACK:
1022 case VIRTIO_MEM_RESP_BUSY:
1025 case VIRTIO_MEM_RESP_ERROR:
1032 dev_dbg(&vm->vdev->dev, "plugging memory failed: %d\n", rc);
1036 static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
1039 const uint64_t nb_vm_blocks = size / vm->device_block_size;
1040 const struct virtio_mem_req req = {
1041 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
1042 .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
1043 .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
1047 if (atomic_read(&vm->config_changed))
1050 dev_dbg(&vm->vdev->dev, "unplugging memory: 0x%llx - 0x%llx\n", addr,
1053 switch (virtio_mem_send_request(vm, &req)) {
1054 case VIRTIO_MEM_RESP_ACK:
1055 vm->plugged_size -= size;
1057 case VIRTIO_MEM_RESP_BUSY:
1060 case VIRTIO_MEM_RESP_ERROR:
1067 dev_dbg(&vm->vdev->dev, "unplugging memory failed: %d\n", rc);
1071 static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
1073 const struct virtio_mem_req req = {
1074 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
1078 dev_dbg(&vm->vdev->dev, "unplugging all memory");
1080 switch (virtio_mem_send_request(vm, &req)) {
1081 case VIRTIO_MEM_RESP_ACK:
1082 vm->unplug_all_required = false;
1083 vm->plugged_size = 0;
1084 /* usable region might have shrunk */
1085 atomic_set(&vm->config_changed, 1);
1087 case VIRTIO_MEM_RESP_BUSY:
1094 dev_dbg(&vm->vdev->dev, "unplugging all memory failed: %d\n", rc);
1099 * Plug selected subblocks. Updates the plugged state, but not the state
1100 * of the memory block.
1102 static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
1103 int sb_id, int count)
1105 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
1106 sb_id * vm->subblock_size;
1107 const uint64_t size = count * vm->subblock_size;
1110 rc = virtio_mem_send_plug_request(vm, addr, size);
1112 virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count);
1117 * Unplug selected subblocks. Updates the plugged state, but not the state
1118 * of the memory block.
1120 static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
1121 int sb_id, int count)
1123 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
1124 sb_id * vm->subblock_size;
1125 const uint64_t size = count * vm->subblock_size;
1128 rc = virtio_mem_send_unplug_request(vm, addr, size);
1130 virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count);
1135 * Unplug the desired number of plugged subblocks of a offline or not-added
1136 * memory block. Will fail if any subblock cannot get unplugged (instead of
1139 * Will not modify the state of the memory block.
1141 * Note: can fail after some subblocks were unplugged.
1143 static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
1144 unsigned long mb_id, uint64_t *nb_sb)
1149 sb_id = vm->nb_sb_per_mb - 1;
1151 /* Find the next candidate subblock */
1152 while (sb_id >= 0 &&
1153 virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1))
1157 /* Try to unplug multiple subblocks at a time */
1159 while (count < *nb_sb && sb_id > 0 &&
1160 virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
1165 rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1176 * Unplug all plugged subblocks of an offline or not-added memory block.
1178 * Will not modify the state of the memory block.
1180 * Note: can fail after some subblocks were unplugged.
1182 static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
1184 uint64_t nb_sb = vm->nb_sb_per_mb;
1186 return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
1190 * Prepare tracking data for the next memory block.
1192 static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
1193 unsigned long *mb_id)
1197 if (vm->next_mb_id > vm->last_usable_mb_id)
1200 /* Resize the state array if required. */
1201 rc = virtio_mem_sbm_mb_states_prepare_next_mb(vm);
1205 /* Resize the subblock bitmap if required. */
1206 rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm);
1210 vm->sbm.mb_count[VIRTIO_MEM_SBM_MB_UNUSED]++;
1211 *mb_id = vm->next_mb_id++;
1216 * Try to plug the desired number of subblocks and add the memory block
1219 * Will modify the state of the memory block.
1221 static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
1222 unsigned long mb_id,
1225 const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
1228 if (WARN_ON_ONCE(!count))
1232 * Plug the requested number of subblocks before adding it to linux,
1233 * so that onlining will directly online all plugged subblocks.
1235 rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
1240 * Mark the block properly offline before adding it to Linux,
1241 * so the memory notifiers will find the block in the right state.
1243 if (count == vm->nb_sb_per_mb)
1244 virtio_mem_sbm_set_mb_state(vm, mb_id,
1245 VIRTIO_MEM_SBM_MB_OFFLINE);
1247 virtio_mem_sbm_set_mb_state(vm, mb_id,
1248 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
1250 /* Add the memory block to linux - if that fails, try to unplug. */
1251 rc = virtio_mem_mb_add(vm, mb_id);
1253 int new_state = VIRTIO_MEM_SBM_MB_UNUSED;
1255 dev_err(&vm->vdev->dev,
1256 "adding memory block %lu failed with %d\n", mb_id, rc);
1259 * TODO: Linux MM does not properly clean up yet in all cases
1260 * where adding of memory failed - especially on -ENOMEM.
1262 if (virtio_mem_mb_unplug_sb(vm, mb_id, 0, count))
1263 new_state = VIRTIO_MEM_SBM_MB_PLUGGED;
1264 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state);
1273 * Try to plug the desired number of subblocks of a memory block that
1274 * is already added to Linux.
1276 * Will modify the state of the memory block.
1278 * Note: Can fail after some subblocks were successfully plugged.
1280 static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
1281 uint64_t *nb_sb, bool online)
1283 unsigned long pfn, nr_pages;
1287 if (WARN_ON_ONCE(!*nb_sb))
1291 sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id);
1292 if (sb_id >= vm->nb_sb_per_mb)
1295 while (count < *nb_sb &&
1296 sb_id + count < vm->nb_sb_per_mb &&
1297 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1))
1300 rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
1307 /* fake-online the pages if the memory block is online */
1308 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1309 sb_id * vm->subblock_size);
1310 nr_pages = PFN_DOWN(count * vm->subblock_size);
1311 virtio_mem_fake_online(pfn, nr_pages);
1314 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1316 virtio_mem_sbm_set_mb_state(vm, mb_id,
1317 VIRTIO_MEM_SBM_MB_ONLINE);
1319 virtio_mem_sbm_set_mb_state(vm, mb_id,
1320 VIRTIO_MEM_SBM_MB_OFFLINE);
1327 * Try to plug the requested amount of memory.
1329 static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
1331 uint64_t nb_sb = diff / vm->subblock_size;
1332 unsigned long mb_id;
1338 /* Don't race with onlining/offlining */
1339 mutex_lock(&vm->hotplug_mutex);
1341 /* Try to plug subblocks of partially plugged online blocks. */
1342 virtio_mem_sbm_for_each_mb(vm, mb_id,
1343 VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL) {
1344 rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
1350 /* Try to plug subblocks of partially plugged offline blocks. */
1351 virtio_mem_sbm_for_each_mb(vm, mb_id,
1352 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) {
1353 rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
1360 * We won't be working on online/offline memory blocks from this point,
1361 * so we can't race with memory onlining/offlining. Drop the mutex.
1363 mutex_unlock(&vm->hotplug_mutex);
1365 /* Try to plug and add unused blocks */
1366 virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_UNUSED) {
1367 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
1370 rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1376 /* Try to prepare, plug and add new blocks */
1378 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
1381 rc = virtio_mem_prepare_next_mb(vm, &mb_id);
1384 rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1392 mutex_unlock(&vm->hotplug_mutex);
1397 * Unplug the desired number of plugged subblocks of an offline memory block.
1398 * Will fail if any subblock cannot get unplugged (instead of skipping it).
1400 * Will modify the state of the memory block. Might temporarily drop the
1403 * Note: Can fail after some subblocks were successfully unplugged.
1405 static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
1406 unsigned long mb_id,
1411 rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
1413 /* some subblocks might have been unplugged even on failure */
1414 if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
1415 virtio_mem_sbm_set_mb_state(vm, mb_id,
1416 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
1420 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1422 * Remove the block from Linux - this should never fail.
1423 * Hinder the block from getting onlined by marking it
1424 * unplugged. Temporarily drop the mutex, so
1425 * any pending GOING_ONLINE requests can be serviced/rejected.
1427 virtio_mem_sbm_set_mb_state(vm, mb_id,
1428 VIRTIO_MEM_SBM_MB_UNUSED);
1430 mutex_unlock(&vm->hotplug_mutex);
1431 rc = virtio_mem_mb_remove(vm, mb_id);
1433 mutex_lock(&vm->hotplug_mutex);
1439 * Unplug the given plugged subblocks of an online memory block.
1441 * Will modify the state of the memory block.
1443 static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
1444 unsigned long mb_id, int sb_id,
1447 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
1448 unsigned long start_pfn;
1451 start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1452 sb_id * vm->subblock_size);
1454 rc = virtio_mem_fake_offline(start_pfn, nr_pages);
1458 /* Try to unplug the allocated memory */
1459 rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1461 /* Return the memory to the buddy. */
1462 virtio_mem_fake_online(start_pfn, nr_pages);
1466 virtio_mem_sbm_set_mb_state(vm, mb_id,
1467 VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL);
1472 * Unplug the desired number of plugged subblocks of an online memory block.
1473 * Will skip subblock that are busy.
1475 * Will modify the state of the memory block. Might temporarily drop the
1478 * Note: Can fail after some subblocks were successfully unplugged. Can
1479 * return 0 even if subblocks were busy and could not get unplugged.
1481 static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
1482 unsigned long mb_id,
1487 /* If possible, try to unplug the complete block in one shot. */
1488 if (*nb_sb >= vm->nb_sb_per_mb &&
1489 virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1490 rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
1493 *nb_sb -= vm->nb_sb_per_mb;
1495 } else if (rc != -EBUSY)
1499 /* Fallback to single subblocks. */
1500 for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
1501 /* Find the next candidate subblock */
1502 while (sb_id >= 0 &&
1503 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
1508 rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
1518 * Once all subblocks of a memory block were unplugged, offline and
1519 * remove it. This will usually not fail, as no memory is in use
1520 * anymore - however some other notifiers might NACK the request.
1522 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1523 mutex_unlock(&vm->hotplug_mutex);
1524 rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
1525 mutex_lock(&vm->hotplug_mutex);
1527 virtio_mem_sbm_set_mb_state(vm, mb_id,
1528 VIRTIO_MEM_SBM_MB_UNUSED);
1535 * Try to unplug the requested amount of memory.
1537 static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
1539 uint64_t nb_sb = diff / vm->subblock_size;
1540 unsigned long mb_id;
1547 * We'll drop the mutex a couple of times when it is safe to do so.
1548 * This might result in some blocks switching the state (online/offline)
1549 * and we could miss them in this run - we will retry again later.
1551 mutex_lock(&vm->hotplug_mutex);
1553 /* Try to unplug subblocks of partially plugged offline blocks. */
1554 virtio_mem_sbm_for_each_mb_rev(vm, mb_id,
1555 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) {
1556 rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1563 /* Try to unplug subblocks of plugged offline blocks. */
1564 virtio_mem_sbm_for_each_mb_rev(vm, mb_id, VIRTIO_MEM_SBM_MB_OFFLINE) {
1565 rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1572 if (!unplug_online) {
1573 mutex_unlock(&vm->hotplug_mutex);
1577 /* Try to unplug subblocks of partially plugged online blocks. */
1578 virtio_mem_sbm_for_each_mb_rev(vm, mb_id,
1579 VIRTIO_MEM_SBM_MB_ONLINE_PARTIAL) {
1580 rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1584 mutex_unlock(&vm->hotplug_mutex);
1586 mutex_lock(&vm->hotplug_mutex);
1589 /* Try to unplug subblocks of plugged online blocks. */
1590 virtio_mem_sbm_for_each_mb_rev(vm, mb_id, VIRTIO_MEM_SBM_MB_ONLINE) {
1591 rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1595 mutex_unlock(&vm->hotplug_mutex);
1597 mutex_lock(&vm->hotplug_mutex);
1600 mutex_unlock(&vm->hotplug_mutex);
1601 return nb_sb ? -EBUSY : 0;
1603 mutex_unlock(&vm->hotplug_mutex);
1608 * Try to unplug all blocks that couldn't be unplugged before, for example,
1609 * because the hypervisor was busy.
1611 static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
1613 unsigned long mb_id;
1616 virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_PLUGGED) {
1617 rc = virtio_mem_mb_unplug(vm, mb_id);
1620 virtio_mem_sbm_set_mb_state(vm, mb_id,
1621 VIRTIO_MEM_SBM_MB_UNUSED);
1628 * Update all parts of the config that could have changed.
1630 static void virtio_mem_refresh_config(struct virtio_mem *vm)
1632 const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1633 uint64_t new_plugged_size, usable_region_size, end_addr;
1635 /* the plugged_size is just a reflection of what _we_ did previously */
1636 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1638 if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
1639 vm->plugged_size = new_plugged_size;
1641 /* calculate the last usable memory block id */
1642 virtio_cread_le(vm->vdev, struct virtio_mem_config,
1643 usable_region_size, &usable_region_size);
1644 end_addr = vm->addr + usable_region_size;
1645 end_addr = min(end_addr, phys_limit);
1646 vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
1648 /* see if there is a request to change the size */
1649 virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
1650 &vm->requested_size);
1652 dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
1653 dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
1657 * Workqueue function for handling plug/unplug requests and config updates.
1659 static void virtio_mem_run_wq(struct work_struct *work)
1661 struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
1665 hrtimer_cancel(&vm->retry_timer);
1670 atomic_set(&vm->wq_active, 1);
1674 /* Make sure we start with a clean state if there are leftovers. */
1675 if (unlikely(vm->unplug_all_required))
1676 rc = virtio_mem_send_unplug_all_request(vm);
1678 if (atomic_read(&vm->config_changed)) {
1679 atomic_set(&vm->config_changed, 0);
1680 virtio_mem_refresh_config(vm);
1683 /* Unplug any leftovers from previous runs */
1685 rc = virtio_mem_unplug_pending_mb(vm);
1687 if (!rc && vm->requested_size != vm->plugged_size) {
1688 if (vm->requested_size > vm->plugged_size) {
1689 diff = vm->requested_size - vm->plugged_size;
1690 rc = virtio_mem_plug_request(vm, diff);
1692 diff = vm->plugged_size - vm->requested_size;
1693 rc = virtio_mem_unplug_request(vm, diff);
1699 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
1703 * We cannot add any more memory (alignment, physical limit)
1704 * or we have too many offline memory blocks.
1709 * The hypervisor cannot process our request right now
1710 * (e.g., out of memory, migrating);
1714 * We cannot free up any memory to unplug it (all plugged memory
1718 /* Out of memory, try again later. */
1719 hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
1723 /* Retry immediately (e.g., the config changed). */
1726 /* Unknown error, mark as broken */
1727 dev_err(&vm->vdev->dev,
1728 "unknown error, marking device broken: %d\n", rc);
1732 atomic_set(&vm->wq_active, 0);
1735 static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
1737 struct virtio_mem *vm = container_of(timer, struct virtio_mem,
1740 virtio_mem_retry(vm);
1741 vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
1742 VIRTIO_MEM_RETRY_TIMER_MAX_MS);
1743 return HRTIMER_NORESTART;
1746 static void virtio_mem_handle_response(struct virtqueue *vq)
1748 struct virtio_mem *vm = vq->vdev->priv;
1750 wake_up(&vm->host_resp);
1753 static int virtio_mem_init_vq(struct virtio_mem *vm)
1755 struct virtqueue *vq;
1757 vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
1766 static int virtio_mem_init(struct virtio_mem *vm)
1768 const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1771 if (!vm->vdev->config->get) {
1772 dev_err(&vm->vdev->dev, "config access disabled\n");
1777 * We don't want to (un)plug or reuse any memory when in kdump. The
1778 * memory is still accessible (but not mapped).
1780 if (is_kdump_kernel()) {
1781 dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
1785 /* Fetch all properties that can't change. */
1786 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1788 virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
1789 &vm->device_block_size);
1790 virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
1792 vm->nid = virtio_mem_translate_node_id(vm, node_id);
1793 virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
1794 virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
1797 /* Determine the nid for the device based on the lowest address. */
1798 if (vm->nid == NUMA_NO_NODE)
1799 vm->nid = memory_add_physaddr_to_nid(vm->addr);
1802 * We always hotplug memory in memory block granularity. This way,
1803 * we have to wait for exactly one memory block to online.
1805 if (vm->device_block_size > memory_block_size_bytes()) {
1806 dev_err(&vm->vdev->dev,
1807 "The block size is not supported (too big).\n");
1811 /* bad device setup - warn only */
1812 if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
1813 dev_warn(&vm->vdev->dev,
1814 "The alignment of the physical start address can make some memory unusable.\n");
1815 if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
1816 dev_warn(&vm->vdev->dev,
1817 "The alignment of the physical end address can make some memory unusable.\n");
1818 if (vm->addr + vm->region_size > phys_limit)
1819 dev_warn(&vm->vdev->dev,
1820 "Some memory is not addressable. This can make some memory unusable.\n");
1823 * We want subblocks to span at least MAX_ORDER_NR_PAGES and
1824 * pageblock_nr_pages pages. This:
1825 * - Simplifies our page onlining code (virtio_mem_online_page_cb)
1826 * and fake page onlining code (virtio_mem_fake_online).
1827 * - Is required for now for alloc_contig_range() to work reliably -
1828 * it doesn't properly handle smaller granularity on ZONE_NORMAL.
1830 vm->subblock_size = max_t(uint64_t, MAX_ORDER_NR_PAGES,
1831 pageblock_nr_pages) * PAGE_SIZE;
1832 vm->subblock_size = max_t(uint64_t, vm->device_block_size,
1834 vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
1836 /* Round up to the next full memory block */
1837 vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
1838 memory_block_size_bytes());
1839 vm->next_mb_id = vm->first_mb_id;
1841 /* Prepare the offline threshold - make sure we can add two blocks. */
1842 vm->offline_threshold = max_t(uint64_t, 2 * memory_block_size_bytes(),
1843 VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD);
1845 dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
1846 dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
1847 dev_info(&vm->vdev->dev, "device block size: 0x%llx",
1848 (unsigned long long)vm->device_block_size);
1849 dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
1850 memory_block_size_bytes());
1851 dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
1852 (unsigned long long)vm->subblock_size);
1853 if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA))
1854 dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
1859 static int virtio_mem_create_resource(struct virtio_mem *vm)
1862 * When force-unloading the driver and removing the device, we
1863 * could have a garbage pointer. Duplicate the string.
1865 const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
1870 vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
1871 name, IORESOURCE_SYSTEM_RAM);
1872 if (!vm->parent_resource) {
1874 dev_warn(&vm->vdev->dev, "could not reserve device region\n");
1875 dev_info(&vm->vdev->dev,
1876 "reloading the driver is not supported\n");
1880 /* The memory is not actually busy - make add_memory() work. */
1881 vm->parent_resource->flags &= ~IORESOURCE_BUSY;
1885 static void virtio_mem_delete_resource(struct virtio_mem *vm)
1889 if (!vm->parent_resource)
1892 name = vm->parent_resource->name;
1893 release_resource(vm->parent_resource);
1894 kfree(vm->parent_resource);
1896 vm->parent_resource = NULL;
1899 static int virtio_mem_range_has_system_ram(struct resource *res, void *arg)
1904 static bool virtio_mem_has_memory_added(struct virtio_mem *vm)
1906 const unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
1908 return walk_iomem_res_desc(IORES_DESC_NONE, flags, vm->addr,
1909 vm->addr + vm->region_size, NULL,
1910 virtio_mem_range_has_system_ram) == 1;
1913 static int virtio_mem_probe(struct virtio_device *vdev)
1915 struct virtio_mem *vm;
1918 BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
1919 BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
1921 vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1925 init_waitqueue_head(&vm->host_resp);
1927 INIT_WORK(&vm->wq, virtio_mem_run_wq);
1928 mutex_init(&vm->hotplug_mutex);
1929 INIT_LIST_HEAD(&vm->next);
1930 spin_lock_init(&vm->removal_lock);
1931 hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1932 vm->retry_timer.function = virtio_mem_timer_expired;
1933 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
1935 /* register the virtqueue */
1936 rc = virtio_mem_init_vq(vm);
1940 /* initialize the device by querying the config */
1941 rc = virtio_mem_init(vm);
1945 /* create the parent resource for all memory */
1946 rc = virtio_mem_create_resource(vm);
1951 * If we still have memory plugged, we have to unplug all memory first.
1952 * Registering our parent resource makes sure that this memory isn't
1953 * actually in use (e.g., trying to reload the driver).
1955 if (vm->plugged_size) {
1956 vm->unplug_all_required = 1;
1957 dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
1960 /* register callbacks */
1961 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
1962 rc = register_memory_notifier(&vm->memory_notifier);
1964 goto out_del_resource;
1965 rc = register_virtio_mem_device(vm);
1969 virtio_device_ready(vdev);
1971 /* trigger a config update to start processing the requested_size */
1972 atomic_set(&vm->config_changed, 1);
1973 queue_work(system_freezable_wq, &vm->wq);
1977 unregister_memory_notifier(&vm->memory_notifier);
1979 virtio_mem_delete_resource(vm);
1981 vdev->config->del_vqs(vdev);
1989 static void virtio_mem_remove(struct virtio_device *vdev)
1991 struct virtio_mem *vm = vdev->priv;
1992 unsigned long mb_id;
1996 * Make sure the workqueue won't be triggered anymore and no memory
1997 * blocks can be onlined/offlined until we're finished here.
1999 mutex_lock(&vm->hotplug_mutex);
2000 spin_lock_irq(&vm->removal_lock);
2001 vm->removing = true;
2002 spin_unlock_irq(&vm->removal_lock);
2003 mutex_unlock(&vm->hotplug_mutex);
2005 /* wait until the workqueue stopped */
2006 cancel_work_sync(&vm->wq);
2007 hrtimer_cancel(&vm->retry_timer);
2010 * After we unregistered our callbacks, user space can online partially
2011 * plugged offline blocks. Make sure to remove them.
2013 virtio_mem_sbm_for_each_mb(vm, mb_id,
2014 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) {
2015 rc = virtio_mem_mb_remove(vm, mb_id);
2017 virtio_mem_sbm_set_mb_state(vm, mb_id,
2018 VIRTIO_MEM_SBM_MB_UNUSED);
2021 * After we unregistered our callbacks, user space can no longer
2022 * offline partially plugged online memory blocks. No need to worry
2026 /* unregister callbacks */
2027 unregister_virtio_mem_device(vm);
2028 unregister_memory_notifier(&vm->memory_notifier);
2031 * There is no way we could reliably remove all memory we have added to
2032 * the system. And there is no way to stop the driver/device from going
2033 * away. Warn at least.
2035 if (virtio_mem_has_memory_added(vm)) {
2036 dev_warn(&vdev->dev, "device still has system memory added\n");
2038 virtio_mem_delete_resource(vm);
2039 kfree_const(vm->resource_name);
2042 /* remove all tracking data - no locking needed */
2043 vfree(vm->sbm.mb_states);
2044 vfree(vm->sbm.sb_states);
2046 /* reset the device and cleanup the queues */
2047 vdev->config->reset(vdev);
2048 vdev->config->del_vqs(vdev);
2054 static void virtio_mem_config_changed(struct virtio_device *vdev)
2056 struct virtio_mem *vm = vdev->priv;
2058 atomic_set(&vm->config_changed, 1);
2059 virtio_mem_retry(vm);
2062 #ifdef CONFIG_PM_SLEEP
2063 static int virtio_mem_freeze(struct virtio_device *vdev)
2066 * When restarting the VM, all memory is usually unplugged. Don't
2067 * allow to suspend/hibernate.
2069 dev_err(&vdev->dev, "save/restore not supported.\n");
2073 static int virtio_mem_restore(struct virtio_device *vdev)
2079 static unsigned int virtio_mem_features[] = {
2080 #if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
2081 VIRTIO_MEM_F_ACPI_PXM,
2085 static const struct virtio_device_id virtio_mem_id_table[] = {
2086 { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
2090 static struct virtio_driver virtio_mem_driver = {
2091 .feature_table = virtio_mem_features,
2092 .feature_table_size = ARRAY_SIZE(virtio_mem_features),
2093 .driver.name = KBUILD_MODNAME,
2094 .driver.owner = THIS_MODULE,
2095 .id_table = virtio_mem_id_table,
2096 .probe = virtio_mem_probe,
2097 .remove = virtio_mem_remove,
2098 .config_changed = virtio_mem_config_changed,
2099 #ifdef CONFIG_PM_SLEEP
2100 .freeze = virtio_mem_freeze,
2101 .restore = virtio_mem_restore,
2105 module_virtio_driver(virtio_mem_driver);
2106 MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
2107 MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
2108 MODULE_DESCRIPTION("Virtio-mem driver");
2109 MODULE_LICENSE("GPL");