1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Virtio-mem device driver.
5 * Copyright Red Hat, Inc. 2020
7 * Author(s): David Hildenbrand <david@redhat.com>
10 #include <linux/virtio.h>
11 #include <linux/virtio_mem.h>
12 #include <linux/workqueue.h>
13 #include <linux/slab.h>
14 #include <linux/module.h>
16 #include <linux/memory_hotplug.h>
17 #include <linux/memory.h>
18 #include <linux/hrtimer.h>
19 #include <linux/crash_dump.h>
20 #include <linux/mutex.h>
21 #include <linux/bitmap.h>
22 #include <linux/lockdep.h>
24 #include <acpi/acpi_numa.h>
26 static bool unplug_online = true;
27 module_param(unplug_online, bool, 0644);
28 MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
30 enum virtio_mem_mb_state {
31 /* Unplugged, not added to Linux. Can be reused later. */
32 VIRTIO_MEM_MB_STATE_UNUSED = 0,
33 /* (Partially) plugged, not added to Linux. Error on add_memory(). */
34 VIRTIO_MEM_MB_STATE_PLUGGED,
35 /* Fully plugged, fully added to Linux, offline. */
36 VIRTIO_MEM_MB_STATE_OFFLINE,
37 /* Partially plugged, fully added to Linux, offline. */
38 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL,
39 /* Fully plugged, fully added to Linux, online. */
40 VIRTIO_MEM_MB_STATE_ONLINE,
41 /* Partially plugged, fully added to Linux, online. */
42 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL,
43 VIRTIO_MEM_MB_STATE_COUNT
47 struct virtio_device *vdev;
49 /* We might first have to unplug all memory when starting up. */
50 bool unplug_all_required;
52 /* Workqueue that processes the plug/unplug requests. */
53 struct work_struct wq;
54 atomic_t config_changed;
56 /* Virtqueue for guest->host requests. */
59 /* Wait for a host response to a guest request. */
60 wait_queue_head_t host_resp;
62 /* Space for one guest request and the host response. */
63 struct virtio_mem_req req;
64 struct virtio_mem_resp resp;
66 /* The current size of the device. */
67 uint64_t plugged_size;
68 /* The requested size of the device. */
69 uint64_t requested_size;
71 /* The device block size (for communicating with the device). */
72 uint64_t device_block_size;
73 /* The determined node id for all memory of the device. */
75 /* Physical start address of the memory region. */
77 /* Maximum region size in bytes. */
80 /* The subblock size. */
81 uint64_t subblock_size;
82 /* The number of subblocks per memory block. */
83 uint32_t nb_sb_per_mb;
85 /* Id of the first memory block of this device. */
86 unsigned long first_mb_id;
87 /* Id of the last memory block of this device. */
88 unsigned long last_mb_id;
89 /* Id of the last usable memory block of this device. */
90 unsigned long last_usable_mb_id;
91 /* Id of the next memory bock to prepare when needed. */
92 unsigned long next_mb_id;
94 /* The parent resource for all memory added via this device. */
95 struct resource *parent_resource;
97 * Copy of "System RAM (virtio_mem)" to be used for
98 * add_memory_driver_managed().
100 const char *resource_name;
102 /* Summary of all memory block states. */
103 unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT];
104 #define VIRTIO_MEM_NB_OFFLINE_THRESHOLD 10
107 * One byte state per memory block.
109 * Allocated via vmalloc(). When preparing new blocks, resized
110 * (alloc+copy+free) when needed (crossing pages with the next mb).
111 * (when crossing pages).
113 * With 128MB memory blocks, we have states for 512GB of memory in one
119 * $nb_sb_per_mb bit per memory block. Handled similar to mb_state.
121 * With 4MB subblocks, we manage 128GB of memory in one page.
123 unsigned long *sb_bitmap;
126 * Mutex that protects the nb_mb_state, mb_state, and sb_bitmap.
128 * When this lock is held the pointers can't change, ONLINE and
129 * OFFLINE blocks can't change the state and no subblocks will get
132 struct mutex hotplug_mutex;
135 /* An error occurred we cannot handle - stop processing requests. */
138 /* The driver is being removed. */
139 spinlock_t removal_lock;
142 /* Timer for retrying to plug/unplug memory. */
143 struct hrtimer retry_timer;
144 unsigned int retry_timer_ms;
145 #define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000
146 #define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000
148 /* Memory notifier (online/offline events). */
149 struct notifier_block memory_notifier;
151 /* Next device in the list of virtio-mem devices. */
152 struct list_head next;
156 * We have to share a single online_page callback among all virtio-mem
157 * devices. We use RCU to iterate the list in the callback.
159 static DEFINE_MUTEX(virtio_mem_mutex);
160 static LIST_HEAD(virtio_mem_devices);
162 static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
165 * Register a virtio-mem device so it will be considered for the online_page
168 static int register_virtio_mem_device(struct virtio_mem *vm)
172 /* First device registers the callback. */
173 mutex_lock(&virtio_mem_mutex);
174 if (list_empty(&virtio_mem_devices))
175 rc = set_online_page_callback(&virtio_mem_online_page_cb);
177 list_add_rcu(&vm->next, &virtio_mem_devices);
178 mutex_unlock(&virtio_mem_mutex);
184 * Unregister a virtio-mem device so it will no longer be considered for the
185 * online_page callback.
187 static void unregister_virtio_mem_device(struct virtio_mem *vm)
189 /* Last device unregisters the callback. */
190 mutex_lock(&virtio_mem_mutex);
191 list_del_rcu(&vm->next);
192 if (list_empty(&virtio_mem_devices))
193 restore_online_page_callback(&virtio_mem_online_page_cb);
194 mutex_unlock(&virtio_mem_mutex);
200 * Calculate the memory block id of a given address.
202 static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
204 return addr / memory_block_size_bytes();
208 * Calculate the physical start address of a given memory block id.
210 static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
212 return mb_id * memory_block_size_bytes();
216 * Calculate the subblock id of a given address.
218 static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
221 const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
222 const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
224 return (addr - mb_addr) / vm->subblock_size;
228 * Set the state of a memory block, taking care of the state counter.
230 static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id,
231 enum virtio_mem_mb_state state)
233 const unsigned long idx = mb_id - vm->first_mb_id;
234 enum virtio_mem_mb_state old_state;
236 old_state = vm->mb_state[idx];
237 vm->mb_state[idx] = state;
239 BUG_ON(vm->nb_mb_state[old_state] == 0);
240 vm->nb_mb_state[old_state]--;
241 vm->nb_mb_state[state]++;
245 * Get the state of a memory block.
247 static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm,
250 const unsigned long idx = mb_id - vm->first_mb_id;
252 return vm->mb_state[idx];
256 * Prepare the state array for the next memory block.
258 static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm)
260 int old_pages = PFN_UP(vm->next_mb_id - vm->first_mb_id);
261 int new_pages = PFN_UP(vm->next_mb_id - vm->first_mb_id + 1);
262 uint8_t *new_mb_state;
264 if (vm->mb_state && old_pages == new_pages)
267 new_mb_state = vzalloc(new_pages * PAGE_SIZE);
271 mutex_lock(&vm->hotplug_mutex);
273 memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE);
275 vm->mb_state = new_mb_state;
276 mutex_unlock(&vm->hotplug_mutex);
281 #define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \
282 for (_mb_id = _vm->first_mb_id; \
283 _mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \
285 if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
287 #define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \
288 for (_mb_id = _vm->next_mb_id - 1; \
289 _mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \
291 if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
294 * Mark all selected subblocks plugged.
296 * Will not modify the state of the memory block.
298 static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,
299 unsigned long mb_id, int sb_id,
302 const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
304 __bitmap_set(vm->sb_bitmap, bit, count);
308 * Mark all selected subblocks unplugged.
310 * Will not modify the state of the memory block.
312 static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,
313 unsigned long mb_id, int sb_id,
316 const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
318 __bitmap_clear(vm->sb_bitmap, bit, count);
322 * Test if all selected subblocks are plugged.
324 static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,
325 unsigned long mb_id, int sb_id,
328 const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
331 return test_bit(bit, vm->sb_bitmap);
333 /* TODO: Helper similar to bitmap_set() */
334 return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=
339 * Test if all selected subblocks are unplugged.
341 static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,
342 unsigned long mb_id, int sb_id,
345 const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
347 /* TODO: Helper similar to bitmap_set() */
348 return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;
352 * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
355 static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,
358 const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb;
360 return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -
365 * Prepare the subblock bitmap for the next memory block.
367 static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)
369 const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
370 const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
371 const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
372 int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
373 int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
374 unsigned long *new_sb_bitmap, *old_sb_bitmap;
376 if (vm->sb_bitmap && old_pages == new_pages)
379 new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);
383 mutex_lock(&vm->hotplug_mutex);
385 memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);
387 old_sb_bitmap = vm->sb_bitmap;
388 vm->sb_bitmap = new_sb_bitmap;
389 mutex_unlock(&vm->hotplug_mutex);
391 vfree(old_sb_bitmap);
396 * Try to add a memory block to Linux. This will usually only fail
399 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
402 * Will not modify the state of the memory block.
404 static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
406 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
409 * When force-unloading the driver and we still have memory added to
410 * Linux, the resource name has to stay.
412 if (!vm->resource_name) {
413 vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
415 if (!vm->resource_name)
419 dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
420 return add_memory_driver_managed(vm->nid, addr,
421 memory_block_size_bytes(),
423 MEMHP_MERGE_RESOURCE);
427 * Try to remove a memory block from Linux. Will only fail if the memory block
430 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
433 * Will not modify the state of the memory block.
435 static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
437 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
439 dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
440 return remove_memory(vm->nid, addr, memory_block_size_bytes());
444 * Try to offline and remove a memory block from Linux.
446 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
449 * Will not modify the state of the memory block.
451 static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
454 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
456 dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
458 return offline_and_remove_memory(vm->nid, addr,
459 memory_block_size_bytes());
463 * Trigger the workqueue so the device can perform its magic.
465 static void virtio_mem_retry(struct virtio_mem *vm)
469 spin_lock_irqsave(&vm->removal_lock, flags);
471 queue_work(system_freezable_wq, &vm->wq);
472 spin_unlock_irqrestore(&vm->removal_lock, flags);
475 static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
477 int node = NUMA_NO_NODE;
479 #if defined(CONFIG_ACPI_NUMA)
480 if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
481 node = pxm_to_node(node_id);
487 * Test if a virtio-mem device overlaps with the given range. Can be called
488 * from (notifier) callbacks lockless.
490 static bool virtio_mem_overlaps_range(struct virtio_mem *vm,
491 unsigned long start, unsigned long size)
493 unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id);
494 unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) +
495 memory_block_size_bytes();
497 return start < dev_end && dev_start < start + size;
501 * Test if a virtio-mem device owns a memory block. Can be called from
502 * (notifier) callbacks lockless.
504 static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id)
506 return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id;
509 static int virtio_mem_notify_going_online(struct virtio_mem *vm,
512 switch (virtio_mem_mb_get_state(vm, mb_id)) {
513 case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
514 case VIRTIO_MEM_MB_STATE_OFFLINE:
519 dev_warn_ratelimited(&vm->vdev->dev,
520 "memory block onlining denied\n");
524 static void virtio_mem_notify_offline(struct virtio_mem *vm,
527 switch (virtio_mem_mb_get_state(vm, mb_id)) {
528 case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL:
529 virtio_mem_mb_set_state(vm, mb_id,
530 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
532 case VIRTIO_MEM_MB_STATE_ONLINE:
533 virtio_mem_mb_set_state(vm, mb_id,
534 VIRTIO_MEM_MB_STATE_OFFLINE);
542 * Trigger the workqueue, maybe we can now unplug memory. Also,
543 * when we offline and remove a memory block, this will re-trigger
544 * us immediately - which is often nice because the removal of
545 * the memory block (e.g., memmap) might have freed up memory
546 * on other memory blocks we manage.
548 virtio_mem_retry(vm);
551 static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id)
553 unsigned long nb_offline;
555 switch (virtio_mem_mb_get_state(vm, mb_id)) {
556 case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
557 virtio_mem_mb_set_state(vm, mb_id,
558 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
560 case VIRTIO_MEM_MB_STATE_OFFLINE:
561 virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_ONLINE);
567 nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
568 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
570 /* see if we can add new blocks now that we onlined one block */
571 if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1)
572 virtio_mem_retry(vm);
575 static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
578 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
583 for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
584 if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
587 * Drop our reference to the pages so the memory can get
588 * offlined and add the unplugged pages to the managed
589 * page counters (so offlining code can correctly subtract
592 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
593 sb_id * vm->subblock_size);
594 adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
595 for (i = 0; i < nr_pages; i++) {
596 page = pfn_to_page(pfn + i);
597 if (WARN_ON(!page_ref_dec_and_test(page)))
598 dump_page(page, "unplugged page referenced");
603 static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
606 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
610 for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
611 if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
614 * Get the reference we dropped when going offline and
615 * subtract the unplugged pages from the managed page
618 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
619 sb_id * vm->subblock_size);
620 adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
621 for (i = 0; i < nr_pages; i++)
622 page_ref_inc(pfn_to_page(pfn + i));
627 * This callback will either be called synchronously from add_memory() or
628 * asynchronously (e.g., triggered via user space). We have to be careful
629 * with locking when calling add_memory().
631 static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
632 unsigned long action, void *arg)
634 struct virtio_mem *vm = container_of(nb, struct virtio_mem,
636 struct memory_notify *mhp = arg;
637 const unsigned long start = PFN_PHYS(mhp->start_pfn);
638 const unsigned long size = PFN_PHYS(mhp->nr_pages);
639 const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
642 if (!virtio_mem_overlaps_range(vm, start, size))
646 * Memory is onlined/offlined in memory block granularity. We cannot
647 * cross virtio-mem device boundaries and memory block boundaries. Bail
648 * out if this ever changes.
650 if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
651 !IS_ALIGNED(start, memory_block_size_bytes())))
655 * Avoid circular locking lockdep warnings. We lock the mutex
656 * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
657 * blocking_notifier_call_chain() has it's own lock, which gets unlocked
658 * between both notifier calls and will bail out. False positive.
663 case MEM_GOING_OFFLINE:
664 mutex_lock(&vm->hotplug_mutex);
666 rc = notifier_from_errno(-EBUSY);
667 mutex_unlock(&vm->hotplug_mutex);
670 vm->hotplug_active = true;
671 virtio_mem_notify_going_offline(vm, mb_id);
673 case MEM_GOING_ONLINE:
674 mutex_lock(&vm->hotplug_mutex);
676 rc = notifier_from_errno(-EBUSY);
677 mutex_unlock(&vm->hotplug_mutex);
680 vm->hotplug_active = true;
681 rc = virtio_mem_notify_going_online(vm, mb_id);
684 virtio_mem_notify_offline(vm, mb_id);
685 vm->hotplug_active = false;
686 mutex_unlock(&vm->hotplug_mutex);
689 virtio_mem_notify_online(vm, mb_id);
690 vm->hotplug_active = false;
691 mutex_unlock(&vm->hotplug_mutex);
693 case MEM_CANCEL_OFFLINE:
694 if (!vm->hotplug_active)
696 virtio_mem_notify_cancel_offline(vm, mb_id);
697 vm->hotplug_active = false;
698 mutex_unlock(&vm->hotplug_mutex);
700 case MEM_CANCEL_ONLINE:
701 if (!vm->hotplug_active)
703 vm->hotplug_active = false;
704 mutex_unlock(&vm->hotplug_mutex);
716 * Set a range of pages PG_offline. Remember pages that were never onlined
717 * (via generic_online_page()) using PageDirty().
719 static void virtio_mem_set_fake_offline(unsigned long pfn,
720 unsigned int nr_pages, bool onlined)
722 for (; nr_pages--; pfn++) {
723 struct page *page = pfn_to_page(pfn);
725 __SetPageOffline(page);
728 /* FIXME: remove after cleanups */
729 ClearPageReserved(page);
735 * Clear PG_offline from a range of pages. If the pages were never onlined,
736 * (via generic_online_page()), clear PageDirty().
738 static void virtio_mem_clear_fake_offline(unsigned long pfn,
739 unsigned int nr_pages, bool onlined)
741 for (; nr_pages--; pfn++) {
742 struct page *page = pfn_to_page(pfn);
744 __ClearPageOffline(page);
746 ClearPageDirty(page);
751 * Release a range of fake-offline pages to the buddy, effectively
752 * fake-onlining them.
754 static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages)
756 const unsigned long max_nr_pages = MAX_ORDER_NR_PAGES;
760 * We are always called at least with MAX_ORDER_NR_PAGES
761 * granularity/alignment (e.g., the way subblocks work). All pages
762 * inside such a block are alike.
764 for (i = 0; i < nr_pages; i += max_nr_pages) {
765 struct page *page = pfn_to_page(pfn + i);
768 * If the page is PageDirty(), it was kept fake-offline when
769 * onlining the memory block. Otherwise, it was allocated
770 * using alloc_contig_range(). All pages in a subblock are
773 if (PageDirty(page)) {
774 virtio_mem_clear_fake_offline(pfn + i, max_nr_pages,
776 generic_online_page(page, MAX_ORDER - 1);
778 virtio_mem_clear_fake_offline(pfn + i, max_nr_pages,
780 free_contig_range(pfn + i, max_nr_pages);
781 adjust_managed_page_count(page, max_nr_pages);
786 static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
788 const unsigned long addr = page_to_phys(page);
789 const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
790 struct virtio_mem *vm;
794 * We exploit here that subblocks have at least MAX_ORDER_NR_PAGES.
795 * size/alignment and that this callback is is called with such a
796 * size/alignment. So we cannot cross subblocks and therefore
797 * also not memory blocks.
800 list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
801 if (!virtio_mem_owned_mb(vm, mb_id))
804 sb_id = virtio_mem_phys_to_sb_id(vm, addr);
806 * If plugged, online the pages, otherwise, set them fake
807 * offline (PageOffline).
809 if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
810 generic_online_page(page, order);
812 virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
819 /* not virtio-mem memory, but e.g., a DIMM. online it */
820 generic_online_page(page, order);
823 static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
824 const struct virtio_mem_req *req)
826 struct scatterlist *sgs[2], sg_req, sg_resp;
830 /* don't use the request residing on the stack (vaddr) */
833 /* out: buffer for request */
834 sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
837 /* in: buffer for response */
838 sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
841 rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
845 virtqueue_kick(vm->vq);
847 /* wait for a response */
848 wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
850 return virtio16_to_cpu(vm->vdev, vm->resp.type);
853 static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
856 const uint64_t nb_vm_blocks = size / vm->device_block_size;
857 const struct virtio_mem_req req = {
858 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
859 .u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
860 .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
863 if (atomic_read(&vm->config_changed))
866 switch (virtio_mem_send_request(vm, &req)) {
867 case VIRTIO_MEM_RESP_ACK:
868 vm->plugged_size += size;
870 case VIRTIO_MEM_RESP_NACK:
872 case VIRTIO_MEM_RESP_BUSY:
874 case VIRTIO_MEM_RESP_ERROR:
881 static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
884 const uint64_t nb_vm_blocks = size / vm->device_block_size;
885 const struct virtio_mem_req req = {
886 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
887 .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
888 .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
891 if (atomic_read(&vm->config_changed))
894 switch (virtio_mem_send_request(vm, &req)) {
895 case VIRTIO_MEM_RESP_ACK:
896 vm->plugged_size -= size;
898 case VIRTIO_MEM_RESP_BUSY:
900 case VIRTIO_MEM_RESP_ERROR:
907 static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
909 const struct virtio_mem_req req = {
910 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
913 switch (virtio_mem_send_request(vm, &req)) {
914 case VIRTIO_MEM_RESP_ACK:
915 vm->unplug_all_required = false;
916 vm->plugged_size = 0;
917 /* usable region might have shrunk */
918 atomic_set(&vm->config_changed, 1);
920 case VIRTIO_MEM_RESP_BUSY:
928 * Plug selected subblocks. Updates the plugged state, but not the state
929 * of the memory block.
931 static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
932 int sb_id, int count)
934 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
935 sb_id * vm->subblock_size;
936 const uint64_t size = count * vm->subblock_size;
939 dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id,
940 sb_id, sb_id + count - 1);
942 rc = virtio_mem_send_plug_request(vm, addr, size);
944 virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);
949 * Unplug selected subblocks. Updates the plugged state, but not the state
950 * of the memory block.
952 static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
953 int sb_id, int count)
955 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
956 sb_id * vm->subblock_size;
957 const uint64_t size = count * vm->subblock_size;
960 dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n",
961 mb_id, sb_id, sb_id + count - 1);
963 rc = virtio_mem_send_unplug_request(vm, addr, size);
965 virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);
970 * Unplug the desired number of plugged subblocks of a offline or not-added
971 * memory block. Will fail if any subblock cannot get unplugged (instead of
974 * Will not modify the state of the memory block.
976 * Note: can fail after some subblocks were unplugged.
978 static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
979 unsigned long mb_id, uint64_t *nb_sb)
984 sb_id = vm->nb_sb_per_mb - 1;
986 /* Find the next candidate subblock */
988 virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))
992 /* Try to unplug multiple subblocks at a time */
994 while (count < *nb_sb && sb_id > 0 &&
995 virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
1000 rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1011 * Unplug all plugged subblocks of an offline or not-added memory block.
1013 * Will not modify the state of the memory block.
1015 * Note: can fail after some subblocks were unplugged.
1017 static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
1019 uint64_t nb_sb = vm->nb_sb_per_mb;
1021 return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
1025 * Prepare tracking data for the next memory block.
1027 static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
1028 unsigned long *mb_id)
1032 if (vm->next_mb_id > vm->last_usable_mb_id)
1035 /* Resize the state array if required. */
1036 rc = virtio_mem_mb_state_prepare_next_mb(vm);
1040 /* Resize the subblock bitmap if required. */
1041 rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);
1045 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++;
1046 *mb_id = vm->next_mb_id++;
1051 * Don't add too many blocks that are not onlined yet to avoid running OOM.
1053 static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm)
1055 unsigned long nb_offline;
1057 nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
1058 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
1059 return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD;
1063 * Try to plug the desired number of subblocks and add the memory block
1066 * Will modify the state of the memory block.
1068 static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
1069 unsigned long mb_id,
1072 const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
1075 if (WARN_ON_ONCE(!count))
1079 * Plug the requested number of subblocks before adding it to linux,
1080 * so that onlining will directly online all plugged subblocks.
1082 rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
1087 * Mark the block properly offline before adding it to Linux,
1088 * so the memory notifiers will find the block in the right state.
1090 if (count == vm->nb_sb_per_mb)
1091 virtio_mem_mb_set_state(vm, mb_id,
1092 VIRTIO_MEM_MB_STATE_OFFLINE);
1094 virtio_mem_mb_set_state(vm, mb_id,
1095 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
1097 /* Add the memory block to linux - if that fails, try to unplug. */
1098 rc = virtio_mem_mb_add(vm, mb_id);
1100 enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED;
1102 dev_err(&vm->vdev->dev,
1103 "adding memory block %lu failed with %d\n", mb_id, rc);
1104 rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count);
1107 * TODO: Linux MM does not properly clean up yet in all cases
1108 * where adding of memory failed - especially on -ENOMEM.
1111 new_state = VIRTIO_MEM_MB_STATE_PLUGGED;
1112 virtio_mem_mb_set_state(vm, mb_id, new_state);
1121 * Try to plug the desired number of subblocks of a memory block that
1122 * is already added to Linux.
1124 * Will modify the state of the memory block.
1126 * Note: Can fail after some subblocks were successfully plugged.
1128 static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
1129 uint64_t *nb_sb, bool online)
1131 unsigned long pfn, nr_pages;
1135 if (WARN_ON_ONCE(!*nb_sb))
1139 sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);
1140 if (sb_id >= vm->nb_sb_per_mb)
1143 while (count < *nb_sb &&
1144 sb_id + count < vm->nb_sb_per_mb &&
1145 !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,
1149 rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
1156 /* fake-online the pages if the memory block is online */
1157 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1158 sb_id * vm->subblock_size);
1159 nr_pages = PFN_DOWN(count * vm->subblock_size);
1160 virtio_mem_fake_online(pfn, nr_pages);
1163 if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1165 virtio_mem_mb_set_state(vm, mb_id,
1166 VIRTIO_MEM_MB_STATE_ONLINE);
1168 virtio_mem_mb_set_state(vm, mb_id,
1169 VIRTIO_MEM_MB_STATE_OFFLINE);
1176 * Try to plug the requested amount of memory.
1178 static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
1180 uint64_t nb_sb = diff / vm->subblock_size;
1181 unsigned long mb_id;
1187 /* Don't race with onlining/offlining */
1188 mutex_lock(&vm->hotplug_mutex);
1190 /* Try to plug subblocks of partially plugged online blocks. */
1191 virtio_mem_for_each_mb_state(vm, mb_id,
1192 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
1193 rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
1199 /* Try to plug subblocks of partially plugged offline blocks. */
1200 virtio_mem_for_each_mb_state(vm, mb_id,
1201 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1202 rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
1209 * We won't be working on online/offline memory blocks from this point,
1210 * so we can't race with memory onlining/offlining. Drop the mutex.
1212 mutex_unlock(&vm->hotplug_mutex);
1214 /* Try to plug and add unused blocks */
1215 virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) {
1216 if (virtio_mem_too_many_mb_offline(vm))
1219 rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1225 /* Try to prepare, plug and add new blocks */
1227 if (virtio_mem_too_many_mb_offline(vm))
1230 rc = virtio_mem_prepare_next_mb(vm, &mb_id);
1233 rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1241 mutex_unlock(&vm->hotplug_mutex);
1246 * Unplug the desired number of plugged subblocks of an offline memory block.
1247 * Will fail if any subblock cannot get unplugged (instead of skipping it).
1249 * Will modify the state of the memory block. Might temporarily drop the
1252 * Note: Can fail after some subblocks were successfully unplugged.
1254 static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
1255 unsigned long mb_id,
1260 rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
1262 /* some subblocks might have been unplugged even on failure */
1263 if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
1264 virtio_mem_mb_set_state(vm, mb_id,
1265 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
1269 if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1271 * Remove the block from Linux - this should never fail.
1272 * Hinder the block from getting onlined by marking it
1273 * unplugged. Temporarily drop the mutex, so
1274 * any pending GOING_ONLINE requests can be serviced/rejected.
1276 virtio_mem_mb_set_state(vm, mb_id,
1277 VIRTIO_MEM_MB_STATE_UNUSED);
1279 mutex_unlock(&vm->hotplug_mutex);
1280 rc = virtio_mem_mb_remove(vm, mb_id);
1282 mutex_lock(&vm->hotplug_mutex);
1288 * Unplug the given plugged subblocks of an online memory block.
1290 * Will modify the state of the memory block.
1292 static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
1293 unsigned long mb_id, int sb_id,
1296 const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
1297 unsigned long start_pfn;
1300 start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1301 sb_id * vm->subblock_size);
1302 rc = alloc_contig_range(start_pfn, start_pfn + nr_pages,
1303 MIGRATE_MOVABLE, GFP_KERNEL);
1305 /* whoops, out of memory */
1310 /* Mark it as fake-offline before unplugging it */
1311 virtio_mem_set_fake_offline(start_pfn, nr_pages, true);
1312 adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
1314 /* Try to unplug the allocated memory */
1315 rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1317 /* Return the memory to the buddy. */
1318 virtio_mem_fake_online(start_pfn, nr_pages);
1322 virtio_mem_mb_set_state(vm, mb_id,
1323 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
1328 * Unplug the desired number of plugged subblocks of an online memory block.
1329 * Will skip subblock that are busy.
1331 * Will modify the state of the memory block. Might temporarily drop the
1334 * Note: Can fail after some subblocks were successfully unplugged. Can
1335 * return 0 even if subblocks were busy and could not get unplugged.
1337 static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
1338 unsigned long mb_id,
1343 /* If possible, try to unplug the complete block in one shot. */
1344 if (*nb_sb >= vm->nb_sb_per_mb &&
1345 virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1346 rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
1349 *nb_sb -= vm->nb_sb_per_mb;
1351 } else if (rc != -EBUSY)
1355 /* Fallback to single subblocks. */
1356 for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
1357 /* Find the next candidate subblock */
1358 while (sb_id >= 0 &&
1359 !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
1364 rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
1374 * Once all subblocks of a memory block were unplugged, offline and
1375 * remove it. This will usually not fail, as no memory is in use
1376 * anymore - however some other notifiers might NACK the request.
1378 if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1379 mutex_unlock(&vm->hotplug_mutex);
1380 rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
1381 mutex_lock(&vm->hotplug_mutex);
1383 virtio_mem_mb_set_state(vm, mb_id,
1384 VIRTIO_MEM_MB_STATE_UNUSED);
1391 * Try to unplug the requested amount of memory.
1393 static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
1395 uint64_t nb_sb = diff / vm->subblock_size;
1396 unsigned long mb_id;
1403 * We'll drop the mutex a couple of times when it is safe to do so.
1404 * This might result in some blocks switching the state (online/offline)
1405 * and we could miss them in this run - we will retry again later.
1407 mutex_lock(&vm->hotplug_mutex);
1409 /* Try to unplug subblocks of partially plugged offline blocks. */
1410 virtio_mem_for_each_mb_state_rev(vm, mb_id,
1411 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1412 rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1419 /* Try to unplug subblocks of plugged offline blocks. */
1420 virtio_mem_for_each_mb_state_rev(vm, mb_id,
1421 VIRTIO_MEM_MB_STATE_OFFLINE) {
1422 rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1429 if (!unplug_online) {
1430 mutex_unlock(&vm->hotplug_mutex);
1434 /* Try to unplug subblocks of partially plugged online blocks. */
1435 virtio_mem_for_each_mb_state_rev(vm, mb_id,
1436 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
1437 rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1441 mutex_unlock(&vm->hotplug_mutex);
1443 mutex_lock(&vm->hotplug_mutex);
1446 /* Try to unplug subblocks of plugged online blocks. */
1447 virtio_mem_for_each_mb_state_rev(vm, mb_id,
1448 VIRTIO_MEM_MB_STATE_ONLINE) {
1449 rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1453 mutex_unlock(&vm->hotplug_mutex);
1455 mutex_lock(&vm->hotplug_mutex);
1458 mutex_unlock(&vm->hotplug_mutex);
1459 return nb_sb ? -EBUSY : 0;
1461 mutex_unlock(&vm->hotplug_mutex);
1466 * Try to unplug all blocks that couldn't be unplugged before, for example,
1467 * because the hypervisor was busy.
1469 static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
1471 unsigned long mb_id;
1474 virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) {
1475 rc = virtio_mem_mb_unplug(vm, mb_id);
1478 virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
1485 * Update all parts of the config that could have changed.
1487 static void virtio_mem_refresh_config(struct virtio_mem *vm)
1489 const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1490 uint64_t new_plugged_size, usable_region_size, end_addr;
1492 /* the plugged_size is just a reflection of what _we_ did previously */
1493 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1495 if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
1496 vm->plugged_size = new_plugged_size;
1498 /* calculate the last usable memory block id */
1499 virtio_cread_le(vm->vdev, struct virtio_mem_config,
1500 usable_region_size, &usable_region_size);
1501 end_addr = vm->addr + usable_region_size;
1502 end_addr = min(end_addr, phys_limit);
1503 vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
1505 /* see if there is a request to change the size */
1506 virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
1507 &vm->requested_size);
1509 dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
1510 dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
1514 * Workqueue function for handling plug/unplug requests and config updates.
1516 static void virtio_mem_run_wq(struct work_struct *work)
1518 struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
1522 hrtimer_cancel(&vm->retry_timer);
1530 /* Make sure we start with a clean state if there are leftovers. */
1531 if (unlikely(vm->unplug_all_required))
1532 rc = virtio_mem_send_unplug_all_request(vm);
1534 if (atomic_read(&vm->config_changed)) {
1535 atomic_set(&vm->config_changed, 0);
1536 virtio_mem_refresh_config(vm);
1539 /* Unplug any leftovers from previous runs */
1541 rc = virtio_mem_unplug_pending_mb(vm);
1543 if (!rc && vm->requested_size != vm->plugged_size) {
1544 if (vm->requested_size > vm->plugged_size) {
1545 diff = vm->requested_size - vm->plugged_size;
1546 rc = virtio_mem_plug_request(vm, diff);
1548 diff = vm->plugged_size - vm->requested_size;
1549 rc = virtio_mem_unplug_request(vm, diff);
1555 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
1559 * We cannot add any more memory (alignment, physical limit)
1560 * or we have too many offline memory blocks.
1565 * The hypervisor cannot process our request right now
1566 * (e.g., out of memory, migrating);
1570 * We cannot free up any memory to unplug it (all plugged memory
1574 /* Out of memory, try again later. */
1575 hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
1579 /* Retry immediately (e.g., the config changed). */
1582 /* Unknown error, mark as broken */
1583 dev_err(&vm->vdev->dev,
1584 "unknown error, marking device broken: %d\n", rc);
1589 static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
1591 struct virtio_mem *vm = container_of(timer, struct virtio_mem,
1594 virtio_mem_retry(vm);
1595 vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
1596 VIRTIO_MEM_RETRY_TIMER_MAX_MS);
1597 return HRTIMER_NORESTART;
1600 static void virtio_mem_handle_response(struct virtqueue *vq)
1602 struct virtio_mem *vm = vq->vdev->priv;
1604 wake_up(&vm->host_resp);
1607 static int virtio_mem_init_vq(struct virtio_mem *vm)
1609 struct virtqueue *vq;
1611 vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
1620 static int virtio_mem_init(struct virtio_mem *vm)
1622 const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1625 if (!vm->vdev->config->get) {
1626 dev_err(&vm->vdev->dev, "config access disabled\n");
1631 * We don't want to (un)plug or reuse any memory when in kdump. The
1632 * memory is still accessible (but not mapped).
1634 if (is_kdump_kernel()) {
1635 dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
1639 /* Fetch all properties that can't change. */
1640 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1642 virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
1643 &vm->device_block_size);
1644 virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
1646 vm->nid = virtio_mem_translate_node_id(vm, node_id);
1647 virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
1648 virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
1651 /* Determine the nid for the device based on the lowest address. */
1652 if (vm->nid == NUMA_NO_NODE)
1653 vm->nid = memory_add_physaddr_to_nid(vm->addr);
1656 * We always hotplug memory in memory block granularity. This way,
1657 * we have to wait for exactly one memory block to online.
1659 if (vm->device_block_size > memory_block_size_bytes()) {
1660 dev_err(&vm->vdev->dev,
1661 "The block size is not supported (too big).\n");
1665 /* bad device setup - warn only */
1666 if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
1667 dev_warn(&vm->vdev->dev,
1668 "The alignment of the physical start address can make some memory unusable.\n");
1669 if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
1670 dev_warn(&vm->vdev->dev,
1671 "The alignment of the physical end address can make some memory unusable.\n");
1672 if (vm->addr + vm->region_size > phys_limit)
1673 dev_warn(&vm->vdev->dev,
1674 "Some memory is not addressable. This can make some memory unusable.\n");
1677 * We want subblocks to span at least MAX_ORDER_NR_PAGES and
1678 * pageblock_nr_pages pages. This:
1679 * - Simplifies our page onlining code (virtio_mem_online_page_cb)
1680 * and fake page onlining code (virtio_mem_fake_online).
1681 * - Is required for now for alloc_contig_range() to work reliably -
1682 * it doesn't properly handle smaller granularity on ZONE_NORMAL.
1684 vm->subblock_size = max_t(uint64_t, MAX_ORDER_NR_PAGES,
1685 pageblock_nr_pages) * PAGE_SIZE;
1686 vm->subblock_size = max_t(uint64_t, vm->device_block_size,
1688 vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
1690 /* Round up to the next full memory block */
1691 vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
1692 memory_block_size_bytes());
1693 vm->next_mb_id = vm->first_mb_id;
1694 vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr +
1695 vm->region_size) - 1;
1697 dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
1698 dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
1699 dev_info(&vm->vdev->dev, "device block size: 0x%llx",
1700 (unsigned long long)vm->device_block_size);
1701 dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
1702 memory_block_size_bytes());
1703 dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
1704 (unsigned long long)vm->subblock_size);
1705 if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA))
1706 dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
1711 static int virtio_mem_create_resource(struct virtio_mem *vm)
1714 * When force-unloading the driver and removing the device, we
1715 * could have a garbage pointer. Duplicate the string.
1717 const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
1722 vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
1723 name, IORESOURCE_SYSTEM_RAM);
1724 if (!vm->parent_resource) {
1726 dev_warn(&vm->vdev->dev, "could not reserve device region\n");
1727 dev_info(&vm->vdev->dev,
1728 "reloading the driver is not supported\n");
1732 /* The memory is not actually busy - make add_memory() work. */
1733 vm->parent_resource->flags &= ~IORESOURCE_BUSY;
1737 static void virtio_mem_delete_resource(struct virtio_mem *vm)
1741 if (!vm->parent_resource)
1744 name = vm->parent_resource->name;
1745 release_resource(vm->parent_resource);
1746 kfree(vm->parent_resource);
1748 vm->parent_resource = NULL;
1751 static int virtio_mem_probe(struct virtio_device *vdev)
1753 struct virtio_mem *vm;
1756 BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
1757 BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
1759 vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1763 init_waitqueue_head(&vm->host_resp);
1765 INIT_WORK(&vm->wq, virtio_mem_run_wq);
1766 mutex_init(&vm->hotplug_mutex);
1767 INIT_LIST_HEAD(&vm->next);
1768 spin_lock_init(&vm->removal_lock);
1769 hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1770 vm->retry_timer.function = virtio_mem_timer_expired;
1771 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
1773 /* register the virtqueue */
1774 rc = virtio_mem_init_vq(vm);
1778 /* initialize the device by querying the config */
1779 rc = virtio_mem_init(vm);
1783 /* create the parent resource for all memory */
1784 rc = virtio_mem_create_resource(vm);
1789 * If we still have memory plugged, we have to unplug all memory first.
1790 * Registering our parent resource makes sure that this memory isn't
1791 * actually in use (e.g., trying to reload the driver).
1793 if (vm->plugged_size) {
1794 vm->unplug_all_required = 1;
1795 dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
1798 /* register callbacks */
1799 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
1800 rc = register_memory_notifier(&vm->memory_notifier);
1802 goto out_del_resource;
1803 rc = register_virtio_mem_device(vm);
1807 virtio_device_ready(vdev);
1809 /* trigger a config update to start processing the requested_size */
1810 atomic_set(&vm->config_changed, 1);
1811 queue_work(system_freezable_wq, &vm->wq);
1815 unregister_memory_notifier(&vm->memory_notifier);
1817 virtio_mem_delete_resource(vm);
1819 vdev->config->del_vqs(vdev);
1827 static void virtio_mem_remove(struct virtio_device *vdev)
1829 struct virtio_mem *vm = vdev->priv;
1830 unsigned long mb_id;
1834 * Make sure the workqueue won't be triggered anymore and no memory
1835 * blocks can be onlined/offlined until we're finished here.
1837 mutex_lock(&vm->hotplug_mutex);
1838 spin_lock_irq(&vm->removal_lock);
1839 vm->removing = true;
1840 spin_unlock_irq(&vm->removal_lock);
1841 mutex_unlock(&vm->hotplug_mutex);
1843 /* wait until the workqueue stopped */
1844 cancel_work_sync(&vm->wq);
1845 hrtimer_cancel(&vm->retry_timer);
1848 * After we unregistered our callbacks, user space can online partially
1849 * plugged offline blocks. Make sure to remove them.
1851 virtio_mem_for_each_mb_state(vm, mb_id,
1852 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1853 rc = virtio_mem_mb_remove(vm, mb_id);
1855 virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
1858 * After we unregistered our callbacks, user space can no longer
1859 * offline partially plugged online memory blocks. No need to worry
1863 /* unregister callbacks */
1864 unregister_virtio_mem_device(vm);
1865 unregister_memory_notifier(&vm->memory_notifier);
1868 * There is no way we could reliably remove all memory we have added to
1869 * the system. And there is no way to stop the driver/device from going
1870 * away. Warn at least.
1872 if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] ||
1873 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] ||
1874 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] ||
1875 vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL]) {
1876 dev_warn(&vdev->dev, "device still has system memory added\n");
1878 virtio_mem_delete_resource(vm);
1879 kfree_const(vm->resource_name);
1882 /* remove all tracking data - no locking needed */
1883 vfree(vm->mb_state);
1884 vfree(vm->sb_bitmap);
1886 /* reset the device and cleanup the queues */
1887 vdev->config->reset(vdev);
1888 vdev->config->del_vqs(vdev);
1894 static void virtio_mem_config_changed(struct virtio_device *vdev)
1896 struct virtio_mem *vm = vdev->priv;
1898 atomic_set(&vm->config_changed, 1);
1899 virtio_mem_retry(vm);
1902 #ifdef CONFIG_PM_SLEEP
1903 static int virtio_mem_freeze(struct virtio_device *vdev)
1906 * When restarting the VM, all memory is usually unplugged. Don't
1907 * allow to suspend/hibernate.
1909 dev_err(&vdev->dev, "save/restore not supported.\n");
1913 static int virtio_mem_restore(struct virtio_device *vdev)
1919 static unsigned int virtio_mem_features[] = {
1920 #if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
1921 VIRTIO_MEM_F_ACPI_PXM,
1925 static const struct virtio_device_id virtio_mem_id_table[] = {
1926 { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
1930 static struct virtio_driver virtio_mem_driver = {
1931 .feature_table = virtio_mem_features,
1932 .feature_table_size = ARRAY_SIZE(virtio_mem_features),
1933 .driver.name = KBUILD_MODNAME,
1934 .driver.owner = THIS_MODULE,
1935 .id_table = virtio_mem_id_table,
1936 .probe = virtio_mem_probe,
1937 .remove = virtio_mem_remove,
1938 .config_changed = virtio_mem_config_changed,
1939 #ifdef CONFIG_PM_SLEEP
1940 .freeze = virtio_mem_freeze,
1941 .restore = virtio_mem_restore,
1945 module_virtio_driver(virtio_mem_driver);
1946 MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
1947 MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
1948 MODULE_DESCRIPTION("Virtio-mem driver");
1949 MODULE_LICENSE("GPL");