1 // SPDX-License-Identifier: GPL-2.0
3 * Gasket generic driver framework. This file contains the implementation
4 * for the Gasket generic driver framework - the functionality that is common
5 * across Gasket devices.
7 * Copyright (C) 2018 Google, Inc.
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include "gasket_core.h"
14 #include "gasket_interrupt.h"
15 #include "gasket_ioctl.h"
16 #include "gasket_page_table.h"
17 #include "gasket_sysfs.h"
19 #include <linux/capability.h>
20 #include <linux/compiler.h>
21 #include <linux/delay.h>
22 #include <linux/device.h>
24 #include <linux/init.h>
26 #include <linux/pid_namespace.h>
27 #include <linux/printk.h>
28 #include <linux/sched.h>
30 #ifdef GASKET_KERNEL_TRACE_SUPPORT
31 #define CREATE_TRACE_POINTS
32 #include <trace/events/gasket_mmap.h>
34 #define trace_gasket_mmap_exit(x)
35 #define trace_gasket_mmap_entry(x, ...)
39 * "Private" members of gasket_driver_desc.
41 * Contains internal per-device type tracking data, i.e., data not appropriate
42 * as part of the public interface for the generic framework.
44 struct gasket_internal_desc {
45 /* Device-specific-driver-provided configuration information. */
46 const struct gasket_driver_desc *driver_desc;
48 /* Protects access to per-driver data (i.e. this structure). */
51 /* Kernel-internal device class. */
54 /* Instantiated / present devices of this type. */
55 struct gasket_dev *devs[GASKET_DEV_MAX];
58 /* do_map_region() needs be able to return more than just true/false. */
59 enum do_map_region_status {
60 /* The region was successfully mapped. */
61 DO_MAP_REGION_SUCCESS,
63 /* Attempted to map region and failed. */
64 DO_MAP_REGION_FAILURE,
66 /* The requested region to map was not part of a mappable region. */
67 DO_MAP_REGION_INVALID,
70 /* Global data definitions. */
71 /* Mutex - only for framework-wide data. Other data should be protected by
72 * finer-grained locks.
74 static DEFINE_MUTEX(g_mutex);
76 /* List of all registered device descriptions & their supporting data. */
77 static struct gasket_internal_desc g_descs[GASKET_FRAMEWORK_DESC_MAX];
79 /* Mapping of statuses to human-readable strings. Must end with {0,NULL}. */
80 static const struct gasket_num_name gasket_status_name_table[] = {
81 { GASKET_STATUS_DEAD, "DEAD" },
82 { GASKET_STATUS_ALIVE, "ALIVE" },
83 { GASKET_STATUS_LAMED, "LAMED" },
84 { GASKET_STATUS_DRIVER_EXIT, "DRIVER_EXITING" },
88 /* Enumeration of the automatic Gasket framework sysfs nodes. */
89 enum gasket_sysfs_attribute_type {
93 ATTR_FRAMEWORK_VERSION,
95 ATTR_HARDWARE_REVISION,
100 ATTR_WRITE_OPEN_COUNT,
105 /* Perform a standard Gasket callback. */
107 check_and_invoke_callback(struct gasket_dev *gasket_dev,
108 int (*cb_function)(struct gasket_dev *))
113 mutex_lock(&gasket_dev->mutex);
114 ret = cb_function(gasket_dev);
115 mutex_unlock(&gasket_dev->mutex);
120 /* Perform a standard Gasket callback without grabbing gasket_dev->mutex. */
122 gasket_check_and_invoke_callback_nolock(struct gasket_dev *gasket_dev,
123 int (*cb_function)(struct gasket_dev *))
128 ret = cb_function(gasket_dev);
133 * Return nonzero if the gasket_cdev_info is owned by the current thread group
136 static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info)
138 return (info->ownership.is_owned &&
139 (info->ownership.owner == current->tgid));
143 * Find the next free gasket_internal_dev slot.
145 * Returns the located slot number on success or a negative number on failure.
147 static int gasket_find_dev_slot(struct gasket_internal_desc *internal_desc,
148 const char *kobj_name)
152 mutex_lock(&internal_desc->mutex);
154 /* Search for a previous instance of this device. */
155 for (i = 0; i < GASKET_DEV_MAX; i++) {
156 if (internal_desc->devs[i] &&
157 strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) {
158 pr_err("Duplicate device %s\n", kobj_name);
159 mutex_unlock(&internal_desc->mutex);
164 /* Find a free device slot. */
165 for (i = 0; i < GASKET_DEV_MAX; i++) {
166 if (!internal_desc->devs[i])
170 if (i == GASKET_DEV_MAX) {
171 pr_err("Too many registered devices; max %d\n", GASKET_DEV_MAX);
172 mutex_unlock(&internal_desc->mutex);
176 mutex_unlock(&internal_desc->mutex);
181 * Allocate and initialize a Gasket device structure, add the device to the
184 * Returns 0 if successful, a negative error code otherwise.
186 static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc,
187 struct device *parent, struct gasket_dev **pdev)
190 const struct gasket_driver_desc *driver_desc =
191 internal_desc->driver_desc;
192 struct gasket_dev *gasket_dev;
193 struct gasket_cdev_info *dev_info;
194 const char *parent_name = dev_name(parent);
196 pr_debug("Allocating a Gasket device, parent %s.\n", parent_name);
200 dev_idx = gasket_find_dev_slot(internal_desc, parent_name);
204 gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL);
206 pr_err("no memory for device, parent %s\n", parent_name);
209 internal_desc->devs[dev_idx] = gasket_dev;
211 mutex_init(&gasket_dev->mutex);
213 gasket_dev->internal_desc = internal_desc;
214 gasket_dev->dev_idx = dev_idx;
215 snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", parent_name);
216 gasket_dev->dev = get_device(parent);
217 /* gasket_bar_data is uninitialized. */
218 gasket_dev->num_page_tables = driver_desc->num_page_tables;
219 /* max_page_table_size and *page table are uninit'ed */
220 /* interrupt_data is not initialized. */
221 /* status is 0, or GASKET_STATUS_DEAD */
223 dev_info = &gasket_dev->dev_info;
224 snprintf(dev_info->name, GASKET_NAME_MAX, "%s_%u", driver_desc->name,
225 gasket_dev->dev_idx);
227 MKDEV(driver_desc->major, driver_desc->minor +
228 gasket_dev->dev_idx);
230 device_create(internal_desc->class, parent, dev_info->devt,
231 gasket_dev, dev_info->name);
233 /* cdev has not yet been added; cdev_added is 0 */
234 dev_info->gasket_dev_ptr = gasket_dev;
235 /* ownership is all 0, indicating no owner or opens. */
240 /* Free a Gasket device. */
241 static void gasket_free_dev(struct gasket_dev *gasket_dev)
243 struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc;
245 mutex_lock(&internal_desc->mutex);
246 internal_desc->devs[gasket_dev->dev_idx] = NULL;
247 mutex_unlock(&internal_desc->mutex);
248 put_device(gasket_dev->dev);
253 * Maps the specified bar into kernel space.
255 * Returns 0 on success, a negative error code otherwise.
256 * A zero-sized BAR will not be mapped, but is not an error.
258 static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num)
260 struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc;
261 const struct gasket_driver_desc *driver_desc =
262 internal_desc->driver_desc;
263 ulong desc_bytes = driver_desc->bar_descriptions[bar_num].size;
264 struct gasket_bar_data *data;
270 if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR) {
271 /* not PCI: skip this entry */
275 data = &gasket_dev->bar_data[bar_num];
278 * pci_resource_start and pci_resource_len return a "resource_size_t",
279 * which is safely castable to ulong (which itself is the arg to
280 * request_mem_region).
283 (ulong)pci_resource_start(gasket_dev->pci_dev, bar_num);
284 if (!data->phys_base) {
285 dev_err(gasket_dev->dev, "Cannot get BAR%u base address\n",
291 (ulong)pci_resource_len(gasket_dev->pci_dev, bar_num);
292 if (data->length_bytes < desc_bytes) {
293 dev_err(gasket_dev->dev,
294 "PCI BAR %u space is too small: %lu; expected >= %lu\n",
295 bar_num, data->length_bytes, desc_bytes);
299 if (!request_mem_region(data->phys_base, data->length_bytes,
300 gasket_dev->dev_info.name)) {
301 dev_err(gasket_dev->dev,
302 "Cannot get BAR %d memory region %p\n",
303 bar_num, &gasket_dev->pci_dev->resource[bar_num]);
307 data->virt_base = ioremap(data->phys_base, data->length_bytes);
308 if (!data->virt_base) {
309 dev_err(gasket_dev->dev,
310 "Cannot remap BAR %d memory region %p\n",
311 bar_num, &gasket_dev->pci_dev->resource[bar_num]);
316 dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64));
317 dma_set_coherent_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64));
322 iounmap(data->virt_base);
323 release_mem_region(data->phys_base, data->length_bytes);
328 * Releases PCI BAR mapping.
330 * A zero-sized or not-mapped BAR will not be unmapped, but is not an error.
332 static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num)
335 struct gasket_internal_desc *internal_desc = dev->internal_desc;
336 const struct gasket_driver_desc *driver_desc =
337 internal_desc->driver_desc;
339 if (driver_desc->bar_descriptions[bar_num].size == 0 ||
340 !dev->bar_data[bar_num].virt_base)
343 if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR)
346 iounmap(dev->bar_data[bar_num].virt_base);
347 dev->bar_data[bar_num].virt_base = NULL;
349 base = pci_resource_start(dev->pci_dev, bar_num);
351 dev_err(dev->dev, "cannot get PCI BAR%u base address\n",
356 bytes = pci_resource_len(dev->pci_dev, bar_num);
357 release_mem_region(base, bytes);
361 * Setup PCI memory mapping for the specified device.
363 * Reads the BAR registers and sets up pointers to the device's memory mapped
366 * Returns 0 on success and a negative value otherwise.
368 static int gasket_setup_pci(struct pci_dev *pci_dev,
369 struct gasket_dev *gasket_dev)
371 int i, mapped_bars, ret;
373 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
374 ret = gasket_map_pci_bar(gasket_dev, i);
384 for (i = 0; i < mapped_bars; i++)
385 gasket_unmap_pci_bar(gasket_dev, i);
390 /* Unmaps memory for the specified device. */
391 static void gasket_cleanup_pci(struct gasket_dev *gasket_dev)
395 for (i = 0; i < PCI_STD_NUM_BARS; i++)
396 gasket_unmap_pci_bar(gasket_dev, i);
399 /* Determine the health of the Gasket device. */
400 static int gasket_get_hw_status(struct gasket_dev *gasket_dev)
404 const struct gasket_driver_desc *driver_desc =
405 gasket_dev->internal_desc->driver_desc;
407 status = gasket_check_and_invoke_callback_nolock(gasket_dev,
408 driver_desc->device_status_cb);
409 if (status != GASKET_STATUS_ALIVE) {
410 dev_dbg(gasket_dev->dev, "Hardware reported status %d.\n",
415 status = gasket_interrupt_system_status(gasket_dev);
416 if (status != GASKET_STATUS_ALIVE) {
417 dev_dbg(gasket_dev->dev,
418 "Interrupt system reported status %d.\n", status);
422 for (i = 0; i < driver_desc->num_page_tables; ++i) {
423 status = gasket_page_table_system_status(gasket_dev->page_table[i]);
424 if (status != GASKET_STATUS_ALIVE) {
425 dev_dbg(gasket_dev->dev,
426 "Page table %d reported status %d.\n",
432 return GASKET_STATUS_ALIVE;
436 gasket_write_mappable_regions(char *buf,
437 const struct gasket_driver_desc *driver_desc,
442 ssize_t total_written = 0;
443 ulong min_addr, max_addr;
444 struct gasket_bar_desc bar_desc =
445 driver_desc->bar_descriptions[bar_index];
447 if (bar_desc.permissions == GASKET_NOMAP)
450 i < bar_desc.num_mappable_regions && total_written < PAGE_SIZE;
452 min_addr = bar_desc.mappable_regions[i].start -
453 driver_desc->legacy_mmap_address_offset;
454 max_addr = bar_desc.mappable_regions[i].start -
455 driver_desc->legacy_mmap_address_offset +
456 bar_desc.mappable_regions[i].length_bytes;
457 written = scnprintf(buf, PAGE_SIZE - total_written,
458 "0x%08lx-0x%08lx\n", min_addr, max_addr);
459 total_written += written;
462 return total_written;
465 static ssize_t gasket_sysfs_data_show(struct device *device,
466 struct device_attribute *attr, char *buf)
469 ssize_t current_written = 0;
470 const struct gasket_driver_desc *driver_desc;
471 struct gasket_dev *gasket_dev;
472 struct gasket_sysfs_attribute *gasket_attr;
473 const struct gasket_bar_desc *bar_desc;
474 enum gasket_sysfs_attribute_type sysfs_type;
476 gasket_dev = gasket_sysfs_get_device_data(device);
478 dev_err(device, "No sysfs mapping found for device\n");
482 gasket_attr = gasket_sysfs_get_attr(device, attr);
484 dev_err(device, "No sysfs attr found for device\n");
485 gasket_sysfs_put_device_data(device, gasket_dev);
489 driver_desc = gasket_dev->internal_desc->driver_desc;
492 (enum gasket_sysfs_attribute_type)gasket_attr->data.attr_type;
493 switch (sysfs_type) {
494 case ATTR_BAR_OFFSETS:
495 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
496 bar_desc = &driver_desc->bar_descriptions[i];
497 if (bar_desc->size == 0)
500 snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i,
501 (ulong)bar_desc->base);
502 buf += current_written;
503 ret += current_written;
507 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
508 bar_desc = &driver_desc->bar_descriptions[i];
509 if (bar_desc->size == 0)
512 snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i,
513 (ulong)bar_desc->size);
514 buf += current_written;
515 ret += current_written;
518 case ATTR_DRIVER_VERSION:
519 ret = snprintf(buf, PAGE_SIZE, "%s\n",
520 gasket_dev->internal_desc->driver_desc->driver_version);
522 case ATTR_FRAMEWORK_VERSION:
523 ret = snprintf(buf, PAGE_SIZE, "%s\n",
524 GASKET_FRAMEWORK_VERSION);
526 case ATTR_DEVICE_TYPE:
527 ret = snprintf(buf, PAGE_SIZE, "%s\n",
528 gasket_dev->internal_desc->driver_desc->name);
530 case ATTR_HARDWARE_REVISION:
531 ret = snprintf(buf, PAGE_SIZE, "%d\n",
532 gasket_dev->hardware_revision);
534 case ATTR_PCI_ADDRESS:
535 ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name);
538 ret = snprintf(buf, PAGE_SIZE, "%s\n",
539 gasket_num_name_lookup(gasket_dev->status,
540 gasket_status_name_table));
542 case ATTR_IS_DEVICE_OWNED:
543 ret = snprintf(buf, PAGE_SIZE, "%d\n",
544 gasket_dev->dev_info.ownership.is_owned);
546 case ATTR_DEVICE_OWNER:
547 ret = snprintf(buf, PAGE_SIZE, "%d\n",
548 gasket_dev->dev_info.ownership.owner);
550 case ATTR_WRITE_OPEN_COUNT:
551 ret = snprintf(buf, PAGE_SIZE, "%d\n",
552 gasket_dev->dev_info.ownership.write_open_count);
554 case ATTR_RESET_COUNT:
555 ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count);
557 case ATTR_USER_MEM_RANGES:
558 for (i = 0; i < PCI_STD_NUM_BARS; ++i) {
560 gasket_write_mappable_regions(buf, driver_desc,
562 buf += current_written;
563 ret += current_written;
567 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
573 gasket_sysfs_put_attr(device, gasket_attr);
574 gasket_sysfs_put_device_data(device, gasket_dev);
578 /* These attributes apply to all Gasket driver instances. */
579 static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = {
580 GASKET_SYSFS_RO(bar_offsets, gasket_sysfs_data_show, ATTR_BAR_OFFSETS),
581 GASKET_SYSFS_RO(bar_sizes, gasket_sysfs_data_show, ATTR_BAR_SIZES),
582 GASKET_SYSFS_RO(driver_version, gasket_sysfs_data_show,
583 ATTR_DRIVER_VERSION),
584 GASKET_SYSFS_RO(framework_version, gasket_sysfs_data_show,
585 ATTR_FRAMEWORK_VERSION),
586 GASKET_SYSFS_RO(device_type, gasket_sysfs_data_show, ATTR_DEVICE_TYPE),
587 GASKET_SYSFS_RO(revision, gasket_sysfs_data_show,
588 ATTR_HARDWARE_REVISION),
589 GASKET_SYSFS_RO(pci_address, gasket_sysfs_data_show, ATTR_PCI_ADDRESS),
590 GASKET_SYSFS_RO(status, gasket_sysfs_data_show, ATTR_STATUS),
591 GASKET_SYSFS_RO(is_device_owned, gasket_sysfs_data_show,
592 ATTR_IS_DEVICE_OWNED),
593 GASKET_SYSFS_RO(device_owner, gasket_sysfs_data_show,
595 GASKET_SYSFS_RO(write_open_count, gasket_sysfs_data_show,
596 ATTR_WRITE_OPEN_COUNT),
597 GASKET_SYSFS_RO(reset_count, gasket_sysfs_data_show, ATTR_RESET_COUNT),
598 GASKET_SYSFS_RO(user_mem_ranges, gasket_sysfs_data_show,
599 ATTR_USER_MEM_RANGES),
600 GASKET_END_OF_ATTR_ARRAY
603 /* Add a char device and related info. */
604 static int gasket_add_cdev(struct gasket_cdev_info *dev_info,
605 const struct file_operations *file_ops,
606 struct module *owner)
610 cdev_init(&dev_info->cdev, file_ops);
611 dev_info->cdev.owner = owner;
612 ret = cdev_add(&dev_info->cdev, dev_info->devt, 1);
614 dev_err(dev_info->gasket_dev_ptr->dev,
615 "cannot add char device [ret=%d]\n", ret);
618 dev_info->cdev_added = 1;
623 /* Disable device operations. */
624 void gasket_disable_device(struct gasket_dev *gasket_dev)
626 const struct gasket_driver_desc *driver_desc =
627 gasket_dev->internal_desc->driver_desc;
630 /* Only delete the device if it has been successfully added. */
631 if (gasket_dev->dev_info.cdev_added)
632 cdev_del(&gasket_dev->dev_info.cdev);
634 gasket_dev->status = GASKET_STATUS_DEAD;
636 gasket_interrupt_cleanup(gasket_dev);
638 for (i = 0; i < driver_desc->num_page_tables; ++i) {
639 if (gasket_dev->page_table[i]) {
640 gasket_page_table_reset(gasket_dev->page_table[i]);
641 gasket_page_table_cleanup(gasket_dev->page_table[i]);
645 EXPORT_SYMBOL(gasket_disable_device);
648 * Registered driver descriptor lookup for PCI devices.
650 * Precondition: Called with g_mutex held (to avoid a race on return).
651 * Returns NULL if no matching device was found.
653 static struct gasket_internal_desc *
654 lookup_pci_internal_desc(struct pci_dev *pci_dev)
658 __must_hold(&g_mutex);
659 for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
660 if (g_descs[i].driver_desc &&
661 g_descs[i].driver_desc->pci_id_table &&
662 pci_match_id(g_descs[i].driver_desc->pci_id_table, pci_dev))
670 * Verifies that the user has permissions to perform the requested mapping and
671 * that the provided descriptor/range is of adequate size to hold the range to
674 static bool gasket_mmap_has_permissions(struct gasket_dev *gasket_dev,
675 struct vm_area_struct *vma,
678 int requested_permissions;
679 /* Always allow sysadmin to access. */
680 if (capable(CAP_SYS_ADMIN))
683 /* Never allow non-sysadmins to access to a dead device. */
684 if (gasket_dev->status != GASKET_STATUS_ALIVE) {
685 dev_dbg(gasket_dev->dev, "Device is dead.\n");
689 /* Make sure that no wrong flags are set. */
690 requested_permissions =
691 (vma->vm_flags & VM_ACCESS_FLAGS);
692 if (requested_permissions & ~(bar_permissions)) {
693 dev_dbg(gasket_dev->dev,
694 "Attempting to map a region with requested permissions 0x%x, but region has permissions 0x%x.\n",
695 requested_permissions, bar_permissions);
699 /* Do not allow a non-owner to write. */
700 if ((vma->vm_flags & VM_WRITE) &&
701 !gasket_owned_by_current_tgid(&gasket_dev->dev_info)) {
702 dev_dbg(gasket_dev->dev,
703 "Attempting to mmap a region for write without owning device.\n");
711 * Verifies that the input address is within the region allocated to coherent
715 gasket_is_coherent_region(const struct gasket_driver_desc *driver_desc,
718 struct gasket_coherent_buffer_desc coh_buff_desc =
719 driver_desc->coherent_buffer_description;
721 if (coh_buff_desc.permissions != GASKET_NOMAP) {
722 if ((address >= coh_buff_desc.base) &&
723 (address < coh_buff_desc.base + coh_buff_desc.size)) {
730 static int gasket_get_bar_index(const struct gasket_dev *gasket_dev,
734 const struct gasket_driver_desc *driver_desc;
736 driver_desc = gasket_dev->internal_desc->driver_desc;
737 for (i = 0; i < PCI_STD_NUM_BARS; ++i) {
738 struct gasket_bar_desc bar_desc =
739 driver_desc->bar_descriptions[i];
741 if (bar_desc.permissions != GASKET_NOMAP) {
742 if (phys_addr >= bar_desc.base &&
743 phys_addr < (bar_desc.base + bar_desc.size)) {
748 /* If we haven't found the address by now, it is invalid. */
753 * Sets the actual bounds to map, given the device's mappable region.
755 * Given the device's mappable region, along with the user-requested mapping
756 * start offset and length of the user region, determine how much of this
757 * mappable region can be mapped into the user's region (start/end offsets),
758 * and the physical offset (phys_offset) into the BAR where the mapping should
759 * begin (either the VMA's or region lower bound).
761 * In other words, this calculates the overlap between the VMA
762 * (bar_offset, requested_length) and the given gasket_mappable_region.
764 * Returns true if there's anything to map, and false otherwise.
767 gasket_mm_get_mapping_addrs(const struct gasket_mappable_region *region,
768 ulong bar_offset, ulong requested_length,
769 struct gasket_mappable_region *mappable_region,
772 ulong range_start = region->start;
773 ulong range_length = region->length_bytes;
774 ulong range_end = range_start + range_length;
777 if (bar_offset + requested_length < range_start) {
779 * If the requested region is completely below the range,
780 * there is nothing to map.
783 } else if (bar_offset <= range_start) {
784 /* If the bar offset is below this range's start
785 * but the requested length continues into it:
786 * 1) Only map starting from the beginning of this
787 * range's phys. offset, so we don't map unmappable
789 * 2) The length of the virtual memory to not map is the
790 * delta between the bar offset and the
791 * mappable start (and since the mappable start is
792 * bigger, start - req.)
793 * 3) The map length is the minimum of the mappable
794 * requested length (requested_length - virt_offset)
795 * and the actual mappable length of the range.
797 mappable_region->start = range_start;
798 *virt_offset = range_start - bar_offset;
799 mappable_region->length_bytes =
800 min(requested_length - *virt_offset, range_length);
802 } else if (bar_offset > range_start &&
803 bar_offset < range_end) {
805 * If the bar offset is within this range:
806 * 1) Map starting from the bar offset.
807 * 2) Because there is no forbidden memory between the
808 * bar offset and the range start,
810 * 3) The map length is the minimum of the requested
811 * length and the remaining length in the buffer
812 * (range_end - bar_offset)
814 mappable_region->start = bar_offset;
816 mappable_region->length_bytes =
817 min(requested_length, range_end - bar_offset);
822 * If the requested [start] offset is above range_end,
823 * there's nothing to map.
829 * Calculates the offset where the VMA range begins in its containing BAR.
830 * The offset is written into bar_offset on success.
831 * Returns zero on success, anything else on error.
833 static int gasket_mm_vma_bar_offset(const struct gasket_dev *gasket_dev,
834 const struct vm_area_struct *vma,
839 const struct gasket_driver_desc *driver_desc =
840 gasket_dev->internal_desc->driver_desc;
842 raw_offset = (vma->vm_pgoff << PAGE_SHIFT) +
843 driver_desc->legacy_mmap_address_offset;
844 bar_index = gasket_get_bar_index(gasket_dev, raw_offset);
846 dev_err(gasket_dev->dev,
847 "Unable to find matching bar for address 0x%lx\n",
849 trace_gasket_mmap_exit(bar_index);
853 raw_offset - driver_desc->bar_descriptions[bar_index].base;
858 int gasket_mm_unmap_region(const struct gasket_dev *gasket_dev,
859 struct vm_area_struct *vma,
860 const struct gasket_mappable_region *map_region)
864 struct gasket_mappable_region mappable_region;
867 if (map_region->length_bytes == 0)
870 ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset);
874 if (!gasket_mm_get_mapping_addrs(map_region, bar_offset,
875 vma->vm_end - vma->vm_start,
876 &mappable_region, &virt_offset))
880 * The length passed to zap_vma_ptes MUST BE A MULTIPLE OF
881 * PAGE_SIZE! Trust me. I have the scars.
883 * Next multiple of y: ceil_div(x, y) * y
885 zap_vma_ptes(vma, vma->vm_start + virt_offset,
886 DIV_ROUND_UP(mappable_region.length_bytes, PAGE_SIZE) *
890 EXPORT_SYMBOL(gasket_mm_unmap_region);
892 /* Maps a virtual address + range to a physical offset of a BAR. */
893 static enum do_map_region_status
894 do_map_region(const struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
895 struct gasket_mappable_region *mappable_region)
897 /* Maximum size of a single call to io_remap_pfn_range. */
898 /* I pulled this number out of thin air. */
899 const ulong max_chunk_size = 64 * 1024 * 1024;
900 ulong chunk_size, mapped_bytes = 0;
902 const struct gasket_driver_desc *driver_desc =
903 gasket_dev->internal_desc->driver_desc;
905 ulong bar_offset, virt_offset;
906 struct gasket_mappable_region region_to_map;
907 ulong phys_offset, map_length;
908 ulong virt_base, phys_base;
911 ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset);
913 return DO_MAP_REGION_INVALID;
915 if (!gasket_mm_get_mapping_addrs(mappable_region, bar_offset,
916 vma->vm_end - vma->vm_start,
917 ®ion_to_map, &virt_offset))
918 return DO_MAP_REGION_INVALID;
919 phys_offset = region_to_map.start;
920 map_length = region_to_map.length_bytes;
922 virt_base = vma->vm_start + virt_offset;
924 gasket_get_bar_index(gasket_dev,
925 (vma->vm_pgoff << PAGE_SHIFT) +
926 driver_desc->legacy_mmap_address_offset);
929 return DO_MAP_REGION_INVALID;
931 phys_base = gasket_dev->bar_data[bar_index].phys_base + phys_offset;
932 while (mapped_bytes < map_length) {
934 * io_remap_pfn_range can take a while, so we chunk its
935 * calls and call cond_resched between each.
937 chunk_size = min(max_chunk_size, map_length - mapped_bytes);
940 ret = io_remap_pfn_range(vma, virt_base + mapped_bytes,
941 (phys_base + mapped_bytes) >>
942 PAGE_SHIFT, chunk_size,
945 dev_err(gasket_dev->dev,
946 "Error remapping PFN range.\n");
949 mapped_bytes += chunk_size;
952 return DO_MAP_REGION_SUCCESS;
955 /* Unmap the partial chunk we mapped. */
956 mappable_region->length_bytes = mapped_bytes;
957 if (gasket_mm_unmap_region(gasket_dev, vma, mappable_region))
958 dev_err(gasket_dev->dev,
959 "Error unmapping partial region 0x%lx (0x%lx bytes)\n",
961 (ulong)mapped_bytes);
963 return DO_MAP_REGION_FAILURE;
966 /* Map a region of coherent memory. */
967 static int gasket_mmap_coherent(struct gasket_dev *gasket_dev,
968 struct vm_area_struct *vma)
970 const struct gasket_driver_desc *driver_desc =
971 gasket_dev->internal_desc->driver_desc;
972 const ulong requested_length = vma->vm_end - vma->vm_start;
976 if (requested_length == 0 || requested_length >
977 gasket_dev->coherent_buffer.length_bytes) {
978 trace_gasket_mmap_exit(-EINVAL);
982 permissions = driver_desc->coherent_buffer_description.permissions;
983 if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) {
984 dev_err(gasket_dev->dev, "Permission checking failed.\n");
985 trace_gasket_mmap_exit(-EPERM);
989 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
991 ret = remap_pfn_range(vma, vma->vm_start,
992 (gasket_dev->coherent_buffer.phys_base) >>
993 PAGE_SHIFT, requested_length, vma->vm_page_prot);
995 dev_err(gasket_dev->dev, "Error remapping PFN range err=%d.\n",
997 trace_gasket_mmap_exit(ret);
1001 /* Record the user virtual to dma_address mapping that was
1002 * created by the kernel.
1004 gasket_set_user_virt(gasket_dev, requested_length,
1005 gasket_dev->coherent_buffer.phys_base,
1010 /* Map a device's BARs into user space. */
1011 static int gasket_mmap(struct file *filp, struct vm_area_struct *vma)
1015 int has_mapped_anything = 0;
1017 ulong raw_offset, vma_size;
1018 bool is_coherent_region;
1019 const struct gasket_driver_desc *driver_desc;
1020 struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data;
1021 const struct gasket_bar_desc *bar_desc;
1022 struct gasket_mappable_region *map_regions = NULL;
1023 int num_map_regions = 0;
1024 enum do_map_region_status map_status;
1026 driver_desc = gasket_dev->internal_desc->driver_desc;
1028 if (vma->vm_start & ~PAGE_MASK) {
1029 dev_err(gasket_dev->dev,
1030 "Base address not page-aligned: 0x%lx\n",
1032 trace_gasket_mmap_exit(-EINVAL);
1036 /* Calculate the offset of this range into physical mem. */
1037 raw_offset = (vma->vm_pgoff << PAGE_SHIFT) +
1038 driver_desc->legacy_mmap_address_offset;
1039 vma_size = vma->vm_end - vma->vm_start;
1040 trace_gasket_mmap_entry(gasket_dev->dev_info.name, raw_offset,
1044 * Check if the raw offset is within a bar region. If not, check if it
1045 * is a coherent region.
1047 bar_index = gasket_get_bar_index(gasket_dev, raw_offset);
1048 is_coherent_region = gasket_is_coherent_region(driver_desc, raw_offset);
1049 if (bar_index < 0 && !is_coherent_region) {
1050 dev_err(gasket_dev->dev,
1051 "Unable to find matching bar for address 0x%lx\n",
1053 trace_gasket_mmap_exit(bar_index);
1056 if (bar_index > 0 && is_coherent_region) {
1057 dev_err(gasket_dev->dev,
1058 "double matching bar and coherent buffers for address 0x%lx\n",
1060 trace_gasket_mmap_exit(bar_index);
1064 vma->vm_private_data = gasket_dev;
1066 if (is_coherent_region)
1067 return gasket_mmap_coherent(gasket_dev, vma);
1069 /* Everything in the rest of this function is for normal BAR mapping. */
1072 * Subtract the base of the bar from the raw offset to get the
1073 * memory location within the bar to map.
1075 bar_desc = &driver_desc->bar_descriptions[bar_index];
1076 permissions = bar_desc->permissions;
1077 if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) {
1078 dev_err(gasket_dev->dev, "Permission checking failed.\n");
1079 trace_gasket_mmap_exit(-EPERM);
1083 if (driver_desc->get_mappable_regions_cb) {
1084 ret = driver_desc->get_mappable_regions_cb(gasket_dev,
1091 if (!gasket_mmap_has_permissions(gasket_dev, vma,
1092 bar_desc->permissions)) {
1093 dev_err(gasket_dev->dev,
1094 "Permission checking failed.\n");
1095 trace_gasket_mmap_exit(-EPERM);
1098 num_map_regions = bar_desc->num_mappable_regions;
1099 map_regions = kcalloc(num_map_regions,
1100 sizeof(*bar_desc->mappable_regions),
1103 memcpy(map_regions, bar_desc->mappable_regions,
1105 sizeof(*bar_desc->mappable_regions));
1109 if (!map_regions || num_map_regions == 0) {
1110 dev_err(gasket_dev->dev, "No mappable regions returned!\n");
1114 /* Marks the VMA's pages as uncacheable. */
1115 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1116 for (i = 0; i < num_map_regions; i++) {
1117 map_status = do_map_region(gasket_dev, vma, &map_regions[i]);
1118 /* Try the next region if this one was not mappable. */
1119 if (map_status == DO_MAP_REGION_INVALID)
1121 if (map_status == DO_MAP_REGION_FAILURE) {
1126 has_mapped_anything = 1;
1131 /* If we could not map any memory, the request was invalid. */
1132 if (!has_mapped_anything) {
1133 dev_err(gasket_dev->dev,
1134 "Map request did not contain a valid region.\n");
1135 trace_gasket_mmap_exit(-EINVAL);
1139 trace_gasket_mmap_exit(0);
1143 /* Need to unmap any mapped ranges. */
1144 num_map_regions = i;
1145 for (i = 0; i < num_map_regions; i++)
1146 if (gasket_mm_unmap_region(gasket_dev, vma,
1147 &bar_desc->mappable_regions[i]))
1148 dev_err(gasket_dev->dev, "Error unmapping range %d.\n",
1156 * Open the char device file.
1158 * If the open is for writing, and the device is not owned, this process becomes
1159 * the owner. If the open is for writing and the device is already owned by
1160 * some other process, it is an error. If this process is the owner, increment
1163 * Returns 0 if successful, a negative error number otherwise.
1165 static int gasket_open(struct inode *inode, struct file *filp)
1168 struct gasket_dev *gasket_dev;
1169 const struct gasket_driver_desc *driver_desc;
1170 struct gasket_ownership *ownership;
1171 char task_name[TASK_COMM_LEN];
1172 struct gasket_cdev_info *dev_info =
1173 container_of(inode->i_cdev, struct gasket_cdev_info, cdev);
1174 struct pid_namespace *pid_ns = task_active_pid_ns(current);
1175 bool is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN);
1177 gasket_dev = dev_info->gasket_dev_ptr;
1178 driver_desc = gasket_dev->internal_desc->driver_desc;
1179 ownership = &dev_info->ownership;
1180 get_task_comm(task_name, current);
1181 filp->private_data = gasket_dev;
1184 dev_dbg(gasket_dev->dev,
1185 "Attempting to open with tgid %u (%s) (f_mode: 0%03o, fmode_write: %d is_root: %u)\n",
1186 current->tgid, task_name, filp->f_mode,
1187 (filp->f_mode & FMODE_WRITE), is_root);
1189 /* Always allow non-writing accesses. */
1190 if (!(filp->f_mode & FMODE_WRITE)) {
1191 dev_dbg(gasket_dev->dev, "Allowing read-only opening.\n");
1195 mutex_lock(&gasket_dev->mutex);
1197 dev_dbg(gasket_dev->dev,
1198 "Current owner open count (owning tgid %u): %d.\n",
1199 ownership->owner, ownership->write_open_count);
1201 /* Opening a node owned by another TGID is an error (unless root) */
1202 if (ownership->is_owned && ownership->owner != current->tgid &&
1204 dev_err(gasket_dev->dev,
1205 "Process %u is opening a node held by %u.\n",
1206 current->tgid, ownership->owner);
1207 mutex_unlock(&gasket_dev->mutex);
1211 /* If the node is not owned, assign it to the current TGID. */
1212 if (!ownership->is_owned) {
1213 ret = gasket_check_and_invoke_callback_nolock(gasket_dev,
1214 driver_desc->device_open_cb);
1216 dev_err(gasket_dev->dev,
1217 "Error in device open cb: %d\n", ret);
1218 mutex_unlock(&gasket_dev->mutex);
1221 ownership->is_owned = 1;
1222 ownership->owner = current->tgid;
1223 dev_dbg(gasket_dev->dev, "Device owner is now tgid %u\n",
1227 ownership->write_open_count++;
1229 dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n",
1230 ownership->owner, ownership->write_open_count);
1232 mutex_unlock(&gasket_dev->mutex);
1237 * Called on a close of the device file. If this process is the owner,
1238 * decrement the open count. On last close by the owner, free up buffers and
1239 * eventfd contexts, and release ownership.
1241 * Returns 0 if successful, a negative error number otherwise.
1243 static int gasket_release(struct inode *inode, struct file *file)
1246 struct gasket_dev *gasket_dev;
1247 struct gasket_ownership *ownership;
1248 const struct gasket_driver_desc *driver_desc;
1249 char task_name[TASK_COMM_LEN];
1250 struct gasket_cdev_info *dev_info =
1251 container_of(inode->i_cdev, struct gasket_cdev_info, cdev);
1252 struct pid_namespace *pid_ns = task_active_pid_ns(current);
1253 bool is_root = ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN);
1255 gasket_dev = dev_info->gasket_dev_ptr;
1256 driver_desc = gasket_dev->internal_desc->driver_desc;
1257 ownership = &dev_info->ownership;
1258 get_task_comm(task_name, current);
1259 mutex_lock(&gasket_dev->mutex);
1261 dev_dbg(gasket_dev->dev,
1262 "Releasing device node. Call origin: tgid %u (%s) (f_mode: 0%03o, fmode_write: %d, is_root: %u)\n",
1263 current->tgid, task_name, file->f_mode,
1264 (file->f_mode & FMODE_WRITE), is_root);
1265 dev_dbg(gasket_dev->dev, "Current open count (owning tgid %u): %d\n",
1266 ownership->owner, ownership->write_open_count);
1268 if (file->f_mode & FMODE_WRITE) {
1269 ownership->write_open_count--;
1270 if (ownership->write_open_count == 0) {
1271 dev_dbg(gasket_dev->dev, "Device is now free\n");
1272 ownership->is_owned = 0;
1273 ownership->owner = 0;
1275 /* Forces chip reset before we unmap the page tables. */
1276 driver_desc->device_reset_cb(gasket_dev);
1278 for (i = 0; i < driver_desc->num_page_tables; ++i) {
1279 gasket_page_table_unmap_all(gasket_dev->page_table[i]);
1280 gasket_page_table_garbage_collect(gasket_dev->page_table[i]);
1281 gasket_free_coherent_memory_all(gasket_dev, i);
1284 /* Closes device, enters power save. */
1285 gasket_check_and_invoke_callback_nolock(gasket_dev,
1286 driver_desc->device_close_cb);
1290 dev_dbg(gasket_dev->dev, "New open count (owning tgid %u): %d\n",
1291 ownership->owner, ownership->write_open_count);
1292 mutex_unlock(&gasket_dev->mutex);
1297 * Gasket ioctl dispatch function.
1299 * Check if the ioctl is a generic ioctl. If not, pass the ioctl to the
1300 * ioctl_handler_cb registered in the driver description.
1301 * If the ioctl is a generic ioctl, pass it to gasket_ioctl_handler.
1303 static long gasket_ioctl(struct file *filp, uint cmd, ulong arg)
1305 struct gasket_dev *gasket_dev;
1306 const struct gasket_driver_desc *driver_desc;
1307 void __user *argp = (void __user *)arg;
1310 gasket_dev = (struct gasket_dev *)filp->private_data;
1311 driver_desc = gasket_dev->internal_desc->driver_desc;
1313 dev_dbg(gasket_dev->dev,
1314 "Unable to find device descriptor for file %s\n",
1315 d_path(&filp->f_path, path, 256));
1319 if (!gasket_is_supported_ioctl(cmd)) {
1321 * The ioctl handler is not a standard Gasket callback, since
1322 * it requires different arguments. This means we can't use
1323 * check_and_invoke_callback.
1325 if (driver_desc->ioctl_handler_cb)
1326 return driver_desc->ioctl_handler_cb(filp, cmd, argp);
1328 dev_dbg(gasket_dev->dev, "Received unknown ioctl 0x%x\n", cmd);
1332 return gasket_handle_ioctl(filp, cmd, argp);
1335 /* File operations for all Gasket devices. */
1336 static const struct file_operations gasket_file_ops = {
1337 .owner = THIS_MODULE,
1338 .llseek = no_llseek,
1339 .mmap = gasket_mmap,
1340 .open = gasket_open,
1341 .release = gasket_release,
1342 .unlocked_ioctl = gasket_ioctl,
1345 /* Perform final init and marks the device as active. */
1346 int gasket_enable_device(struct gasket_dev *gasket_dev)
1350 const struct gasket_driver_desc *driver_desc =
1351 gasket_dev->internal_desc->driver_desc;
1353 ret = gasket_interrupt_init(gasket_dev);
1355 dev_err(gasket_dev->dev,
1356 "Critical failure to allocate interrupts: %d\n", ret);
1357 gasket_interrupt_cleanup(gasket_dev);
1361 for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) {
1362 dev_dbg(gasket_dev->dev, "Initializing page table %d.\n",
1364 ret = gasket_page_table_init(&gasket_dev->page_table[tbl_idx],
1365 &gasket_dev->bar_data[driver_desc->page_table_bar_index],
1366 &driver_desc->page_table_configs[tbl_idx],
1368 gasket_dev->pci_dev);
1370 dev_err(gasket_dev->dev,
1371 "Couldn't init page table %d: %d\n",
1376 * Make sure that the page table is clear and set to simple
1379 gasket_page_table_reset(gasket_dev->page_table[tbl_idx]);
1383 * hardware_revision_cb returns a positive integer (the rev) if
1386 ret = check_and_invoke_callback(gasket_dev,
1387 driver_desc->hardware_revision_cb);
1389 dev_err(gasket_dev->dev,
1390 "Error getting hardware revision: %d\n", ret);
1393 gasket_dev->hardware_revision = ret;
1395 /* device_status_cb returns a device status, not an error code. */
1396 gasket_dev->status = gasket_get_hw_status(gasket_dev);
1397 if (gasket_dev->status == GASKET_STATUS_DEAD)
1398 dev_err(gasket_dev->dev, "Device reported as unhealthy.\n");
1400 ret = gasket_add_cdev(&gasket_dev->dev_info, &gasket_file_ops,
1401 driver_desc->module);
1407 EXPORT_SYMBOL(gasket_enable_device);
1409 static int __gasket_add_device(struct device *parent_dev,
1410 struct gasket_internal_desc *internal_desc,
1411 struct gasket_dev **gasket_devp)
1414 struct gasket_dev *gasket_dev;
1415 const struct gasket_driver_desc *driver_desc =
1416 internal_desc->driver_desc;
1418 ret = gasket_alloc_dev(internal_desc, parent_dev, &gasket_dev);
1421 if (IS_ERR(gasket_dev->dev_info.device)) {
1422 dev_err(parent_dev, "Cannot create %s device %s [ret = %ld]\n",
1423 driver_desc->name, gasket_dev->dev_info.name,
1424 PTR_ERR(gasket_dev->dev_info.device));
1426 goto free_gasket_dev;
1429 ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device,
1434 ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device,
1435 gasket_sysfs_generic_attrs);
1437 goto remove_sysfs_mapping;
1439 *gasket_devp = gasket_dev;
1442 remove_sysfs_mapping:
1443 gasket_sysfs_remove_mapping(gasket_dev->dev_info.device);
1445 device_destroy(internal_desc->class, gasket_dev->dev_info.devt);
1447 gasket_free_dev(gasket_dev);
1451 static void __gasket_remove_device(struct gasket_internal_desc *internal_desc,
1452 struct gasket_dev *gasket_dev)
1454 gasket_sysfs_remove_mapping(gasket_dev->dev_info.device);
1455 device_destroy(internal_desc->class, gasket_dev->dev_info.devt);
1456 gasket_free_dev(gasket_dev);
1460 * Add PCI gasket device.
1462 * Called by Gasket device probe function.
1463 * Allocates device metadata and maps device memory. The device driver must
1464 * call gasket_enable_device after driver init is complete to place the device
1467 int gasket_pci_add_device(struct pci_dev *pci_dev,
1468 struct gasket_dev **gasket_devp)
1471 struct gasket_internal_desc *internal_desc;
1472 struct gasket_dev *gasket_dev;
1473 struct device *parent;
1475 dev_dbg(&pci_dev->dev, "add PCI gasket device\n");
1477 mutex_lock(&g_mutex);
1478 internal_desc = lookup_pci_internal_desc(pci_dev);
1479 mutex_unlock(&g_mutex);
1480 if (!internal_desc) {
1481 dev_err(&pci_dev->dev,
1482 "PCI add device called for unknown driver type\n");
1486 parent = &pci_dev->dev;
1487 ret = __gasket_add_device(parent, internal_desc, &gasket_dev);
1491 gasket_dev->pci_dev = pci_dev;
1492 ret = gasket_setup_pci(pci_dev, gasket_dev);
1497 * Once we've created the mapping structures successfully, attempt to
1498 * create a symlink to the pci directory of this object.
1500 ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj,
1501 &pci_dev->dev.kobj, dev_name(&pci_dev->dev));
1503 dev_err(gasket_dev->dev,
1504 "Cannot create sysfs pci link: %d\n", ret);
1508 *gasket_devp = gasket_dev;
1512 gasket_cleanup_pci(gasket_dev);
1513 __gasket_remove_device(internal_desc, gasket_dev);
1516 EXPORT_SYMBOL(gasket_pci_add_device);
1518 /* Remove a PCI gasket device. */
1519 void gasket_pci_remove_device(struct pci_dev *pci_dev)
1522 struct gasket_internal_desc *internal_desc;
1523 struct gasket_dev *gasket_dev = NULL;
1524 /* Find the device desc. */
1525 mutex_lock(&g_mutex);
1526 internal_desc = lookup_pci_internal_desc(pci_dev);
1527 if (!internal_desc) {
1528 mutex_unlock(&g_mutex);
1531 mutex_unlock(&g_mutex);
1533 /* Now find the specific device */
1534 mutex_lock(&internal_desc->mutex);
1535 for (i = 0; i < GASKET_DEV_MAX; i++) {
1536 if (internal_desc->devs[i] &&
1537 internal_desc->devs[i]->pci_dev == pci_dev) {
1538 gasket_dev = internal_desc->devs[i];
1542 mutex_unlock(&internal_desc->mutex);
1547 dev_dbg(gasket_dev->dev, "remove %s PCI gasket device\n",
1548 internal_desc->driver_desc->name);
1550 gasket_cleanup_pci(gasket_dev);
1551 __gasket_remove_device(internal_desc, gasket_dev);
1553 EXPORT_SYMBOL(gasket_pci_remove_device);
1556 * Lookup a name by number in a num_name table.
1557 * @num: Number to lookup.
1558 * @table: Array of num_name structures, the table for the lookup.
1560 * Description: Searches for num in the table. If found, the
1561 * corresponding name is returned; otherwise NULL
1564 * The table must have a NULL name pointer at the end.
1566 const char *gasket_num_name_lookup(uint num,
1567 const struct gasket_num_name *table)
1571 while (table[i].snn_name) {
1572 if (num == table[i].snn_num)
1577 return table[i].snn_name;
1579 EXPORT_SYMBOL(gasket_num_name_lookup);
1581 int gasket_reset(struct gasket_dev *gasket_dev)
1585 mutex_lock(&gasket_dev->mutex);
1586 ret = gasket_reset_nolock(gasket_dev);
1587 mutex_unlock(&gasket_dev->mutex);
1590 EXPORT_SYMBOL(gasket_reset);
1592 int gasket_reset_nolock(struct gasket_dev *gasket_dev)
1596 const struct gasket_driver_desc *driver_desc;
1598 driver_desc = gasket_dev->internal_desc->driver_desc;
1599 if (!driver_desc->device_reset_cb)
1602 ret = driver_desc->device_reset_cb(gasket_dev);
1604 dev_dbg(gasket_dev->dev, "Device reset cb returned %d.\n",
1609 /* Reinitialize the page tables and interrupt framework. */
1610 for (i = 0; i < driver_desc->num_page_tables; ++i)
1611 gasket_page_table_reset(gasket_dev->page_table[i]);
1613 ret = gasket_interrupt_reinit(gasket_dev);
1615 dev_dbg(gasket_dev->dev, "Unable to reinit interrupts: %d.\n",
1620 /* Get current device health. */
1621 gasket_dev->status = gasket_get_hw_status(gasket_dev);
1622 if (gasket_dev->status == GASKET_STATUS_DEAD) {
1623 dev_dbg(gasket_dev->dev, "Device reported as dead.\n");
1629 EXPORT_SYMBOL(gasket_reset_nolock);
1631 gasket_ioctl_permissions_cb_t
1632 gasket_get_ioctl_permissions_cb(struct gasket_dev *gasket_dev)
1634 return gasket_dev->internal_desc->driver_desc->ioctl_permissions_cb;
1636 EXPORT_SYMBOL(gasket_get_ioctl_permissions_cb);
1638 /* Get the driver structure for a given gasket_dev.
1639 * @dev: pointer to gasket_dev, implementing the requested driver.
1641 const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev)
1643 return dev->internal_desc->driver_desc;
1646 /* Get the device structure for a given gasket_dev.
1647 * @dev: pointer to gasket_dev, implementing the requested driver.
1649 struct device *gasket_get_device(struct gasket_dev *dev)
1655 * Asynchronously waits on device.
1656 * @gasket_dev: Device struct.
1658 * @offset: Register offset
1659 * @mask: Register mask
1660 * @val: Expected value
1661 * @max_retries: number of sleep periods
1662 * @delay_ms: Timeout in milliseconds
1664 * Description: Busy waits for a specific combination of bits to be set on a
1667 int gasket_wait_with_reschedule(struct gasket_dev *gasket_dev, int bar,
1668 u64 offset, u64 mask, u64 val,
1669 uint max_retries, u64 delay_ms)
1674 while (retries < max_retries) {
1675 tmp = gasket_dev_read_64(gasket_dev, bar, offset);
1676 if ((tmp & mask) == val)
1681 dev_dbg(gasket_dev->dev, "%s timeout: reg %llx timeout (%llu ms)\n",
1682 __func__, offset, max_retries * delay_ms);
1685 EXPORT_SYMBOL(gasket_wait_with_reschedule);
1687 /* See gasket_core.h for description. */
1688 int gasket_register_device(const struct gasket_driver_desc *driver_desc)
1692 struct gasket_internal_desc *internal;
1694 pr_debug("Loading %s driver version %s\n", driver_desc->name,
1695 driver_desc->driver_version);
1696 /* Check for duplicates and find a free slot. */
1697 mutex_lock(&g_mutex);
1699 for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
1700 if (g_descs[i].driver_desc == driver_desc) {
1701 pr_err("%s driver already loaded/registered\n",
1703 mutex_unlock(&g_mutex);
1708 /* This and the above loop could be combined, but this reads easier. */
1709 for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
1710 if (!g_descs[i].driver_desc) {
1711 g_descs[i].driver_desc = driver_desc;
1716 mutex_unlock(&g_mutex);
1718 if (desc_idx == -1) {
1719 pr_err("too many drivers loaded, max %d\n",
1720 GASKET_FRAMEWORK_DESC_MAX);
1724 internal = &g_descs[desc_idx];
1725 mutex_init(&internal->mutex);
1726 memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX);
1728 class_create(driver_desc->module, driver_desc->name);
1730 if (IS_ERR(internal->class)) {
1731 pr_err("Cannot register %s class [ret=%ld]\n",
1732 driver_desc->name, PTR_ERR(internal->class));
1733 ret = PTR_ERR(internal->class);
1734 goto unregister_gasket_driver;
1737 ret = register_chrdev_region(MKDEV(driver_desc->major,
1738 driver_desc->minor), GASKET_DEV_MAX,
1741 pr_err("cannot register %s char driver [ret=%d]\n",
1742 driver_desc->name, ret);
1749 class_destroy(internal->class);
1751 unregister_gasket_driver:
1752 mutex_lock(&g_mutex);
1753 g_descs[desc_idx].driver_desc = NULL;
1754 mutex_unlock(&g_mutex);
1757 EXPORT_SYMBOL(gasket_register_device);
1759 /* See gasket_core.h for description. */
1760 void gasket_unregister_device(const struct gasket_driver_desc *driver_desc)
1763 struct gasket_internal_desc *internal_desc = NULL;
1765 mutex_lock(&g_mutex);
1766 for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
1767 if (g_descs[i].driver_desc == driver_desc) {
1768 internal_desc = &g_descs[i];
1774 if (!internal_desc) {
1775 mutex_unlock(&g_mutex);
1776 pr_err("request to unregister unknown desc: %s, %d:%d\n",
1777 driver_desc->name, driver_desc->major,
1778 driver_desc->minor);
1782 unregister_chrdev_region(MKDEV(driver_desc->major, driver_desc->minor),
1785 class_destroy(internal_desc->class);
1787 /* Finally, effectively "remove" the driver. */
1788 g_descs[desc_idx].driver_desc = NULL;
1789 mutex_unlock(&g_mutex);
1791 pr_debug("removed %s driver\n", driver_desc->name);
1793 EXPORT_SYMBOL(gasket_unregister_device);
1795 static int __init gasket_init(void)
1799 mutex_lock(&g_mutex);
1800 for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
1801 g_descs[i].driver_desc = NULL;
1802 mutex_init(&g_descs[i].mutex);
1805 gasket_sysfs_init();
1807 mutex_unlock(&g_mutex);
1811 MODULE_DESCRIPTION("Google Gasket driver framework");
1812 MODULE_VERSION(GASKET_FRAMEWORK_VERSION);
1813 MODULE_LICENSE("GPL v2");
1814 MODULE_AUTHOR("Rob Springer <rspringer@google.com>");
1815 module_init(gasket_init);