2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/string.h>
36 #include <linux/errno.h>
37 #include <linux/kernel.h>
38 #include <linux/slab.h>
39 #include <linux/init.h>
40 #include <linux/mutex.h>
41 #include <linux/netdevice.h>
42 #include <linux/security.h>
43 #include <linux/notifier.h>
44 #include <rdma/rdma_netlink.h>
45 #include <rdma/ib_addr.h>
46 #include <rdma/ib_cache.h>
48 #include "core_priv.h"
50 MODULE_AUTHOR("Roland Dreier");
51 MODULE_DESCRIPTION("core kernel InfiniBand API");
52 MODULE_LICENSE("Dual BSD/GPL");
54 struct ib_client_data {
55 struct list_head list;
56 struct ib_client *client;
58 /* The device or client is going down. Do not call client or device
59 * callbacks other than remove(). */
63 struct workqueue_struct *ib_comp_wq;
64 struct workqueue_struct *ib_comp_unbound_wq;
65 struct workqueue_struct *ib_wq;
66 EXPORT_SYMBOL_GPL(ib_wq);
68 /* The device_list and client_list contain devices and clients after their
69 * registration has completed, and the devices and clients are removed
70 * during unregistration. */
71 static LIST_HEAD(device_list);
72 static LIST_HEAD(client_list);
75 * device_mutex and lists_rwsem protect access to both device_list and
76 * client_list. device_mutex protects writer access by device and client
77 * registration / de-registration. lists_rwsem protects reader access to
78 * these lists. Iterators of these lists must lock it for read, while updates
79 * to the lists must be done with a write lock. A special case is when the
80 * device_mutex is locked. In this case locking the lists for read access is
81 * not necessary as the device_mutex implies it.
83 * lists_rwsem also protects access to the client data list.
85 static DEFINE_MUTEX(device_mutex);
86 static DECLARE_RWSEM(lists_rwsem);
88 static int ib_security_change(struct notifier_block *nb, unsigned long event,
90 static void ib_policy_change_task(struct work_struct *work);
91 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task);
93 static struct notifier_block ibdev_lsm_nb = {
94 .notifier_call = ib_security_change,
97 static int ib_device_check_mandatory(struct ib_device *device)
99 #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }
100 static const struct {
103 } mandatory_table[] = {
104 IB_MANDATORY_FUNC(query_device),
105 IB_MANDATORY_FUNC(query_port),
106 IB_MANDATORY_FUNC(query_pkey),
107 IB_MANDATORY_FUNC(alloc_pd),
108 IB_MANDATORY_FUNC(dealloc_pd),
109 IB_MANDATORY_FUNC(create_qp),
110 IB_MANDATORY_FUNC(modify_qp),
111 IB_MANDATORY_FUNC(destroy_qp),
112 IB_MANDATORY_FUNC(post_send),
113 IB_MANDATORY_FUNC(post_recv),
114 IB_MANDATORY_FUNC(create_cq),
115 IB_MANDATORY_FUNC(destroy_cq),
116 IB_MANDATORY_FUNC(poll_cq),
117 IB_MANDATORY_FUNC(req_notify_cq),
118 IB_MANDATORY_FUNC(get_dma_mr),
119 IB_MANDATORY_FUNC(dereg_mr),
120 IB_MANDATORY_FUNC(get_port_immutable)
124 for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) {
125 if (!*(void **) ((void *) &device->ops +
126 mandatory_table[i].offset)) {
127 dev_warn(&device->dev,
128 "Device is missing mandatory function %s\n",
129 mandatory_table[i].name);
137 static struct ib_device *__ib_device_get_by_index(u32 index)
139 struct ib_device *device;
141 list_for_each_entry(device, &device_list, core_list)
142 if (device->index == index)
149 * Caller must perform ib_device_put() to return the device reference count
150 * when ib_device_get_by_index() returns valid device pointer.
152 struct ib_device *ib_device_get_by_index(u32 index)
154 struct ib_device *device;
156 down_read(&lists_rwsem);
157 device = __ib_device_get_by_index(index);
159 if (!ib_device_try_get(device))
162 up_read(&lists_rwsem);
167 * ib_device_put - Release IB device reference
168 * @device: device whose reference to be released
170 * ib_device_put() releases reference to the IB device to allow it to be
171 * unregistered and eventually free.
173 void ib_device_put(struct ib_device *device)
175 if (refcount_dec_and_test(&device->refcount))
176 complete(&device->unreg_completion);
178 EXPORT_SYMBOL(ib_device_put);
180 static struct ib_device *__ib_device_get_by_name(const char *name)
182 struct ib_device *device;
184 list_for_each_entry(device, &device_list, core_list)
185 if (!strcmp(name, dev_name(&device->dev)))
191 int ib_device_rename(struct ib_device *ibdev, const char *name)
193 struct ib_device *device;
196 if (!strcmp(name, dev_name(&ibdev->dev)))
199 mutex_lock(&device_mutex);
200 list_for_each_entry(device, &device_list, core_list) {
201 if (!strcmp(name, dev_name(&device->dev))) {
207 ret = device_rename(&ibdev->dev, name);
210 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
212 mutex_unlock(&device_mutex);
216 static int alloc_name(struct ib_device *ibdev, const char *name)
218 unsigned long *inuse;
219 struct ib_device *device;
222 inuse = (unsigned long *) get_zeroed_page(GFP_KERNEL);
226 list_for_each_entry(device, &device_list, core_list) {
227 char buf[IB_DEVICE_NAME_MAX];
229 if (sscanf(dev_name(&device->dev), name, &i) != 1)
231 if (i < 0 || i >= PAGE_SIZE * 8)
233 snprintf(buf, sizeof buf, name, i);
234 if (!strcmp(buf, dev_name(&device->dev)))
238 i = find_first_zero_bit(inuse, PAGE_SIZE * 8);
239 free_page((unsigned long) inuse);
241 return dev_set_name(&ibdev->dev, name, i);
244 static void ib_device_release(struct device *device)
246 struct ib_device *dev = container_of(device, struct ib_device, dev);
248 WARN_ON(dev->reg_state == IB_DEV_REGISTERED);
249 if (dev->reg_state == IB_DEV_UNREGISTERED) {
251 * In IB_DEV_UNINITIALIZED state, cache or port table
252 * is not even created. Free cache and port table only when
253 * device reaches UNREGISTERED state.
255 ib_cache_release_one(dev);
256 kfree(dev->port_immutable);
261 static int ib_device_uevent(struct device *device,
262 struct kobj_uevent_env *env)
264 if (add_uevent_var(env, "NAME=%s", dev_name(device)))
268 * It would be nice to pass the node GUID with the event...
274 static struct class ib_class = {
275 .name = "infiniband",
276 .dev_release = ib_device_release,
277 .dev_uevent = ib_device_uevent,
281 * ib_alloc_device - allocate an IB device struct
282 * @size:size of structure to allocate
284 * Low-level drivers should use ib_alloc_device() to allocate &struct
285 * ib_device. @size is the size of the structure to be allocated,
286 * including any private data used by the low-level driver.
287 * ib_dealloc_device() must be used to free structures allocated with
290 struct ib_device *ib_alloc_device(size_t size)
292 struct ib_device *device;
294 if (WARN_ON(size < sizeof(struct ib_device)))
297 device = kzalloc(size, GFP_KERNEL);
301 rdma_restrack_init(&device->res);
303 device->dev.class = &ib_class;
304 device_initialize(&device->dev);
306 dev_set_drvdata(&device->dev, device);
308 INIT_LIST_HEAD(&device->event_handler_list);
309 spin_lock_init(&device->event_handler_lock);
310 rwlock_init(&device->client_data_lock);
311 INIT_LIST_HEAD(&device->client_data_list);
312 INIT_LIST_HEAD(&device->port_list);
313 init_completion(&device->unreg_completion);
317 EXPORT_SYMBOL(ib_alloc_device);
320 * ib_dealloc_device - free an IB device struct
321 * @device:structure to free
323 * Free a structure allocated with ib_alloc_device().
325 void ib_dealloc_device(struct ib_device *device)
327 WARN_ON(!list_empty(&device->client_data_list));
328 WARN_ON(device->reg_state != IB_DEV_UNREGISTERED &&
329 device->reg_state != IB_DEV_UNINITIALIZED);
330 rdma_restrack_clean(&device->res);
331 put_device(&device->dev);
333 EXPORT_SYMBOL(ib_dealloc_device);
335 static int add_client_context(struct ib_device *device, struct ib_client *client)
337 struct ib_client_data *context;
339 context = kmalloc(sizeof(*context), GFP_KERNEL);
343 context->client = client;
344 context->data = NULL;
345 context->going_down = false;
347 down_write(&lists_rwsem);
348 write_lock_irq(&device->client_data_lock);
349 list_add(&context->list, &device->client_data_list);
350 write_unlock_irq(&device->client_data_lock);
351 up_write(&lists_rwsem);
356 static int verify_immutable(const struct ib_device *dev, u8 port)
358 return WARN_ON(!rdma_cap_ib_mad(dev, port) &&
359 rdma_max_mad_size(dev, port) != 0);
362 static int read_port_immutable(struct ib_device *device)
365 u8 start_port = rdma_start_port(device);
366 u8 end_port = rdma_end_port(device);
370 * device->port_immutable is indexed directly by the port number to make
371 * access to this data as efficient as possible.
373 * Therefore port_immutable is declared as a 1 based array with
374 * potential empty slots at the beginning.
376 device->port_immutable = kcalloc(end_port + 1,
377 sizeof(*device->port_immutable),
379 if (!device->port_immutable)
382 for (port = start_port; port <= end_port; ++port) {
383 ret = device->ops.get_port_immutable(
384 device, port, &device->port_immutable[port]);
388 if (verify_immutable(device, port))
394 void ib_get_device_fw_str(struct ib_device *dev, char *str)
396 if (dev->ops.get_dev_fw_str)
397 dev->ops.get_dev_fw_str(dev, str);
401 EXPORT_SYMBOL(ib_get_device_fw_str);
403 static int setup_port_pkey_list(struct ib_device *device)
408 * device->port_pkey_list is indexed directly by the port number,
409 * Therefore it is declared as a 1 based array with potential empty
410 * slots at the beginning.
412 device->port_pkey_list = kcalloc(rdma_end_port(device) + 1,
413 sizeof(*device->port_pkey_list),
416 if (!device->port_pkey_list)
419 for (i = 0; i < (rdma_end_port(device) + 1); i++) {
420 spin_lock_init(&device->port_pkey_list[i].list_lock);
421 INIT_LIST_HEAD(&device->port_pkey_list[i].pkey_list);
427 static void ib_policy_change_task(struct work_struct *work)
429 struct ib_device *dev;
431 down_read(&lists_rwsem);
432 list_for_each_entry(dev, &device_list, core_list) {
435 for (i = rdma_start_port(dev); i <= rdma_end_port(dev); i++) {
437 int ret = ib_get_cached_subnet_prefix(dev,
442 "ib_get_cached_subnet_prefix err: %d, this should never happen here\n",
445 ib_security_cache_change(dev, i, sp);
448 up_read(&lists_rwsem);
451 static int ib_security_change(struct notifier_block *nb, unsigned long event,
454 if (event != LSM_POLICY_CHANGE)
457 schedule_work(&ib_policy_change_work);
463 * __dev_new_index - allocate an device index
465 * Returns a suitable unique value for a new device interface
466 * number. It assumes that there are less than 2^32-1 ib devices
467 * will be present in the system.
469 static u32 __dev_new_index(void)
472 * The device index to allow stable naming.
473 * Similar to struct net -> ifindex.
481 if (!__ib_device_get_by_index(index))
486 static void setup_dma_device(struct ib_device *device)
488 struct device *parent = device->dev.parent;
490 WARN_ON_ONCE(device->dma_device);
491 if (device->dev.dma_ops) {
493 * The caller provided custom DMA operations. Copy the
494 * DMA-related fields that are used by e.g. dma_alloc_coherent()
497 device->dma_device = &device->dev;
498 if (!device->dev.dma_mask) {
500 device->dev.dma_mask = parent->dma_mask;
504 if (!device->dev.coherent_dma_mask) {
506 device->dev.coherent_dma_mask =
507 parent->coherent_dma_mask;
513 * The caller did not provide custom DMA operations. Use the
514 * DMA mapping operations of the parent device.
516 WARN_ON_ONCE(!parent);
517 device->dma_device = parent;
521 static void cleanup_device(struct ib_device *device)
523 ib_cache_cleanup_one(device);
524 ib_cache_release_one(device);
525 kfree(device->port_pkey_list);
526 kfree(device->port_immutable);
529 static int setup_device(struct ib_device *device)
531 struct ib_udata uhw = {.outlen = 0, .inlen = 0};
534 ret = ib_device_check_mandatory(device);
538 ret = read_port_immutable(device);
540 dev_warn(&device->dev,
541 "Couldn't create per port immutable data\n");
545 memset(&device->attrs, 0, sizeof(device->attrs));
546 ret = device->ops.query_device(device, &device->attrs, &uhw);
548 dev_warn(&device->dev,
549 "Couldn't query the device attributes\n");
553 ret = setup_port_pkey_list(device);
555 dev_warn(&device->dev, "Couldn't create per port_pkey_list\n");
559 ret = ib_cache_setup_one(device);
561 dev_warn(&device->dev,
562 "Couldn't set up InfiniBand P_Key/GID cache\n");
568 kfree(device->port_pkey_list);
570 kfree(device->port_immutable);
575 * ib_register_device - Register an IB device with IB core
576 * @device:Device to register
578 * Low-level drivers use ib_register_device() to register their
579 * devices with the IB core. All registered clients will receive a
580 * callback for each device that is added. @device must be allocated
581 * with ib_alloc_device().
583 int ib_register_device(struct ib_device *device, const char *name,
584 int (*port_callback)(struct ib_device *, u8,
588 struct ib_client *client;
590 setup_dma_device(device);
592 mutex_lock(&device_mutex);
594 if (strchr(name, '%')) {
595 ret = alloc_name(device, name);
599 ret = dev_set_name(&device->dev, name);
603 if (__ib_device_get_by_name(dev_name(&device->dev))) {
607 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
609 ret = setup_device(device);
613 device->index = __dev_new_index();
615 ret = ib_device_register_rdmacg(device);
617 dev_warn(&device->dev,
618 "Couldn't register device with rdma cgroup\n");
622 ret = ib_device_register_sysfs(device, port_callback);
624 dev_warn(&device->dev,
625 "Couldn't register device with driver model\n");
629 refcount_set(&device->refcount, 1);
630 device->reg_state = IB_DEV_REGISTERED;
632 list_for_each_entry(client, &client_list, list)
633 if (!add_client_context(device, client) && client->add)
636 down_write(&lists_rwsem);
637 list_add_tail(&device->core_list, &device_list);
638 up_write(&lists_rwsem);
639 mutex_unlock(&device_mutex);
643 ib_device_unregister_rdmacg(device);
645 cleanup_device(device);
647 mutex_unlock(&device_mutex);
650 EXPORT_SYMBOL(ib_register_device);
653 * ib_unregister_device - Unregister an IB device
654 * @device:Device to unregister
656 * Unregister an IB device. All clients will receive a remove callback.
658 void ib_unregister_device(struct ib_device *device)
660 struct ib_client_data *context, *tmp;
664 * Wait for all netlink command callers to finish working on the
667 ib_device_put(device);
668 wait_for_completion(&device->unreg_completion);
670 mutex_lock(&device_mutex);
672 down_write(&lists_rwsem);
673 list_del(&device->core_list);
674 write_lock_irq(&device->client_data_lock);
675 list_for_each_entry(context, &device->client_data_list, list)
676 context->going_down = true;
677 write_unlock_irq(&device->client_data_lock);
678 downgrade_write(&lists_rwsem);
680 list_for_each_entry(context, &device->client_data_list, list) {
681 if (context->client->remove)
682 context->client->remove(device, context->data);
684 up_read(&lists_rwsem);
686 ib_device_unregister_sysfs(device);
687 ib_device_unregister_rdmacg(device);
689 mutex_unlock(&device_mutex);
691 ib_cache_cleanup_one(device);
693 ib_security_destroy_port_pkey_list(device);
694 kfree(device->port_pkey_list);
696 down_write(&lists_rwsem);
697 write_lock_irqsave(&device->client_data_lock, flags);
698 list_for_each_entry_safe(context, tmp, &device->client_data_list,
700 list_del(&context->list);
703 write_unlock_irqrestore(&device->client_data_lock, flags);
704 up_write(&lists_rwsem);
706 device->reg_state = IB_DEV_UNREGISTERED;
708 EXPORT_SYMBOL(ib_unregister_device);
711 * ib_register_client - Register an IB client
712 * @client:Client to register
714 * Upper level users of the IB drivers can use ib_register_client() to
715 * register callbacks for IB device addition and removal. When an IB
716 * device is added, each registered client's add method will be called
717 * (in the order the clients were registered), and when a device is
718 * removed, each client's remove method will be called (in the reverse
719 * order that clients were registered). In addition, when
720 * ib_register_client() is called, the client will receive an add
721 * callback for all devices already registered.
723 int ib_register_client(struct ib_client *client)
725 struct ib_device *device;
727 mutex_lock(&device_mutex);
729 list_for_each_entry(device, &device_list, core_list)
730 if (!add_client_context(device, client) && client->add)
733 down_write(&lists_rwsem);
734 list_add_tail(&client->list, &client_list);
735 up_write(&lists_rwsem);
737 mutex_unlock(&device_mutex);
741 EXPORT_SYMBOL(ib_register_client);
744 * ib_unregister_client - Unregister an IB client
745 * @client:Client to unregister
747 * Upper level users use ib_unregister_client() to remove their client
748 * registration. When ib_unregister_client() is called, the client
749 * will receive a remove callback for each IB device still registered.
751 void ib_unregister_client(struct ib_client *client)
753 struct ib_client_data *context;
754 struct ib_device *device;
756 mutex_lock(&device_mutex);
758 down_write(&lists_rwsem);
759 list_del(&client->list);
760 up_write(&lists_rwsem);
762 list_for_each_entry(device, &device_list, core_list) {
763 struct ib_client_data *found_context = NULL;
765 down_write(&lists_rwsem);
766 write_lock_irq(&device->client_data_lock);
767 list_for_each_entry(context, &device->client_data_list, list)
768 if (context->client == client) {
769 context->going_down = true;
770 found_context = context;
773 write_unlock_irq(&device->client_data_lock);
774 up_write(&lists_rwsem);
777 client->remove(device, found_context ?
778 found_context->data : NULL);
780 if (!found_context) {
781 dev_warn(&device->dev,
782 "No client context found for %s\n",
787 down_write(&lists_rwsem);
788 write_lock_irq(&device->client_data_lock);
789 list_del(&found_context->list);
790 write_unlock_irq(&device->client_data_lock);
791 up_write(&lists_rwsem);
792 kfree(found_context);
795 mutex_unlock(&device_mutex);
797 EXPORT_SYMBOL(ib_unregister_client);
800 * ib_get_client_data - Get IB client context
801 * @device:Device to get context for
802 * @client:Client to get context for
804 * ib_get_client_data() returns client context set with
805 * ib_set_client_data().
807 void *ib_get_client_data(struct ib_device *device, struct ib_client *client)
809 struct ib_client_data *context;
813 read_lock_irqsave(&device->client_data_lock, flags);
814 list_for_each_entry(context, &device->client_data_list, list)
815 if (context->client == client) {
819 read_unlock_irqrestore(&device->client_data_lock, flags);
823 EXPORT_SYMBOL(ib_get_client_data);
826 * ib_set_client_data - Set IB client context
827 * @device:Device to set context for
828 * @client:Client to set context for
829 * @data:Context to set
831 * ib_set_client_data() sets client context that can be retrieved with
832 * ib_get_client_data().
834 void ib_set_client_data(struct ib_device *device, struct ib_client *client,
837 struct ib_client_data *context;
840 write_lock_irqsave(&device->client_data_lock, flags);
841 list_for_each_entry(context, &device->client_data_list, list)
842 if (context->client == client) {
843 context->data = data;
847 dev_warn(&device->dev, "No client context found for %s\n",
851 write_unlock_irqrestore(&device->client_data_lock, flags);
853 EXPORT_SYMBOL(ib_set_client_data);
856 * ib_register_event_handler - Register an IB event handler
857 * @event_handler:Handler to register
859 * ib_register_event_handler() registers an event handler that will be
860 * called back when asynchronous IB events occur (as defined in
861 * chapter 11 of the InfiniBand Architecture Specification). This
862 * callback may occur in interrupt context.
864 void ib_register_event_handler(struct ib_event_handler *event_handler)
868 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags);
869 list_add_tail(&event_handler->list,
870 &event_handler->device->event_handler_list);
871 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags);
873 EXPORT_SYMBOL(ib_register_event_handler);
876 * ib_unregister_event_handler - Unregister an event handler
877 * @event_handler:Handler to unregister
879 * Unregister an event handler registered with
880 * ib_register_event_handler().
882 void ib_unregister_event_handler(struct ib_event_handler *event_handler)
886 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags);
887 list_del(&event_handler->list);
888 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags);
890 EXPORT_SYMBOL(ib_unregister_event_handler);
893 * ib_dispatch_event - Dispatch an asynchronous event
894 * @event:Event to dispatch
896 * Low-level drivers must call ib_dispatch_event() to dispatch the
897 * event to all registered event handlers when an asynchronous event
900 void ib_dispatch_event(struct ib_event *event)
903 struct ib_event_handler *handler;
905 spin_lock_irqsave(&event->device->event_handler_lock, flags);
907 list_for_each_entry(handler, &event->device->event_handler_list, list)
908 handler->handler(handler, event);
910 spin_unlock_irqrestore(&event->device->event_handler_lock, flags);
912 EXPORT_SYMBOL(ib_dispatch_event);
915 * ib_query_port - Query IB port attributes
916 * @device:Device to query
917 * @port_num:Port number to query
918 * @port_attr:Port attributes
920 * ib_query_port() returns the attributes of a port through the
921 * @port_attr pointer.
923 int ib_query_port(struct ib_device *device,
925 struct ib_port_attr *port_attr)
930 if (!rdma_is_port_valid(device, port_num))
933 memset(port_attr, 0, sizeof(*port_attr));
934 err = device->ops.query_port(device, port_num, port_attr);
935 if (err || port_attr->subnet_prefix)
938 if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND)
941 err = device->ops.query_gid(device, port_num, 0, &gid);
945 port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix);
948 EXPORT_SYMBOL(ib_query_port);
951 * ib_enum_roce_netdev - enumerate all RoCE ports
952 * @ib_dev : IB device we want to query
953 * @filter: Should we call the callback?
954 * @filter_cookie: Cookie passed to filter
955 * @cb: Callback to call for each found RoCE ports
956 * @cookie: Cookie passed back to the callback
958 * Enumerates all of the physical RoCE ports of ib_dev
959 * which are related to netdevice and calls callback() on each
960 * device for which filter() function returns non zero.
962 void ib_enum_roce_netdev(struct ib_device *ib_dev,
963 roce_netdev_filter filter,
965 roce_netdev_callback cb,
970 for (port = rdma_start_port(ib_dev); port <= rdma_end_port(ib_dev);
972 if (rdma_protocol_roce(ib_dev, port)) {
973 struct net_device *idev = NULL;
975 if (ib_dev->ops.get_netdev)
976 idev = ib_dev->ops.get_netdev(ib_dev, port);
979 idev->reg_state >= NETREG_UNREGISTERED) {
984 if (filter(ib_dev, port, idev, filter_cookie))
985 cb(ib_dev, port, idev, cookie);
993 * ib_enum_all_roce_netdevs - enumerate all RoCE devices
994 * @filter: Should we call the callback?
995 * @filter_cookie: Cookie passed to filter
996 * @cb: Callback to call for each found RoCE ports
997 * @cookie: Cookie passed back to the callback
999 * Enumerates all RoCE devices' physical ports which are related
1000 * to netdevices and calls callback() on each device for which
1001 * filter() function returns non zero.
1003 void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
1004 void *filter_cookie,
1005 roce_netdev_callback cb,
1008 struct ib_device *dev;
1010 down_read(&lists_rwsem);
1011 list_for_each_entry(dev, &device_list, core_list)
1012 ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie);
1013 up_read(&lists_rwsem);
1017 * ib_enum_all_devs - enumerate all ib_devices
1018 * @cb: Callback to call for each found ib_device
1020 * Enumerates all ib_devices and calls callback() on each device.
1022 int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
1023 struct netlink_callback *cb)
1025 struct ib_device *dev;
1026 unsigned int idx = 0;
1029 down_read(&lists_rwsem);
1030 list_for_each_entry(dev, &device_list, core_list) {
1031 ret = nldev_cb(dev, skb, cb, idx);
1037 up_read(&lists_rwsem);
1042 * ib_query_pkey - Get P_Key table entry
1043 * @device:Device to query
1044 * @port_num:Port number to query
1045 * @index:P_Key table index to query
1046 * @pkey:Returned P_Key
1048 * ib_query_pkey() fetches the specified P_Key table entry.
1050 int ib_query_pkey(struct ib_device *device,
1051 u8 port_num, u16 index, u16 *pkey)
1053 if (!rdma_is_port_valid(device, port_num))
1056 return device->ops.query_pkey(device, port_num, index, pkey);
1058 EXPORT_SYMBOL(ib_query_pkey);
1061 * ib_modify_device - Change IB device attributes
1062 * @device:Device to modify
1063 * @device_modify_mask:Mask of attributes to change
1064 * @device_modify:New attribute values
1066 * ib_modify_device() changes a device's attributes as specified by
1067 * the @device_modify_mask and @device_modify structure.
1069 int ib_modify_device(struct ib_device *device,
1070 int device_modify_mask,
1071 struct ib_device_modify *device_modify)
1073 if (!device->ops.modify_device)
1076 return device->ops.modify_device(device, device_modify_mask,
1079 EXPORT_SYMBOL(ib_modify_device);
1082 * ib_modify_port - Modifies the attributes for the specified port.
1083 * @device: The device to modify.
1084 * @port_num: The number of the port to modify.
1085 * @port_modify_mask: Mask used to specify which attributes of the port
1087 * @port_modify: New attribute values for the port.
1089 * ib_modify_port() changes a port's attributes as specified by the
1090 * @port_modify_mask and @port_modify structure.
1092 int ib_modify_port(struct ib_device *device,
1093 u8 port_num, int port_modify_mask,
1094 struct ib_port_modify *port_modify)
1098 if (!rdma_is_port_valid(device, port_num))
1101 if (device->ops.modify_port)
1102 rc = device->ops.modify_port(device, port_num,
1106 rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS;
1109 EXPORT_SYMBOL(ib_modify_port);
1112 * ib_find_gid - Returns the port number and GID table index where
1113 * a specified GID value occurs. Its searches only for IB link layer.
1114 * @device: The device to query.
1115 * @gid: The GID value to search for.
1116 * @port_num: The port number of the device where the GID value was found.
1117 * @index: The index into the GID table where the GID was found. This
1118 * parameter may be NULL.
1120 int ib_find_gid(struct ib_device *device, union ib_gid *gid,
1121 u8 *port_num, u16 *index)
1123 union ib_gid tmp_gid;
1126 for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) {
1127 if (!rdma_protocol_ib(device, port))
1130 for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) {
1131 ret = rdma_query_gid(device, port, i, &tmp_gid);
1134 if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
1145 EXPORT_SYMBOL(ib_find_gid);
1148 * ib_find_pkey - Returns the PKey table index where a specified
1149 * PKey value occurs.
1150 * @device: The device to query.
1151 * @port_num: The port number of the device to search for the PKey.
1152 * @pkey: The PKey value to search for.
1153 * @index: The index into the PKey table where the PKey was found.
1155 int ib_find_pkey(struct ib_device *device,
1156 u8 port_num, u16 pkey, u16 *index)
1160 int partial_ix = -1;
1162 for (i = 0; i < device->port_immutable[port_num].pkey_tbl_len; ++i) {
1163 ret = ib_query_pkey(device, port_num, i, &tmp_pkey);
1166 if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) {
1167 /* if there is full-member pkey take it.*/
1168 if (tmp_pkey & 0x8000) {
1177 /*no full-member, if exists take the limited*/
1178 if (partial_ix >= 0) {
1179 *index = partial_ix;
1184 EXPORT_SYMBOL(ib_find_pkey);
1187 * ib_get_net_dev_by_params() - Return the appropriate net_dev
1188 * for a received CM request
1189 * @dev: An RDMA device on which the request has been received.
1190 * @port: Port number on the RDMA device.
1191 * @pkey: The Pkey the request came on.
1192 * @gid: A GID that the net_dev uses to communicate.
1193 * @addr: Contains the IP address that the request specified as its
1196 struct net_device *ib_get_net_dev_by_params(struct ib_device *dev,
1199 const union ib_gid *gid,
1200 const struct sockaddr *addr)
1202 struct net_device *net_dev = NULL;
1203 struct ib_client_data *context;
1205 if (!rdma_protocol_ib(dev, port))
1208 down_read(&lists_rwsem);
1210 list_for_each_entry(context, &dev->client_data_list, list) {
1211 struct ib_client *client = context->client;
1213 if (context->going_down)
1216 if (client->get_net_dev_by_params) {
1217 net_dev = client->get_net_dev_by_params(dev, port, pkey,
1225 up_read(&lists_rwsem);
1229 EXPORT_SYMBOL(ib_get_net_dev_by_params);
1231 void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
1233 struct ib_device_ops *dev_ops = &dev->ops;
1234 #define SET_DEVICE_OP(ptr, name) \
1237 if (!((ptr)->name)) \
1238 (ptr)->name = ops->name; \
1241 SET_DEVICE_OP(dev_ops, add_gid);
1242 SET_DEVICE_OP(dev_ops, advise_mr);
1243 SET_DEVICE_OP(dev_ops, alloc_dm);
1244 SET_DEVICE_OP(dev_ops, alloc_fmr);
1245 SET_DEVICE_OP(dev_ops, alloc_hw_stats);
1246 SET_DEVICE_OP(dev_ops, alloc_mr);
1247 SET_DEVICE_OP(dev_ops, alloc_mw);
1248 SET_DEVICE_OP(dev_ops, alloc_pd);
1249 SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);
1250 SET_DEVICE_OP(dev_ops, alloc_ucontext);
1251 SET_DEVICE_OP(dev_ops, alloc_xrcd);
1252 SET_DEVICE_OP(dev_ops, attach_mcast);
1253 SET_DEVICE_OP(dev_ops, check_mr_status);
1254 SET_DEVICE_OP(dev_ops, create_ah);
1255 SET_DEVICE_OP(dev_ops, create_counters);
1256 SET_DEVICE_OP(dev_ops, create_cq);
1257 SET_DEVICE_OP(dev_ops, create_flow);
1258 SET_DEVICE_OP(dev_ops, create_flow_action_esp);
1259 SET_DEVICE_OP(dev_ops, create_qp);
1260 SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
1261 SET_DEVICE_OP(dev_ops, create_srq);
1262 SET_DEVICE_OP(dev_ops, create_wq);
1263 SET_DEVICE_OP(dev_ops, dealloc_dm);
1264 SET_DEVICE_OP(dev_ops, dealloc_fmr);
1265 SET_DEVICE_OP(dev_ops, dealloc_mw);
1266 SET_DEVICE_OP(dev_ops, dealloc_pd);
1267 SET_DEVICE_OP(dev_ops, dealloc_ucontext);
1268 SET_DEVICE_OP(dev_ops, dealloc_xrcd);
1269 SET_DEVICE_OP(dev_ops, del_gid);
1270 SET_DEVICE_OP(dev_ops, dereg_mr);
1271 SET_DEVICE_OP(dev_ops, destroy_ah);
1272 SET_DEVICE_OP(dev_ops, destroy_counters);
1273 SET_DEVICE_OP(dev_ops, destroy_cq);
1274 SET_DEVICE_OP(dev_ops, destroy_flow);
1275 SET_DEVICE_OP(dev_ops, destroy_flow_action);
1276 SET_DEVICE_OP(dev_ops, destroy_qp);
1277 SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table);
1278 SET_DEVICE_OP(dev_ops, destroy_srq);
1279 SET_DEVICE_OP(dev_ops, destroy_wq);
1280 SET_DEVICE_OP(dev_ops, detach_mcast);
1281 SET_DEVICE_OP(dev_ops, disassociate_ucontext);
1282 SET_DEVICE_OP(dev_ops, drain_rq);
1283 SET_DEVICE_OP(dev_ops, drain_sq);
1284 SET_DEVICE_OP(dev_ops, get_dev_fw_str);
1285 SET_DEVICE_OP(dev_ops, get_dma_mr);
1286 SET_DEVICE_OP(dev_ops, get_hw_stats);
1287 SET_DEVICE_OP(dev_ops, get_link_layer);
1288 SET_DEVICE_OP(dev_ops, get_netdev);
1289 SET_DEVICE_OP(dev_ops, get_port_immutable);
1290 SET_DEVICE_OP(dev_ops, get_vector_affinity);
1291 SET_DEVICE_OP(dev_ops, get_vf_config);
1292 SET_DEVICE_OP(dev_ops, get_vf_stats);
1293 SET_DEVICE_OP(dev_ops, map_mr_sg);
1294 SET_DEVICE_OP(dev_ops, map_phys_fmr);
1295 SET_DEVICE_OP(dev_ops, mmap);
1296 SET_DEVICE_OP(dev_ops, modify_ah);
1297 SET_DEVICE_OP(dev_ops, modify_cq);
1298 SET_DEVICE_OP(dev_ops, modify_device);
1299 SET_DEVICE_OP(dev_ops, modify_flow_action_esp);
1300 SET_DEVICE_OP(dev_ops, modify_port);
1301 SET_DEVICE_OP(dev_ops, modify_qp);
1302 SET_DEVICE_OP(dev_ops, modify_srq);
1303 SET_DEVICE_OP(dev_ops, modify_wq);
1304 SET_DEVICE_OP(dev_ops, peek_cq);
1305 SET_DEVICE_OP(dev_ops, poll_cq);
1306 SET_DEVICE_OP(dev_ops, post_recv);
1307 SET_DEVICE_OP(dev_ops, post_send);
1308 SET_DEVICE_OP(dev_ops, post_srq_recv);
1309 SET_DEVICE_OP(dev_ops, process_mad);
1310 SET_DEVICE_OP(dev_ops, query_ah);
1311 SET_DEVICE_OP(dev_ops, query_device);
1312 SET_DEVICE_OP(dev_ops, query_gid);
1313 SET_DEVICE_OP(dev_ops, query_pkey);
1314 SET_DEVICE_OP(dev_ops, query_port);
1315 SET_DEVICE_OP(dev_ops, query_qp);
1316 SET_DEVICE_OP(dev_ops, query_srq);
1317 SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
1318 SET_DEVICE_OP(dev_ops, read_counters);
1319 SET_DEVICE_OP(dev_ops, reg_dm_mr);
1320 SET_DEVICE_OP(dev_ops, reg_user_mr);
1321 SET_DEVICE_OP(dev_ops, req_ncomp_notif);
1322 SET_DEVICE_OP(dev_ops, req_notify_cq);
1323 SET_DEVICE_OP(dev_ops, rereg_user_mr);
1324 SET_DEVICE_OP(dev_ops, resize_cq);
1325 SET_DEVICE_OP(dev_ops, set_vf_guid);
1326 SET_DEVICE_OP(dev_ops, set_vf_link_state);
1327 SET_DEVICE_OP(dev_ops, unmap_fmr);
1329 EXPORT_SYMBOL(ib_set_device_ops);
1331 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
1332 [RDMA_NL_LS_OP_RESOLVE] = {
1333 .doit = ib_nl_handle_resolve_resp,
1334 .flags = RDMA_NL_ADMIN_PERM,
1336 [RDMA_NL_LS_OP_SET_TIMEOUT] = {
1337 .doit = ib_nl_handle_set_timeout,
1338 .flags = RDMA_NL_ADMIN_PERM,
1340 [RDMA_NL_LS_OP_IP_RESOLVE] = {
1341 .doit = ib_nl_handle_ip_res_resp,
1342 .flags = RDMA_NL_ADMIN_PERM,
1346 static int __init ib_core_init(void)
1350 ib_wq = alloc_workqueue("infiniband", 0, 0);
1354 ib_comp_wq = alloc_workqueue("ib-comp-wq",
1355 WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1361 ib_comp_unbound_wq =
1362 alloc_workqueue("ib-comp-unb-wq",
1363 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM |
1364 WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);
1365 if (!ib_comp_unbound_wq) {
1370 ret = class_register(&ib_class);
1372 pr_warn("Couldn't create InfiniBand device class\n");
1373 goto err_comp_unbound;
1376 ret = rdma_nl_init();
1378 pr_warn("Couldn't init IB netlink interface: err %d\n", ret);
1384 pr_warn("Could't init IB address resolution\n");
1388 ret = ib_mad_init();
1390 pr_warn("Couldn't init IB MAD\n");
1396 pr_warn("Couldn't init SA\n");
1400 ret = register_lsm_notifier(&ibdev_lsm_nb);
1402 pr_warn("Couldn't register LSM notifier. ret %d\n", ret);
1407 rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
1408 roce_gid_mgmt_init();
1421 class_unregister(&ib_class);
1423 destroy_workqueue(ib_comp_unbound_wq);
1425 destroy_workqueue(ib_comp_wq);
1427 destroy_workqueue(ib_wq);
1431 static void __exit ib_core_cleanup(void)
1433 roce_gid_mgmt_cleanup();
1435 rdma_nl_unregister(RDMA_NL_LS);
1436 unregister_lsm_notifier(&ibdev_lsm_nb);
1441 class_unregister(&ib_class);
1442 destroy_workqueue(ib_comp_unbound_wq);
1443 destroy_workqueue(ib_comp_wq);
1444 /* Make sure that any pending umem accounting work is done. */
1445 destroy_workqueue(ib_wq);
1448 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
1450 subsys_initcall(ib_core_init);
1451 module_exit(ib_core_cleanup);