1 /*
2 * Copyright © 2009 CNRS
3 * Copyright © 2009-2016 Inria. All rights reserved.
4 * Copyright © 2009-2010 Université Bordeaux
5 * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved.
6 * See COPYING in top-level directory.
7 */
8
9 /** \file
10 * \brief Macros to help interaction between hwloc and OpenFabrics
11 * verbs.
12 *
13 * Applications that use both hwloc and OpenFabrics verbs may want to
14 * include this file so as to get topology information for OpenFabrics
15 * hardware (InfiniBand, etc).
16 *
17 */
18
19 #ifndef HWLOC_OPENFABRICS_VERBS_H
20 #define HWLOC_OPENFABRICS_VERBS_H
21
22 #include <hwloc.h>
23 #include <hwloc/autogen/config.h>
24 #ifdef HWLOC_LINUX_SYS
25 #include <hwloc/linux.h>
26 #endif
27
28 #include <infiniband/verbs.h>
29
30
31 #ifdef __cplusplus
32 extern "C" {
33 #endif
34
35
36 /** \defgroup hwlocality_openfabrics Interoperability with OpenFabrics
37 *
38 * This interface offers ways to retrieve topology information about
39 * OpenFabrics devices (InfiniBand, Omni-Path, usNIC, etc).
40 *
41 * @{
42 */
43
44 /** \brief Get the CPU set of logical processors that are physically
45 * close to device \p ibdev.
46 *
47 * Return the CPU set describing the locality of the OpenFabrics
48 * device \p ibdev (InfiniBand, etc).
49 *
50 * Topology \p topology and device \p ibdev must match the local machine.
51 * I/O devices detection is not needed in the topology.
52 *
53 * The function only returns the locality of the device.
54 * If more information about the device is needed, OS objects should
55 * be used instead, see hwloc_ibv_get_device_osdev()
56 * and hwloc_ibv_get_device_osdev_by_name().
57 *
58 * This function is currently only implemented in a meaningful way for
59 * Linux; other systems will simply get a full cpuset.
60 */
61 static __hwloc_inline int
62 hwloc_ibv_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused,
63 struct ibv_device *ibdev, hwloc_cpuset_t set)
64 {
65 #ifdef HWLOC_LINUX_SYS
66 /* If we're on Linux, use the verbs-provided sysfs mechanism to
67 get the local cpus */
68 #define HWLOC_OPENFABRICS_VERBS_SYSFS_PATH_MAX 128
69 char path[HWLOC_OPENFABRICS_VERBS_SYSFS_PATH_MAX];
70
71 if (!hwloc_topology_is_thissystem(topology)) {
72 errno = EINVAL;
73 return -1;
74 }
75
76 sprintf(path, "/sys/class/infiniband/%s/device/local_cpus",
77 ibv_get_device_name(ibdev));
78 if (hwloc_linux_read_path_as_cpumask(path, set) < 0
79 || hwloc_bitmap_iszero(set))
80 hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
81 #else
82 /* Non-Linux systems simply get a full cpuset */
83 hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
84 #endif
85 return 0;
86 }
87
88 /** \brief Get the hwloc OS device object corresponding to the OpenFabrics
89 * device named \p ibname.
90 *
91 * Return the OS device object describing the OpenFabrics device
92 * (InfiniBand, Omni-Path, usNIC, etc) whose name is \p ibname
93 * (mlx5_0, hfi1_0, usnic_0, qib0, etc).
94 * Returns NULL if there is none.
95 * The name \p ibname is usually obtained from ibv_get_device_name().
96 *
97 * The topology \p topology does not necessarily have to match the current
98 * machine. For instance the topology may be an XML import of a remote host.
99 * I/O devices detection must be enabled in the topology.
100 *
101 * \note The corresponding PCI device object can be obtained by looking
102 * at the OS device parent object.
103 */
104 static __hwloc_inline hwloc_obj_t
105 hwloc_ibv_get_device_osdev_by_name(hwloc_topology_t topology,
106 const char *ibname)
107 {
108 hwloc_obj_t osdev = NULL;
109 while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
110 if (HWLOC_OBJ_OSDEV_OPENFABRICS == osdev->attr->osdev.type
111 && osdev->name && !strcmp(ibname, osdev->name))
112 return osdev;
113 }
114 return NULL;
115 }
116
117 /** \brief Get the hwloc OS device object corresponding to the OpenFabrics
118 * device \p ibdev.
119 *
120 * Return the OS device object describing the OpenFabrics device \p ibdev
121 * (InfiniBand, etc). Returns NULL if there is none.
122 *
123 * Topology \p topology and device \p ibdev must match the local machine.
124 * I/O devices detection must be enabled in the topology.
125 * If not, the locality of the object may still be found using
126 * hwloc_ibv_get_device_cpuset().
127 *
128 * \note The corresponding PCI device object can be obtained by looking
129 * at the OS device parent object.
130 */
131 static __hwloc_inline hwloc_obj_t
132 hwloc_ibv_get_device_osdev(hwloc_topology_t topology,
133 struct ibv_device *ibdev)
134 {
135 if (!hwloc_topology_is_thissystem(topology)) {
136 errno = EINVAL;
137 return NULL;
138 }
139 return hwloc_ibv_get_device_osdev_by_name(topology, ibv_get_device_name(ibdev));
140 }
141
142 /** @} */
143
144
145 #ifdef __cplusplus
146 } /* extern "C" */
147 #endif
148
149
150 #endif /* HWLOC_OPENFABRICS_VERBS_H */