1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8
9#include <linux/amba/bus.h>
10#include <linux/device.h>
11#include <linux/kernel.h>
12#include <linux/bits.h>
13#include <linux/bug.h>
14#include <linux/types.h>
15#include <linux/init.h>
16#include <linux/export.h>
17#include <linux/slab.h>
18#include <linux/errno.h>
19#include <linux/host1x_context_bus.h>
20#include <linux/iommu.h>
21#include <linux/iommufd.h>
22#include <linux/idr.h>
23#include <linux/err.h>
24#include <linux/pci.h>
25#include <linux/pci-ats.h>
26#include <linux/bitops.h>
27#include <linux/platform_device.h>
28#include <linux/property.h>
29#include <linux/fsl/mc.h>
30#include <linux/module.h>
31#include <linux/cc_platform.h>
32#include <linux/cdx/cdx_bus.h>
33#include <trace/events/iommu.h>
34#include <linux/sched/mm.h>
35#include <linux/msi.h>
36#include <uapi/linux/iommufd.h>
37
38#include "dma-iommu.h"
39#include "iommu-priv.h"
40
41static struct kset *iommu_group_kset;
42static DEFINE_IDA(iommu_group_ida);
43static DEFINE_IDA(iommu_global_pasid_ida);
44
45static unsigned int iommu_def_domain_type __read_mostly;
46static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
47static u32 iommu_cmd_line __read_mostly;
48
49/* Tags used with xa_tag_pointer() in group->pasid_array */
50enum { IOMMU_PASID_ARRAY_DOMAIN = 0, IOMMU_PASID_ARRAY_HANDLE = 1 };
51
52struct iommu_group {
53 struct kobject kobj;
54 struct kobject *devices_kobj;
55 struct list_head devices;
56 struct xarray pasid_array;
57 struct mutex mutex;
58 void *iommu_data;
59 void (*iommu_data_release)(void *iommu_data);
60 char *name;
61 int id;
62 struct iommu_domain *default_domain;
63 struct iommu_domain *blocking_domain;
64 struct iommu_domain *domain;
65 struct list_head entry;
66 unsigned int owner_cnt;
67 void *owner;
68};
69
70struct group_device {
71 struct list_head list;
72 struct device *dev;
73 char *name;
74};
75
76/* Iterate over each struct group_device in a struct iommu_group */
77#define for_each_group_device(group, pos) \
78 list_for_each_entry(pos, &(group)->devices, list)
79
80struct iommu_group_attribute {
81 struct attribute attr;
82 ssize_t (*show)(struct iommu_group *group, char *buf);
83 ssize_t (*store)(struct iommu_group *group,
84 const char *buf, size_t count);
85};
86
87static const char * const iommu_group_resv_type_string[] = {
88 [IOMMU_RESV_DIRECT] = "direct",
89 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
90 [IOMMU_RESV_RESERVED] = "reserved",
91 [IOMMU_RESV_MSI] = "msi",
92 [IOMMU_RESV_SW_MSI] = "msi",
93};
94
95#define IOMMU_CMD_LINE_DMA_API BIT(0)
96#define IOMMU_CMD_LINE_STRICT BIT(1)
97
98static int bus_iommu_probe(const struct bus_type *bus);
99static int iommu_bus_notifier(struct notifier_block *nb,
100 unsigned long action, void *data);
101static void iommu_release_device(struct device *dev);
102static int __iommu_attach_device(struct iommu_domain *domain,
103 struct device *dev);
104static int __iommu_attach_group(struct iommu_domain *domain,
105 struct iommu_group *group);
106static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
107 unsigned int type,
108 unsigned int flags);
109
110enum {
111 IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
112};
113
114static int __iommu_device_set_domain(struct iommu_group *group,
115 struct device *dev,
116 struct iommu_domain *new_domain,
117 unsigned int flags);
118static int __iommu_group_set_domain_internal(struct iommu_group *group,
119 struct iommu_domain *new_domain,
120 unsigned int flags);
121static int __iommu_group_set_domain(struct iommu_group *group,
122 struct iommu_domain *new_domain)
123{
124 return __iommu_group_set_domain_internal(group, new_domain, flags: 0);
125}
126static void __iommu_group_set_domain_nofail(struct iommu_group *group,
127 struct iommu_domain *new_domain)
128{
129 WARN_ON(__iommu_group_set_domain_internal(
130 group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED));
131}
132
133static int iommu_setup_default_domain(struct iommu_group *group,
134 int target_type);
135static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
136 struct device *dev);
137static ssize_t iommu_group_store_type(struct iommu_group *group,
138 const char *buf, size_t count);
139static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
140 struct device *dev);
141static void __iommu_group_free_device(struct iommu_group *group,
142 struct group_device *grp_dev);
143static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
144 const struct iommu_ops *ops);
145
146#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
147struct iommu_group_attribute iommu_group_attr_##_name = \
148 __ATTR(_name, _mode, _show, _store)
149
150#define to_iommu_group_attr(_attr) \
151 container_of(_attr, struct iommu_group_attribute, attr)
152#define to_iommu_group(_kobj) \
153 container_of(_kobj, struct iommu_group, kobj)
154
155static LIST_HEAD(iommu_device_list);
156static DEFINE_SPINLOCK(iommu_device_lock);
157
158static const struct bus_type * const iommu_buses[] = {
159 &platform_bus_type,
160#ifdef CONFIG_PCI
161 &pci_bus_type,
162#endif
163#ifdef CONFIG_ARM_AMBA
164 &amba_bustype,
165#endif
166#ifdef CONFIG_FSL_MC_BUS
167 &fsl_mc_bus_type,
168#endif
169#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
170 &host1x_context_device_bus_type,
171#endif
172#ifdef CONFIG_CDX_BUS
173 &cdx_bus_type,
174#endif
175};
176
177/*
178 * Use a function instead of an array here because the domain-type is a
179 * bit-field, so an array would waste memory.
180 */
181static const char *iommu_domain_type_str(unsigned int t)
182{
183 switch (t) {
184 case IOMMU_DOMAIN_BLOCKED:
185 return "Blocked";
186 case IOMMU_DOMAIN_IDENTITY:
187 return "Passthrough";
188 case IOMMU_DOMAIN_UNMANAGED:
189 return "Unmanaged";
190 case IOMMU_DOMAIN_DMA:
191 case IOMMU_DOMAIN_DMA_FQ:
192 return "Translated";
193 case IOMMU_DOMAIN_PLATFORM:
194 return "Platform";
195 default:
196 return "Unknown";
197 }
198}
199
200static int __init iommu_subsys_init(void)
201{
202 struct notifier_block *nb;
203
204 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
205 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
206 iommu_set_default_passthrough(cmd_line: false);
207 else
208 iommu_set_default_translated(cmd_line: false);
209
210 if (iommu_default_passthrough() && cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT)) {
211 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
212 iommu_set_default_translated(cmd_line: false);
213 }
214 }
215
216 if (!iommu_default_passthrough() && !iommu_dma_strict)
217 iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
218
219 pr_info("Default domain type: %s%s\n",
220 iommu_domain_type_str(iommu_def_domain_type),
221 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
222 " (set via kernel command line)" : "");
223
224 if (!iommu_default_passthrough())
225 pr_info("DMA domain TLB invalidation policy: %s mode%s\n",
226 iommu_dma_strict ? "strict" : "lazy",
227 (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
228 " (set via kernel command line)" : "");
229
230 nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
231 if (!nb)
232 return -ENOMEM;
233
234 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
235 nb[i].notifier_call = iommu_bus_notifier;
236 bus_register_notifier(bus: iommu_buses[i], nb: &nb[i]);
237 }
238
239 return 0;
240}
241subsys_initcall(iommu_subsys_init);
242
243static int remove_iommu_group(struct device *dev, void *data)
244{
245 if (dev->iommu && dev->iommu->iommu_dev == data)
246 iommu_release_device(dev);
247
248 return 0;
249}
250
251/**
252 * iommu_device_register() - Register an IOMMU hardware instance
253 * @iommu: IOMMU handle for the instance
254 * @ops: IOMMU ops to associate with the instance
255 * @hwdev: (optional) actual instance device, used for fwnode lookup
256 *
257 * Return: 0 on success, or an error.
258 */
259int iommu_device_register(struct iommu_device *iommu,
260 const struct iommu_ops *ops, struct device *hwdev)
261{
262 int err = 0;
263
264 /* We need to be able to take module references appropriately */
265 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
266 return -EINVAL;
267
268 iommu->ops = ops;
269 if (hwdev)
270 iommu->fwnode = dev_fwnode(hwdev);
271
272 spin_lock(lock: &iommu_device_lock);
273 list_add_tail(new: &iommu->list, head: &iommu_device_list);
274 spin_unlock(lock: &iommu_device_lock);
275
276 for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++)
277 err = bus_iommu_probe(bus: iommu_buses[i]);
278 if (err)
279 iommu_device_unregister(iommu);
280 else
281 WRITE_ONCE(iommu->ready, true);
282 return err;
283}
284EXPORT_SYMBOL_GPL(iommu_device_register);
285
286void iommu_device_unregister(struct iommu_device *iommu)
287{
288 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
289 bus_for_each_dev(bus: iommu_buses[i], NULL, data: iommu, fn: remove_iommu_group);
290
291 spin_lock(lock: &iommu_device_lock);
292 list_del(entry: &iommu->list);
293 spin_unlock(lock: &iommu_device_lock);
294
295 /* Pairs with the alloc in generic_single_device_group() */
296 iommu_group_put(group: iommu->singleton_group);
297 iommu->singleton_group = NULL;
298}
299EXPORT_SYMBOL_GPL(iommu_device_unregister);
300
301#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
302void iommu_device_unregister_bus(struct iommu_device *iommu,
303 const struct bus_type *bus,
304 struct notifier_block *nb)
305{
306 bus_unregister_notifier(bus, nb);
307 fwnode_remove_software_node(iommu->fwnode);
308 iommu_device_unregister(iommu);
309}
310EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
311
312/*
313 * Register an iommu driver against a single bus. This is only used by iommufd
314 * selftest to create a mock iommu driver. The caller must provide
315 * some memory to hold a notifier_block.
316 */
317int iommu_device_register_bus(struct iommu_device *iommu,
318 const struct iommu_ops *ops,
319 const struct bus_type *bus,
320 struct notifier_block *nb)
321{
322 int err;
323
324 iommu->ops = ops;
325 nb->notifier_call = iommu_bus_notifier;
326 err = bus_register_notifier(bus, nb);
327 if (err)
328 return err;
329
330 iommu->fwnode = fwnode_create_software_node(NULL, NULL);
331 if (IS_ERR(iommu->fwnode)) {
332 bus_unregister_notifier(bus, nb);
333 return PTR_ERR(iommu->fwnode);
334 }
335
336 spin_lock(&iommu_device_lock);
337 list_add_tail(&iommu->list, &iommu_device_list);
338 spin_unlock(&iommu_device_lock);
339
340 err = bus_iommu_probe(bus);
341 if (err) {
342 iommu_device_unregister_bus(iommu, bus, nb);
343 return err;
344 }
345 WRITE_ONCE(iommu->ready, true);
346 return 0;
347}
348EXPORT_SYMBOL_GPL(iommu_device_register_bus);
349
350int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)
351{
352 int rc;
353
354 mutex_lock(&iommu_probe_device_lock);
355 rc = iommu_fwspec_init(dev, iommu->fwnode);
356 mutex_unlock(&iommu_probe_device_lock);
357
358 if (rc)
359 return rc;
360
361 rc = device_add(dev);
362 if (rc)
363 iommu_fwspec_free(dev);
364 return rc;
365}
366EXPORT_SYMBOL_GPL(iommu_mock_device_add);
367#endif
368
369static struct dev_iommu *dev_iommu_get(struct device *dev)
370{
371 struct dev_iommu *param = dev->iommu;
372
373 lockdep_assert_held(&iommu_probe_device_lock);
374
375 if (param)
376 return param;
377
378 param = kzalloc(sizeof(*param), GFP_KERNEL);
379 if (!param)
380 return NULL;
381
382 mutex_init(&param->lock);
383 dev->iommu = param;
384 return param;
385}
386
387void dev_iommu_free(struct device *dev)
388{
389 struct dev_iommu *param = dev->iommu;
390
391 dev->iommu = NULL;
392 if (param->fwspec) {
393 fwnode_handle_put(fwnode: param->fwspec->iommu_fwnode);
394 kfree(objp: param->fwspec);
395 }
396 kfree(objp: param);
397}
398
399/*
400 * Internal equivalent of device_iommu_mapped() for when we care that a device
401 * actually has API ops, and don't want false positives from VFIO-only groups.
402 */
403static bool dev_has_iommu(struct device *dev)
404{
405 return dev->iommu && dev->iommu->iommu_dev;
406}
407
408static u32 dev_iommu_get_max_pasids(struct device *dev)
409{
410 u32 max_pasids = 0, bits = 0;
411 int ret;
412
413 if (dev_is_pci(dev)) {
414 ret = pci_max_pasids(to_pci_dev(dev));
415 if (ret > 0)
416 max_pasids = ret;
417 } else {
418 ret = device_property_read_u32(dev, propname: "pasid-num-bits", val: &bits);
419 if (!ret)
420 max_pasids = 1UL << bits;
421 }
422
423 return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
424}
425
426void dev_iommu_priv_set(struct device *dev, void *priv)
427{
428 /* FSL_PAMU does something weird */
429 if (!IS_ENABLED(CONFIG_FSL_PAMU))
430 lockdep_assert_held(&iommu_probe_device_lock);
431 dev->iommu->priv = priv;
432}
433EXPORT_SYMBOL_GPL(dev_iommu_priv_set);
434
435/*
436 * Init the dev->iommu and dev->iommu_group in the struct device and get the
437 * driver probed
438 */
439static int iommu_init_device(struct device *dev)
440{
441 const struct iommu_ops *ops;
442 struct iommu_device *iommu_dev;
443 struct iommu_group *group;
444 int ret;
445
446 if (!dev_iommu_get(dev))
447 return -ENOMEM;
448 /*
449 * For FDT-based systems and ACPI IORT/VIOT, the common firmware parsing
450 * is buried in the bus dma_configure path. Properly unpicking that is
451 * still a big job, so for now just invoke the whole thing. The device
452 * already having a driver bound means dma_configure has already run and
453 * found no IOMMU to wait for, so there's no point calling it again.
454 */
455 if (!dev->iommu->fwspec && !dev->driver && dev->bus->dma_configure) {
456 mutex_unlock(lock: &iommu_probe_device_lock);
457 dev->bus->dma_configure(dev);
458 mutex_lock(lock: &iommu_probe_device_lock);
459 /* If another instance finished the job for us, skip it */
460 if (!dev->iommu || dev->iommu_group)
461 return -ENODEV;
462 }
463 /*
464 * At this point, relevant devices either now have a fwspec which will
465 * match ops registered with a non-NULL fwnode, or we can reasonably
466 * assume that only one of Intel, AMD, s390, PAMU or legacy SMMUv2 can
467 * be present, and that any of their registered instances has suitable
468 * ops for probing, and thus cheekily co-opt the same mechanism.
469 */
470 ops = iommu_fwspec_ops(fwspec: dev->iommu->fwspec);
471 if (!ops) {
472 ret = -ENODEV;
473 goto err_free;
474 }
475
476 if (!try_module_get(module: ops->owner)) {
477 ret = -EINVAL;
478 goto err_free;
479 }
480
481 iommu_dev = ops->probe_device(dev);
482 if (IS_ERR(ptr: iommu_dev)) {
483 ret = PTR_ERR(ptr: iommu_dev);
484 goto err_module_put;
485 }
486 dev->iommu->iommu_dev = iommu_dev;
487
488 ret = iommu_device_link(iommu: iommu_dev, link: dev);
489 if (ret)
490 goto err_release;
491
492 group = ops->device_group(dev);
493 if (WARN_ON_ONCE(group == NULL))
494 group = ERR_PTR(error: -EINVAL);
495 if (IS_ERR(ptr: group)) {
496 ret = PTR_ERR(ptr: group);
497 goto err_unlink;
498 }
499 dev->iommu_group = group;
500
501 dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
502 if (ops->is_attach_deferred)
503 dev->iommu->attach_deferred = ops->is_attach_deferred(dev);
504 return 0;
505
506err_unlink:
507 iommu_device_unlink(iommu: iommu_dev, link: dev);
508err_release:
509 if (ops->release_device)
510 ops->release_device(dev);
511err_module_put:
512 module_put(module: ops->owner);
513err_free:
514 dev->iommu->iommu_dev = NULL;
515 dev_iommu_free(dev);
516 return ret;
517}
518
519static void iommu_deinit_device(struct device *dev)
520{
521 struct iommu_group *group = dev->iommu_group;
522 const struct iommu_ops *ops = dev_iommu_ops(dev);
523
524 lockdep_assert_held(&group->mutex);
525
526 iommu_device_unlink(iommu: dev->iommu->iommu_dev, link: dev);
527
528 /*
529 * release_device() must stop using any attached domain on the device.
530 * If there are still other devices in the group, they are not affected
531 * by this callback.
532 *
533 * If the iommu driver provides release_domain, the core code ensures
534 * that domain is attached prior to calling release_device. Drivers can
535 * use this to enforce a translation on the idle iommu. Typically, the
536 * global static blocked_domain is a good choice.
537 *
538 * Otherwise, the iommu driver must set the device to either an identity
539 * or a blocking translation in release_device() and stop using any
540 * domain pointer, as it is going to be freed.
541 *
542 * Regardless, if a delayed attach never occurred, then the release
543 * should still avoid touching any hardware configuration either.
544 */
545 if (!dev->iommu->attach_deferred && ops->release_domain)
546 ops->release_domain->ops->attach_dev(ops->release_domain, dev);
547
548 if (ops->release_device)
549 ops->release_device(dev);
550
551 /*
552 * If this is the last driver to use the group then we must free the
553 * domains before we do the module_put().
554 */
555 if (list_empty(head: &group->devices)) {
556 if (group->default_domain) {
557 iommu_domain_free(domain: group->default_domain);
558 group->default_domain = NULL;
559 }
560 if (group->blocking_domain) {
561 iommu_domain_free(domain: group->blocking_domain);
562 group->blocking_domain = NULL;
563 }
564 group->domain = NULL;
565 }
566
567 /* Caller must put iommu_group */
568 dev->iommu_group = NULL;
569 module_put(module: ops->owner);
570 dev_iommu_free(dev);
571#ifdef CONFIG_IOMMU_DMA
572 dev->dma_iommu = false;
573#endif
574}
575
576static struct iommu_domain *pasid_array_entry_to_domain(void *entry)
577{
578 if (xa_pointer_tag(entry) == IOMMU_PASID_ARRAY_DOMAIN)
579 return xa_untag_pointer(entry);
580 return ((struct iommu_attach_handle *)xa_untag_pointer(entry))->domain;
581}
582
583DEFINE_MUTEX(iommu_probe_device_lock);
584
585static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
586{
587 struct iommu_group *group;
588 struct group_device *gdev;
589 int ret;
590
591 /*
592 * Serialise to avoid races between IOMMU drivers registering in
593 * parallel and/or the "replay" calls from ACPI/OF code via client
594 * driver probe. Once the latter have been cleaned up we should
595 * probably be able to use device_lock() here to minimise the scope,
596 * but for now enforcing a simple global ordering is fine.
597 */
598 lockdep_assert_held(&iommu_probe_device_lock);
599
600 /* Device is probed already if in a group */
601 if (dev->iommu_group)
602 return 0;
603
604 ret = iommu_init_device(dev);
605 if (ret)
606 return ret;
607 /*
608 * And if we do now see any replay calls, they would indicate someone
609 * misusing the dma_configure path outside bus code.
610 */
611 if (dev->driver)
612 dev_WARN(dev, "late IOMMU probe at driver bind, something fishy here!\n");
613
614 group = dev->iommu_group;
615 gdev = iommu_group_alloc_device(group, dev);
616 mutex_lock(lock: &group->mutex);
617 if (IS_ERR(ptr: gdev)) {
618 ret = PTR_ERR(ptr: gdev);
619 goto err_put_group;
620 }
621
622 /*
623 * The gdev must be in the list before calling
624 * iommu_setup_default_domain()
625 */
626 list_add_tail(new: &gdev->list, head: &group->devices);
627 WARN_ON(group->default_domain && !group->domain);
628 if (group->default_domain)
629 iommu_create_device_direct_mappings(domain: group->default_domain, dev);
630 if (group->domain) {
631 ret = __iommu_device_set_domain(group, dev, new_domain: group->domain, flags: 0);
632 if (ret)
633 goto err_remove_gdev;
634 } else if (!group->default_domain && !group_list) {
635 ret = iommu_setup_default_domain(group, target_type: 0);
636 if (ret)
637 goto err_remove_gdev;
638 } else if (!group->default_domain) {
639 /*
640 * With a group_list argument we defer the default_domain setup
641 * to the caller by providing a de-duplicated list of groups
642 * that need further setup.
643 */
644 if (list_empty(head: &group->entry))
645 list_add_tail(new: &group->entry, head: group_list);
646 }
647
648 if (group->default_domain)
649 iommu_setup_dma_ops(dev);
650
651 mutex_unlock(lock: &group->mutex);
652
653 return 0;
654
655err_remove_gdev:
656 list_del(entry: &gdev->list);
657 __iommu_group_free_device(group, grp_dev: gdev);
658err_put_group:
659 iommu_deinit_device(dev);
660 mutex_unlock(lock: &group->mutex);
661 iommu_group_put(group);
662
663 return ret;
664}
665
666int iommu_probe_device(struct device *dev)
667{
668 const struct iommu_ops *ops;
669 int ret;
670
671 mutex_lock(lock: &iommu_probe_device_lock);
672 ret = __iommu_probe_device(dev, NULL);
673 mutex_unlock(lock: &iommu_probe_device_lock);
674 if (ret)
675 return ret;
676
677 ops = dev_iommu_ops(dev);
678 if (ops->probe_finalize)
679 ops->probe_finalize(dev);
680
681 return 0;
682}
683
684static void __iommu_group_free_device(struct iommu_group *group,
685 struct group_device *grp_dev)
686{
687 struct device *dev = grp_dev->dev;
688
689 sysfs_remove_link(kobj: group->devices_kobj, name: grp_dev->name);
690 sysfs_remove_link(kobj: &dev->kobj, name: "iommu_group");
691
692 trace_remove_device_from_group(group_id: group->id, dev);
693
694 /*
695 * If the group has become empty then ownership must have been
696 * released, and the current domain must be set back to NULL or
697 * the default domain.
698 */
699 if (list_empty(head: &group->devices))
700 WARN_ON(group->owner_cnt ||
701 group->domain != group->default_domain);
702
703 kfree(objp: grp_dev->name);
704 kfree(objp: grp_dev);
705}
706
707/* Remove the iommu_group from the struct device. */
708static void __iommu_group_remove_device(struct device *dev)
709{
710 struct iommu_group *group = dev->iommu_group;
711 struct group_device *device;
712
713 mutex_lock(lock: &group->mutex);
714 for_each_group_device(group, device) {
715 if (device->dev != dev)
716 continue;
717
718 list_del(entry: &device->list);
719 __iommu_group_free_device(group, grp_dev: device);
720 if (dev_has_iommu(dev))
721 iommu_deinit_device(dev);
722 else
723 dev->iommu_group = NULL;
724 break;
725 }
726 mutex_unlock(lock: &group->mutex);
727
728 /*
729 * Pairs with the get in iommu_init_device() or
730 * iommu_group_add_device()
731 */
732 iommu_group_put(group);
733}
734
735static void iommu_release_device(struct device *dev)
736{
737 struct iommu_group *group = dev->iommu_group;
738
739 if (group)
740 __iommu_group_remove_device(dev);
741
742 /* Free any fwspec if no iommu_driver was ever attached */
743 if (dev->iommu)
744 dev_iommu_free(dev);
745}
746
747static int __init iommu_set_def_domain_type(char *str)
748{
749 bool pt;
750 int ret;
751
752 ret = kstrtobool(s: str, res: &pt);
753 if (ret)
754 return ret;
755
756 if (pt)
757 iommu_set_default_passthrough(cmd_line: true);
758 else
759 iommu_set_default_translated(cmd_line: true);
760
761 return 0;
762}
763early_param("iommu.passthrough", iommu_set_def_domain_type);
764
765static int __init iommu_dma_setup(char *str)
766{
767 int ret = kstrtobool(s: str, res: &iommu_dma_strict);
768
769 if (!ret)
770 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
771 return ret;
772}
773early_param("iommu.strict", iommu_dma_setup);
774
775void iommu_set_dma_strict(void)
776{
777 iommu_dma_strict = true;
778 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
779 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
780}
781
782static ssize_t iommu_group_attr_show(struct kobject *kobj,
783 struct attribute *__attr, char *buf)
784{
785 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
786 struct iommu_group *group = to_iommu_group(kobj);
787 ssize_t ret = -EIO;
788
789 if (attr->show)
790 ret = attr->show(group, buf);
791 return ret;
792}
793
794static ssize_t iommu_group_attr_store(struct kobject *kobj,
795 struct attribute *__attr,
796 const char *buf, size_t count)
797{
798 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
799 struct iommu_group *group = to_iommu_group(kobj);
800 ssize_t ret = -EIO;
801
802 if (attr->store)
803 ret = attr->store(group, buf, count);
804 return ret;
805}
806
807static const struct sysfs_ops iommu_group_sysfs_ops = {
808 .show = iommu_group_attr_show,
809 .store = iommu_group_attr_store,
810};
811
812static int iommu_group_create_file(struct iommu_group *group,
813 struct iommu_group_attribute *attr)
814{
815 return sysfs_create_file(kobj: &group->kobj, attr: &attr->attr);
816}
817
818static void iommu_group_remove_file(struct iommu_group *group,
819 struct iommu_group_attribute *attr)
820{
821 sysfs_remove_file(kobj: &group->kobj, attr: &attr->attr);
822}
823
824static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
825{
826 return sysfs_emit(buf, fmt: "%s\n", group->name);
827}
828
829/**
830 * iommu_insert_resv_region - Insert a new region in the
831 * list of reserved regions.
832 * @new: new region to insert
833 * @regions: list of regions
834 *
835 * Elements are sorted by start address and overlapping segments
836 * of the same type are merged.
837 */
838static int iommu_insert_resv_region(struct iommu_resv_region *new,
839 struct list_head *regions)
840{
841 struct iommu_resv_region *iter, *tmp, *nr, *top;
842 LIST_HEAD(stack);
843
844 nr = iommu_alloc_resv_region(start: new->start, length: new->length,
845 prot: new->prot, type: new->type, GFP_KERNEL);
846 if (!nr)
847 return -ENOMEM;
848
849 /* First add the new element based on start address sorting */
850 list_for_each_entry(iter, regions, list) {
851 if (nr->start < iter->start ||
852 (nr->start == iter->start && nr->type <= iter->type))
853 break;
854 }
855 list_add_tail(new: &nr->list, head: &iter->list);
856
857 /* Merge overlapping segments of type nr->type in @regions, if any */
858 list_for_each_entry_safe(iter, tmp, regions, list) {
859 phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
860
861 /* no merge needed on elements of different types than @new */
862 if (iter->type != new->type) {
863 list_move_tail(list: &iter->list, head: &stack);
864 continue;
865 }
866
867 /* look for the last stack element of same type as @iter */
868 list_for_each_entry_reverse(top, &stack, list)
869 if (top->type == iter->type)
870 goto check_overlap;
871
872 list_move_tail(list: &iter->list, head: &stack);
873 continue;
874
875check_overlap:
876 top_end = top->start + top->length - 1;
877
878 if (iter->start > top_end + 1) {
879 list_move_tail(list: &iter->list, head: &stack);
880 } else {
881 top->length = max(top_end, iter_end) - top->start + 1;
882 list_del(entry: &iter->list);
883 kfree(objp: iter);
884 }
885 }
886 list_splice(list: &stack, head: regions);
887 return 0;
888}
889
890static int
891iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
892 struct list_head *group_resv_regions)
893{
894 struct iommu_resv_region *entry;
895 int ret = 0;
896
897 list_for_each_entry(entry, dev_resv_regions, list) {
898 ret = iommu_insert_resv_region(new: entry, regions: group_resv_regions);
899 if (ret)
900 break;
901 }
902 return ret;
903}
904
905int iommu_get_group_resv_regions(struct iommu_group *group,
906 struct list_head *head)
907{
908 struct group_device *device;
909 int ret = 0;
910
911 mutex_lock(lock: &group->mutex);
912 for_each_group_device(group, device) {
913 struct list_head dev_resv_regions;
914
915 /*
916 * Non-API groups still expose reserved_regions in sysfs,
917 * so filter out calls that get here that way.
918 */
919 if (!dev_has_iommu(dev: device->dev))
920 break;
921
922 INIT_LIST_HEAD(list: &dev_resv_regions);
923 iommu_get_resv_regions(dev: device->dev, list: &dev_resv_regions);
924 ret = iommu_insert_device_resv_regions(dev_resv_regions: &dev_resv_regions, group_resv_regions: head);
925 iommu_put_resv_regions(dev: device->dev, list: &dev_resv_regions);
926 if (ret)
927 break;
928 }
929 mutex_unlock(lock: &group->mutex);
930 return ret;
931}
932EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
933
934static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
935 char *buf)
936{
937 struct iommu_resv_region *region, *next;
938 struct list_head group_resv_regions;
939 int offset = 0;
940
941 INIT_LIST_HEAD(list: &group_resv_regions);
942 iommu_get_group_resv_regions(group, &group_resv_regions);
943
944 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
945 offset += sysfs_emit_at(buf, at: offset, fmt: "0x%016llx 0x%016llx %s\n",
946 (long long)region->start,
947 (long long)(region->start +
948 region->length - 1),
949 iommu_group_resv_type_string[region->type]);
950 kfree(objp: region);
951 }
952
953 return offset;
954}
955
956static ssize_t iommu_group_show_type(struct iommu_group *group,
957 char *buf)
958{
959 char *type = "unknown";
960
961 mutex_lock(lock: &group->mutex);
962 if (group->default_domain) {
963 switch (group->default_domain->type) {
964 case IOMMU_DOMAIN_BLOCKED:
965 type = "blocked";
966 break;
967 case IOMMU_DOMAIN_IDENTITY:
968 type = "identity";
969 break;
970 case IOMMU_DOMAIN_UNMANAGED:
971 type = "unmanaged";
972 break;
973 case IOMMU_DOMAIN_DMA:
974 type = "DMA";
975 break;
976 case IOMMU_DOMAIN_DMA_FQ:
977 type = "DMA-FQ";
978 break;
979 }
980 }
981 mutex_unlock(lock: &group->mutex);
982
983 return sysfs_emit(buf, fmt: "%s\n", type);
984}
985
986static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
987
988static IOMMU_GROUP_ATTR(reserved_regions, 0444,
989 iommu_group_show_resv_regions, NULL);
990
991static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
992 iommu_group_store_type);
993
994static void iommu_group_release(struct kobject *kobj)
995{
996 struct iommu_group *group = to_iommu_group(kobj);
997
998 pr_debug("Releasing group %d\n", group->id);
999
1000 if (group->iommu_data_release)
1001 group->iommu_data_release(group->iommu_data);
1002
1003 ida_free(&iommu_group_ida, id: group->id);
1004
1005 /* Domains are free'd by iommu_deinit_device() */
1006 WARN_ON(group->default_domain);
1007 WARN_ON(group->blocking_domain);
1008
1009 kfree(objp: group->name);
1010 kfree(objp: group);
1011}
1012
1013static const struct kobj_type iommu_group_ktype = {
1014 .sysfs_ops = &iommu_group_sysfs_ops,
1015 .release = iommu_group_release,
1016};
1017
1018/**
1019 * iommu_group_alloc - Allocate a new group
1020 *
1021 * This function is called by an iommu driver to allocate a new iommu
1022 * group. The iommu group represents the minimum granularity of the iommu.
1023 * Upon successful return, the caller holds a reference to the supplied
1024 * group in order to hold the group until devices are added. Use
1025 * iommu_group_put() to release this extra reference count, allowing the
1026 * group to be automatically reclaimed once it has no devices or external
1027 * references.
1028 */
1029struct iommu_group *iommu_group_alloc(void)
1030{
1031 struct iommu_group *group;
1032 int ret;
1033
1034 group = kzalloc(sizeof(*group), GFP_KERNEL);
1035 if (!group)
1036 return ERR_PTR(error: -ENOMEM);
1037
1038 group->kobj.kset = iommu_group_kset;
1039 mutex_init(&group->mutex);
1040 INIT_LIST_HEAD(list: &group->devices);
1041 INIT_LIST_HEAD(list: &group->entry);
1042 xa_init(xa: &group->pasid_array);
1043
1044 ret = ida_alloc(ida: &iommu_group_ida, GFP_KERNEL);
1045 if (ret < 0) {
1046 kfree(objp: group);
1047 return ERR_PTR(error: ret);
1048 }
1049 group->id = ret;
1050
1051 ret = kobject_init_and_add(kobj: &group->kobj, ktype: &iommu_group_ktype,
1052 NULL, fmt: "%d", group->id);
1053 if (ret) {
1054 kobject_put(kobj: &group->kobj);
1055 return ERR_PTR(error: ret);
1056 }
1057
1058 group->devices_kobj = kobject_create_and_add(name: "devices", parent: &group->kobj);
1059 if (!group->devices_kobj) {
1060 kobject_put(kobj: &group->kobj); /* triggers .release & free */
1061 return ERR_PTR(error: -ENOMEM);
1062 }
1063
1064 /*
1065 * The devices_kobj holds a reference on the group kobject, so
1066 * as long as that exists so will the group. We can therefore
1067 * use the devices_kobj for reference counting.
1068 */
1069 kobject_put(kobj: &group->kobj);
1070
1071 ret = iommu_group_create_file(group,
1072 attr: &iommu_group_attr_reserved_regions);
1073 if (ret) {
1074 kobject_put(kobj: group->devices_kobj);
1075 return ERR_PTR(error: ret);
1076 }
1077
1078 ret = iommu_group_create_file(group, attr: &iommu_group_attr_type);
1079 if (ret) {
1080 kobject_put(kobj: group->devices_kobj);
1081 return ERR_PTR(error: ret);
1082 }
1083
1084 pr_debug("Allocated group %d\n", group->id);
1085
1086 return group;
1087}
1088EXPORT_SYMBOL_GPL(iommu_group_alloc);
1089
1090/**
1091 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
1092 * @group: the group
1093 *
1094 * iommu drivers can store data in the group for use when doing iommu
1095 * operations. This function provides a way to retrieve it. Caller
1096 * should hold a group reference.
1097 */
1098void *iommu_group_get_iommudata(struct iommu_group *group)
1099{
1100 return group->iommu_data;
1101}
1102EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
1103
1104/**
1105 * iommu_group_set_iommudata - set iommu_data for a group
1106 * @group: the group
1107 * @iommu_data: new data
1108 * @release: release function for iommu_data
1109 *
1110 * iommu drivers can store data in the group for use when doing iommu
1111 * operations. This function provides a way to set the data after
1112 * the group has been allocated. Caller should hold a group reference.
1113 */
1114void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
1115 void (*release)(void *iommu_data))
1116{
1117 group->iommu_data = iommu_data;
1118 group->iommu_data_release = release;
1119}
1120EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
1121
1122/**
1123 * iommu_group_set_name - set name for a group
1124 * @group: the group
1125 * @name: name
1126 *
1127 * Allow iommu driver to set a name for a group. When set it will
1128 * appear in a name attribute file under the group in sysfs.
1129 */
1130int iommu_group_set_name(struct iommu_group *group, const char *name)
1131{
1132 int ret;
1133
1134 if (group->name) {
1135 iommu_group_remove_file(group, attr: &iommu_group_attr_name);
1136 kfree(objp: group->name);
1137 group->name = NULL;
1138 if (!name)
1139 return 0;
1140 }
1141
1142 group->name = kstrdup(s: name, GFP_KERNEL);
1143 if (!group->name)
1144 return -ENOMEM;
1145
1146 ret = iommu_group_create_file(group, attr: &iommu_group_attr_name);
1147 if (ret) {
1148 kfree(objp: group->name);
1149 group->name = NULL;
1150 return ret;
1151 }
1152
1153 return 0;
1154}
1155EXPORT_SYMBOL_GPL(iommu_group_set_name);
1156
1157static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
1158 struct device *dev)
1159{
1160 struct iommu_resv_region *entry;
1161 struct list_head mappings;
1162 unsigned long pg_size;
1163 int ret = 0;
1164
1165 pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
1166 INIT_LIST_HEAD(list: &mappings);
1167
1168 if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
1169 return -EINVAL;
1170
1171 iommu_get_resv_regions(dev, list: &mappings);
1172
1173 /* We need to consider overlapping regions for different devices */
1174 list_for_each_entry(entry, &mappings, list) {
1175 dma_addr_t start, end, addr;
1176 size_t map_size = 0;
1177
1178 if (entry->type == IOMMU_RESV_DIRECT)
1179 dev->iommu->require_direct = 1;
1180
1181 if ((entry->type != IOMMU_RESV_DIRECT &&
1182 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) ||
1183 !iommu_is_dma_domain(domain))
1184 continue;
1185
1186 start = ALIGN(entry->start, pg_size);
1187 end = ALIGN(entry->start + entry->length, pg_size);
1188
1189 for (addr = start; addr <= end; addr += pg_size) {
1190 phys_addr_t phys_addr;
1191
1192 if (addr == end)
1193 goto map_end;
1194
1195 phys_addr = iommu_iova_to_phys(domain, iova: addr);
1196 if (!phys_addr) {
1197 map_size += pg_size;
1198 continue;
1199 }
1200
1201map_end:
1202 if (map_size) {
1203 ret = iommu_map(domain, iova: addr - map_size,
1204 paddr: addr - map_size, size: map_size,
1205 prot: entry->prot, GFP_KERNEL);
1206 if (ret)
1207 goto out;
1208 map_size = 0;
1209 }
1210 }
1211
1212 }
1213out:
1214 iommu_put_resv_regions(dev, list: &mappings);
1215
1216 return ret;
1217}
1218
1219/* This is undone by __iommu_group_free_device() */
1220static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
1221 struct device *dev)
1222{
1223 int ret, i = 0;
1224 struct group_device *device;
1225
1226 device = kzalloc(sizeof(*device), GFP_KERNEL);
1227 if (!device)
1228 return ERR_PTR(error: -ENOMEM);
1229
1230 device->dev = dev;
1231
1232 ret = sysfs_create_link(kobj: &dev->kobj, target: &group->kobj, name: "iommu_group");
1233 if (ret)
1234 goto err_free_device;
1235
1236 device->name = kasprintf(GFP_KERNEL, fmt: "%s", kobject_name(kobj: &dev->kobj));
1237rename:
1238 if (!device->name) {
1239 ret = -ENOMEM;
1240 goto err_remove_link;
1241 }
1242
1243 ret = sysfs_create_link_nowarn(kobj: group->devices_kobj,
1244 target: &dev->kobj, name: device->name);
1245 if (ret) {
1246 if (ret == -EEXIST && i >= 0) {
1247 /*
1248 * Account for the slim chance of collision
1249 * and append an instance to the name.
1250 */
1251 kfree(objp: device->name);
1252 device->name = kasprintf(GFP_KERNEL, fmt: "%s.%d",
1253 kobject_name(kobj: &dev->kobj), i++);
1254 goto rename;
1255 }
1256 goto err_free_name;
1257 }
1258
1259 trace_add_device_to_group(group_id: group->id, dev);
1260
1261 dev_info(dev, "Adding to iommu group %d\n", group->id);
1262
1263 return device;
1264
1265err_free_name:
1266 kfree(objp: device->name);
1267err_remove_link:
1268 sysfs_remove_link(kobj: &dev->kobj, name: "iommu_group");
1269err_free_device:
1270 kfree(objp: device);
1271 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
1272 return ERR_PTR(error: ret);
1273}
1274
1275/**
1276 * iommu_group_add_device - add a device to an iommu group
1277 * @group: the group into which to add the device (reference should be held)
1278 * @dev: the device
1279 *
1280 * This function is called by an iommu driver to add a device into a
1281 * group. Adding a device increments the group reference count.
1282 */
1283int iommu_group_add_device(struct iommu_group *group, struct device *dev)
1284{
1285 struct group_device *gdev;
1286
1287 gdev = iommu_group_alloc_device(group, dev);
1288 if (IS_ERR(ptr: gdev))
1289 return PTR_ERR(ptr: gdev);
1290
1291 iommu_group_ref_get(group);
1292 dev->iommu_group = group;
1293
1294 mutex_lock(lock: &group->mutex);
1295 list_add_tail(new: &gdev->list, head: &group->devices);
1296 mutex_unlock(lock: &group->mutex);
1297 return 0;
1298}
1299EXPORT_SYMBOL_GPL(iommu_group_add_device);
1300
1301/**
1302 * iommu_group_remove_device - remove a device from it's current group
1303 * @dev: device to be removed
1304 *
1305 * This function is called by an iommu driver to remove the device from
1306 * it's current group. This decrements the iommu group reference count.
1307 */
1308void iommu_group_remove_device(struct device *dev)
1309{
1310 struct iommu_group *group = dev->iommu_group;
1311
1312 if (!group)
1313 return;
1314
1315 dev_info(dev, "Removing from iommu group %d\n", group->id);
1316
1317 __iommu_group_remove_device(dev);
1318}
1319EXPORT_SYMBOL_GPL(iommu_group_remove_device);
1320
1321#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
1322/**
1323 * iommu_group_mutex_assert - Check device group mutex lock
1324 * @dev: the device that has group param set
1325 *
1326 * This function is called by an iommu driver to check whether it holds
1327 * group mutex lock for the given device or not.
1328 *
1329 * Note that this function must be called after device group param is set.
1330 */
1331void iommu_group_mutex_assert(struct device *dev)
1332{
1333 struct iommu_group *group = dev->iommu_group;
1334
1335 lockdep_assert_held(&group->mutex);
1336}
1337EXPORT_SYMBOL_GPL(iommu_group_mutex_assert);
1338#endif
1339
1340static struct device *iommu_group_first_dev(struct iommu_group *group)
1341{
1342 lockdep_assert_held(&group->mutex);
1343 return list_first_entry(&group->devices, struct group_device, list)->dev;
1344}
1345
1346/**
1347 * iommu_group_for_each_dev - iterate over each device in the group
1348 * @group: the group
1349 * @data: caller opaque data to be passed to callback function
1350 * @fn: caller supplied callback function
1351 *
1352 * This function is called by group users to iterate over group devices.
1353 * Callers should hold a reference count to the group during callback.
1354 * The group->mutex is held across callbacks, which will block calls to
1355 * iommu_group_add/remove_device.
1356 */
1357int iommu_group_for_each_dev(struct iommu_group *group, void *data,
1358 int (*fn)(struct device *, void *))
1359{
1360 struct group_device *device;
1361 int ret = 0;
1362
1363 mutex_lock(lock: &group->mutex);
1364 for_each_group_device(group, device) {
1365 ret = fn(device->dev, data);
1366 if (ret)
1367 break;
1368 }
1369 mutex_unlock(lock: &group->mutex);
1370
1371 return ret;
1372}
1373EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
1374
1375/**
1376 * iommu_group_get - Return the group for a device and increment reference
1377 * @dev: get the group that this device belongs to
1378 *
1379 * This function is called by iommu drivers and users to get the group
1380 * for the specified device. If found, the group is returned and the group
1381 * reference in incremented, else NULL.
1382 */
1383struct iommu_group *iommu_group_get(struct device *dev)
1384{
1385 struct iommu_group *group = dev->iommu_group;
1386
1387 if (group)
1388 kobject_get(kobj: group->devices_kobj);
1389
1390 return group;
1391}
1392EXPORT_SYMBOL_GPL(iommu_group_get);
1393
1394/**
1395 * iommu_group_ref_get - Increment reference on a group
1396 * @group: the group to use, must not be NULL
1397 *
1398 * This function is called by iommu drivers to take additional references on an
1399 * existing group. Returns the given group for convenience.
1400 */
1401struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
1402{
1403 kobject_get(kobj: group->devices_kobj);
1404 return group;
1405}
1406EXPORT_SYMBOL_GPL(iommu_group_ref_get);
1407
1408/**
1409 * iommu_group_put - Decrement group reference
1410 * @group: the group to use
1411 *
1412 * This function is called by iommu drivers and users to release the
1413 * iommu group. Once the reference count is zero, the group is released.
1414 */
1415void iommu_group_put(struct iommu_group *group)
1416{
1417 if (group)
1418 kobject_put(kobj: group->devices_kobj);
1419}
1420EXPORT_SYMBOL_GPL(iommu_group_put);
1421
1422/**
1423 * iommu_group_id - Return ID for a group
1424 * @group: the group to ID
1425 *
1426 * Return the unique ID for the group matching the sysfs group number.
1427 */
1428int iommu_group_id(struct iommu_group *group)
1429{
1430 return group->id;
1431}
1432EXPORT_SYMBOL_GPL(iommu_group_id);
1433
1434static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1435 unsigned long *devfns);
1436
1437/*
1438 * To consider a PCI device isolated, we require ACS to support Source
1439 * Validation, Request Redirection, Completer Redirection, and Upstream
1440 * Forwarding. This effectively means that devices cannot spoof their
1441 * requester ID, requests and completions cannot be redirected, and all
1442 * transactions are forwarded upstream, even as it passes through a
1443 * bridge where the target device is downstream.
1444 */
1445#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1446
1447/*
1448 * For multifunction devices which are not isolated from each other, find
1449 * all the other non-isolated functions and look for existing groups. For
1450 * each function, we also need to look for aliases to or from other devices
1451 * that may already have a group.
1452 */
1453static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1454 unsigned long *devfns)
1455{
1456 struct pci_dev *tmp = NULL;
1457 struct iommu_group *group;
1458
1459 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1460 return NULL;
1461
1462 for_each_pci_dev(tmp) {
1463 if (tmp == pdev || tmp->bus != pdev->bus ||
1464 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1465 pci_acs_enabled(pdev: tmp, REQ_ACS_FLAGS))
1466 continue;
1467
1468 group = get_pci_alias_group(pdev: tmp, devfns);
1469 if (group) {
1470 pci_dev_put(dev: tmp);
1471 return group;
1472 }
1473 }
1474
1475 return NULL;
1476}
1477
1478/*
1479 * Look for aliases to or from the given device for existing groups. DMA
1480 * aliases are only supported on the same bus, therefore the search
1481 * space is quite small (especially since we're really only looking at pcie
1482 * device, and therefore only expect multiple slots on the root complex or
1483 * downstream switch ports). It's conceivable though that a pair of
1484 * multifunction devices could have aliases between them that would cause a
1485 * loop. To prevent this, we use a bitmap to track where we've been.
1486 */
1487static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1488 unsigned long *devfns)
1489{
1490 struct pci_dev *tmp = NULL;
1491 struct iommu_group *group;
1492
1493 if (test_and_set_bit(nr: pdev->devfn & 0xff, addr: devfns))
1494 return NULL;
1495
1496 group = iommu_group_get(&pdev->dev);
1497 if (group)
1498 return group;
1499
1500 for_each_pci_dev(tmp) {
1501 if (tmp == pdev || tmp->bus != pdev->bus)
1502 continue;
1503
1504 /* We alias them or they alias us */
1505 if (pci_devs_are_dma_aliases(dev1: pdev, dev2: tmp)) {
1506 group = get_pci_alias_group(pdev: tmp, devfns);
1507 if (group) {
1508 pci_dev_put(dev: tmp);
1509 return group;
1510 }
1511
1512 group = get_pci_function_alias_group(pdev: tmp, devfns);
1513 if (group) {
1514 pci_dev_put(dev: tmp);
1515 return group;
1516 }
1517 }
1518 }
1519
1520 return NULL;
1521}
1522
1523struct group_for_pci_data {
1524 struct pci_dev *pdev;
1525 struct iommu_group *group;
1526};
1527
1528/*
1529 * DMA alias iterator callback, return the last seen device. Stop and return
1530 * the IOMMU group if we find one along the way.
1531 */
1532static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1533{
1534 struct group_for_pci_data *data = opaque;
1535
1536 data->pdev = pdev;
1537 data->group = iommu_group_get(&pdev->dev);
1538
1539 return data->group != NULL;
1540}
1541
1542/*
1543 * Generic device_group call-back function. It just allocates one
1544 * iommu-group per device.
1545 */
1546struct iommu_group *generic_device_group(struct device *dev)
1547{
1548 return iommu_group_alloc();
1549}
1550EXPORT_SYMBOL_GPL(generic_device_group);
1551
1552/*
1553 * Generic device_group call-back function. It just allocates one
1554 * iommu-group per iommu driver instance shared by every device
1555 * probed by that iommu driver.
1556 */
1557struct iommu_group *generic_single_device_group(struct device *dev)
1558{
1559 struct iommu_device *iommu = dev->iommu->iommu_dev;
1560
1561 if (!iommu->singleton_group) {
1562 struct iommu_group *group;
1563
1564 group = iommu_group_alloc();
1565 if (IS_ERR(ptr: group))
1566 return group;
1567 iommu->singleton_group = group;
1568 }
1569 return iommu_group_ref_get(iommu->singleton_group);
1570}
1571EXPORT_SYMBOL_GPL(generic_single_device_group);
1572
1573/*
1574 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1575 * to find or create an IOMMU group for a device.
1576 */
1577struct iommu_group *pci_device_group(struct device *dev)
1578{
1579 struct pci_dev *pdev = to_pci_dev(dev);
1580 struct group_for_pci_data data;
1581 struct pci_bus *bus;
1582 struct iommu_group *group = NULL;
1583 u64 devfns[4] = { 0 };
1584
1585 if (WARN_ON(!dev_is_pci(dev)))
1586 return ERR_PTR(error: -EINVAL);
1587
1588 /*
1589 * Find the upstream DMA alias for the device. A device must not
1590 * be aliased due to topology in order to have its own IOMMU group.
1591 * If we find an alias along the way that already belongs to a
1592 * group, use it.
1593 */
1594 if (pci_for_each_dma_alias(pdev, fn: get_pci_alias_or_group, data: &data))
1595 return data.group;
1596
1597 pdev = data.pdev;
1598
1599 /*
1600 * Continue upstream from the point of minimum IOMMU granularity
1601 * due to aliases to the point where devices are protected from
1602 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
1603 * group, use it.
1604 */
1605 for (bus = pdev->bus; !pci_is_root_bus(pbus: bus); bus = bus->parent) {
1606 if (!bus->self)
1607 continue;
1608
1609 if (pci_acs_path_enabled(start: bus->self, NULL, REQ_ACS_FLAGS))
1610 break;
1611
1612 pdev = bus->self;
1613
1614 group = iommu_group_get(&pdev->dev);
1615 if (group)
1616 return group;
1617 }
1618
1619 /*
1620 * Look for existing groups on device aliases. If we alias another
1621 * device or another device aliases us, use the same group.
1622 */
1623 group = get_pci_alias_group(pdev, devfns: (unsigned long *)devfns);
1624 if (group)
1625 return group;
1626
1627 /*
1628 * Look for existing groups on non-isolated functions on the same
1629 * slot and aliases of those funcions, if any. No need to clear
1630 * the search bitmap, the tested devfns are still valid.
1631 */
1632 group = get_pci_function_alias_group(pdev, devfns: (unsigned long *)devfns);
1633 if (group)
1634 return group;
1635
1636 /* No shared group found, allocate new */
1637 return iommu_group_alloc();
1638}
1639EXPORT_SYMBOL_GPL(pci_device_group);
1640
1641/* Get the IOMMU group for device on fsl-mc bus */
1642struct iommu_group *fsl_mc_device_group(struct device *dev)
1643{
1644 struct device *cont_dev = fsl_mc_cont_dev(dev);
1645 struct iommu_group *group;
1646
1647 group = iommu_group_get(cont_dev);
1648 if (!group)
1649 group = iommu_group_alloc();
1650 return group;
1651}
1652EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1653
1654static struct iommu_domain *__iommu_alloc_identity_domain(struct device *dev)
1655{
1656 const struct iommu_ops *ops = dev_iommu_ops(dev);
1657 struct iommu_domain *domain;
1658
1659 if (ops->identity_domain)
1660 return ops->identity_domain;
1661
1662 if (ops->domain_alloc_identity) {
1663 domain = ops->domain_alloc_identity(dev);
1664 if (IS_ERR(ptr: domain))
1665 return domain;
1666 } else {
1667 return ERR_PTR(error: -EOPNOTSUPP);
1668 }
1669
1670 iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops);
1671 return domain;
1672}
1673
1674static struct iommu_domain *
1675__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1676{
1677 struct device *dev = iommu_group_first_dev(group);
1678 struct iommu_domain *dom;
1679
1680 if (group->default_domain && group->default_domain->type == req_type)
1681 return group->default_domain;
1682
1683 /*
1684 * When allocating the DMA API domain assume that the driver is going to
1685 * use PASID and make sure the RID's domain is PASID compatible.
1686 */
1687 if (req_type & __IOMMU_DOMAIN_PAGING) {
1688 dom = __iommu_paging_domain_alloc_flags(dev, type: req_type,
1689 flags: dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);
1690
1691 /*
1692 * If driver does not support PASID feature then
1693 * try to allocate non-PASID domain
1694 */
1695 if (PTR_ERR(ptr: dom) == -EOPNOTSUPP)
1696 dom = __iommu_paging_domain_alloc_flags(dev, type: req_type, flags: 0);
1697
1698 return dom;
1699 }
1700
1701 if (req_type == IOMMU_DOMAIN_IDENTITY)
1702 return __iommu_alloc_identity_domain(dev);
1703
1704 return ERR_PTR(error: -EINVAL);
1705}
1706
1707/*
1708 * req_type of 0 means "auto" which means to select a domain based on
1709 * iommu_def_domain_type or what the driver actually supports.
1710 */
1711static struct iommu_domain *
1712iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1713{
1714 const struct iommu_ops *ops = dev_iommu_ops(dev: iommu_group_first_dev(group));
1715 struct iommu_domain *dom;
1716
1717 lockdep_assert_held(&group->mutex);
1718
1719 /*
1720 * Allow legacy drivers to specify the domain that will be the default
1721 * domain. This should always be either an IDENTITY/BLOCKED/PLATFORM
1722 * domain. Do not use in new drivers.
1723 */
1724 if (ops->default_domain) {
1725 if (req_type != ops->default_domain->type)
1726 return ERR_PTR(error: -EINVAL);
1727 return ops->default_domain;
1728 }
1729
1730 if (req_type)
1731 return __iommu_group_alloc_default_domain(group, req_type);
1732
1733 /* The driver gave no guidance on what type to use, try the default */
1734 dom = __iommu_group_alloc_default_domain(group, req_type: iommu_def_domain_type);
1735 if (!IS_ERR(ptr: dom))
1736 return dom;
1737
1738 /* Otherwise IDENTITY and DMA_FQ defaults will try DMA */
1739 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA)
1740 return ERR_PTR(error: -EINVAL);
1741 dom = __iommu_group_alloc_default_domain(group, IOMMU_DOMAIN_DMA);
1742 if (IS_ERR(ptr: dom))
1743 return dom;
1744
1745 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
1746 iommu_def_domain_type, group->name);
1747 return dom;
1748}
1749
1750struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1751{
1752 return group->default_domain;
1753}
1754
1755static int probe_iommu_group(struct device *dev, void *data)
1756{
1757 struct list_head *group_list = data;
1758 int ret;
1759
1760 mutex_lock(lock: &iommu_probe_device_lock);
1761 ret = __iommu_probe_device(dev, group_list);
1762 mutex_unlock(lock: &iommu_probe_device_lock);
1763 if (ret == -ENODEV)
1764 ret = 0;
1765
1766 return ret;
1767}
1768
1769static int iommu_bus_notifier(struct notifier_block *nb,
1770 unsigned long action, void *data)
1771{
1772 struct device *dev = data;
1773
1774 if (action == BUS_NOTIFY_ADD_DEVICE) {
1775 int ret;
1776
1777 ret = iommu_probe_device(dev);
1778 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1779 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1780 iommu_release_device(dev);
1781 return NOTIFY_OK;
1782 }
1783
1784 return 0;
1785}
1786
1787/*
1788 * Combine the driver's chosen def_domain_type across all the devices in a
1789 * group. Drivers must give a consistent result.
1790 */
1791static int iommu_get_def_domain_type(struct iommu_group *group,
1792 struct device *dev, int cur_type)
1793{
1794 const struct iommu_ops *ops = dev_iommu_ops(dev);
1795 int type;
1796
1797 if (ops->default_domain) {
1798 /*
1799 * Drivers that declare a global static default_domain will
1800 * always choose that.
1801 */
1802 type = ops->default_domain->type;
1803 } else {
1804 if (ops->def_domain_type)
1805 type = ops->def_domain_type(dev);
1806 else
1807 return cur_type;
1808 }
1809 if (!type || cur_type == type)
1810 return cur_type;
1811 if (!cur_type)
1812 return type;
1813
1814 dev_err_ratelimited(
1815 dev,
1816 "IOMMU driver error, requesting conflicting def_domain_type, %s and %s, for devices in group %u.\n",
1817 iommu_domain_type_str(cur_type), iommu_domain_type_str(type),
1818 group->id);
1819
1820 /*
1821 * Try to recover, drivers are allowed to force IDENTITY or DMA, IDENTITY
1822 * takes precedence.
1823 */
1824 if (type == IOMMU_DOMAIN_IDENTITY)
1825 return type;
1826 return cur_type;
1827}
1828
1829/*
1830 * A target_type of 0 will select the best domain type. 0 can be returned in
1831 * this case meaning the global default should be used.
1832 */
1833static int iommu_get_default_domain_type(struct iommu_group *group,
1834 int target_type)
1835{
1836 struct device *untrusted = NULL;
1837 struct group_device *gdev;
1838 int driver_type = 0;
1839
1840 lockdep_assert_held(&group->mutex);
1841
1842 /*
1843 * ARM32 drivers supporting CONFIG_ARM_DMA_USE_IOMMU can declare an
1844 * identity_domain and it will automatically become their default
1845 * domain. Later on ARM_DMA_USE_IOMMU will install its UNMANAGED domain.
1846 * Override the selection to IDENTITY.
1847 */
1848 if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
1849 static_assert(!(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) &&
1850 IS_ENABLED(CONFIG_IOMMU_DMA)));
1851 driver_type = IOMMU_DOMAIN_IDENTITY;
1852 }
1853
1854 for_each_group_device(group, gdev) {
1855 driver_type = iommu_get_def_domain_type(group, dev: gdev->dev,
1856 cur_type: driver_type);
1857
1858 if (dev_is_pci(gdev->dev) && to_pci_dev(gdev->dev)->untrusted) {
1859 /*
1860 * No ARM32 using systems will set untrusted, it cannot
1861 * work.
1862 */
1863 if (WARN_ON(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)))
1864 return -1;
1865 untrusted = gdev->dev;
1866 }
1867 }
1868
1869 /*
1870 * If the common dma ops are not selected in kconfig then we cannot use
1871 * IOMMU_DOMAIN_DMA at all. Force IDENTITY if nothing else has been
1872 * selected.
1873 */
1874 if (!IS_ENABLED(CONFIG_IOMMU_DMA)) {
1875 if (WARN_ON(driver_type == IOMMU_DOMAIN_DMA))
1876 return -1;
1877 if (!driver_type)
1878 driver_type = IOMMU_DOMAIN_IDENTITY;
1879 }
1880
1881 if (untrusted) {
1882 if (driver_type && driver_type != IOMMU_DOMAIN_DMA) {
1883 dev_err_ratelimited(
1884 untrusted,
1885 "Device is not trusted, but driver is overriding group %u to %s, refusing to probe.\n",
1886 group->id, iommu_domain_type_str(driver_type));
1887 return -1;
1888 }
1889 driver_type = IOMMU_DOMAIN_DMA;
1890 }
1891
1892 if (target_type) {
1893 if (driver_type && target_type != driver_type)
1894 return -1;
1895 return target_type;
1896 }
1897 return driver_type;
1898}
1899
1900static void iommu_group_do_probe_finalize(struct device *dev)
1901{
1902 const struct iommu_ops *ops = dev_iommu_ops(dev);
1903
1904 if (ops->probe_finalize)
1905 ops->probe_finalize(dev);
1906}
1907
1908static int bus_iommu_probe(const struct bus_type *bus)
1909{
1910 struct iommu_group *group, *next;
1911 LIST_HEAD(group_list);
1912 int ret;
1913
1914 ret = bus_for_each_dev(bus, NULL, data: &group_list, fn: probe_iommu_group);
1915 if (ret)
1916 return ret;
1917
1918 list_for_each_entry_safe(group, next, &group_list, entry) {
1919 struct group_device *gdev;
1920
1921 mutex_lock(lock: &group->mutex);
1922
1923 /* Remove item from the list */
1924 list_del_init(entry: &group->entry);
1925
1926 /*
1927 * We go to the trouble of deferred default domain creation so
1928 * that the cross-group default domain type and the setup of the
1929 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
1930 */
1931 ret = iommu_setup_default_domain(group, target_type: 0);
1932 if (ret) {
1933 mutex_unlock(lock: &group->mutex);
1934 return ret;
1935 }
1936 for_each_group_device(group, gdev)
1937 iommu_setup_dma_ops(dev: gdev->dev);
1938 mutex_unlock(lock: &group->mutex);
1939
1940 /*
1941 * FIXME: Mis-locked because the ops->probe_finalize() call-back
1942 * of some IOMMU drivers calls arm_iommu_attach_device() which
1943 * in-turn might call back into IOMMU core code, where it tries
1944 * to take group->mutex, resulting in a deadlock.
1945 */
1946 for_each_group_device(group, gdev)
1947 iommu_group_do_probe_finalize(dev: gdev->dev);
1948 }
1949
1950 return 0;
1951}
1952
1953/**
1954 * device_iommu_capable() - check for a general IOMMU capability
1955 * @dev: device to which the capability would be relevant, if available
1956 * @cap: IOMMU capability
1957 *
1958 * Return: true if an IOMMU is present and supports the given capability
1959 * for the given device, otherwise false.
1960 */
1961bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
1962{
1963 const struct iommu_ops *ops;
1964
1965 if (!dev_has_iommu(dev))
1966 return false;
1967
1968 ops = dev_iommu_ops(dev);
1969 if (!ops->capable)
1970 return false;
1971
1972 return ops->capable(dev, cap);
1973}
1974EXPORT_SYMBOL_GPL(device_iommu_capable);
1975
1976/**
1977 * iommu_group_has_isolated_msi() - Compute msi_device_has_isolated_msi()
1978 * for a group
1979 * @group: Group to query
1980 *
1981 * IOMMU groups should not have differing values of
1982 * msi_device_has_isolated_msi() for devices in a group. However nothing
1983 * directly prevents this, so ensure mistakes don't result in isolation failures
1984 * by checking that all the devices are the same.
1985 */
1986bool iommu_group_has_isolated_msi(struct iommu_group *group)
1987{
1988 struct group_device *group_dev;
1989 bool ret = true;
1990
1991 mutex_lock(lock: &group->mutex);
1992 for_each_group_device(group, group_dev)
1993 ret &= msi_device_has_isolated_msi(dev: group_dev->dev);
1994 mutex_unlock(lock: &group->mutex);
1995 return ret;
1996}
1997EXPORT_SYMBOL_GPL(iommu_group_has_isolated_msi);
1998
1999/**
2000 * iommu_set_fault_handler() - set a fault handler for an iommu domain
2001 * @domain: iommu domain
2002 * @handler: fault handler
2003 * @token: user data, will be passed back to the fault handler
2004 *
2005 * This function should be used by IOMMU users which want to be notified
2006 * whenever an IOMMU fault happens.
2007 *
2008 * The fault handler itself should return 0 on success, and an appropriate
2009 * error code otherwise.
2010 */
2011void iommu_set_fault_handler(struct iommu_domain *domain,
2012 iommu_fault_handler_t handler,
2013 void *token)
2014{
2015 if (WARN_ON(!domain || domain->cookie_type != IOMMU_COOKIE_NONE))
2016 return;
2017
2018 domain->cookie_type = IOMMU_COOKIE_FAULT_HANDLER;
2019 domain->handler = handler;
2020 domain->handler_token = token;
2021}
2022EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
2023
2024static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
2025 const struct iommu_ops *ops)
2026{
2027 domain->type = type;
2028 domain->owner = ops;
2029 if (!domain->ops)
2030 domain->ops = ops->default_domain_ops;
2031}
2032
2033static struct iommu_domain *
2034__iommu_paging_domain_alloc_flags(struct device *dev, unsigned int type,
2035 unsigned int flags)
2036{
2037 const struct iommu_ops *ops;
2038 struct iommu_domain *domain;
2039
2040 if (!dev_has_iommu(dev))
2041 return ERR_PTR(error: -ENODEV);
2042
2043 ops = dev_iommu_ops(dev);
2044
2045 if (ops->domain_alloc_paging && !flags)
2046 domain = ops->domain_alloc_paging(dev);
2047 else if (ops->domain_alloc_paging_flags)
2048 domain = ops->domain_alloc_paging_flags(dev, flags, NULL);
2049#if IS_ENABLED(CONFIG_FSL_PAMU)
2050 else if (ops->domain_alloc && !flags)
2051 domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
2052#endif
2053 else
2054 return ERR_PTR(error: -EOPNOTSUPP);
2055
2056 if (IS_ERR(ptr: domain))
2057 return domain;
2058 if (!domain)
2059 return ERR_PTR(error: -ENOMEM);
2060
2061 iommu_domain_init(domain, type, ops);
2062 return domain;
2063}
2064
2065/**
2066 * iommu_paging_domain_alloc_flags() - Allocate a paging domain
2067 * @dev: device for which the domain is allocated
2068 * @flags: Bitmap of iommufd_hwpt_alloc_flags
2069 *
2070 * Allocate a paging domain which will be managed by a kernel driver. Return
2071 * allocated domain if successful, or an ERR pointer for failure.
2072 */
2073struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
2074 unsigned int flags)
2075{
2076 return __iommu_paging_domain_alloc_flags(dev,
2077 IOMMU_DOMAIN_UNMANAGED, flags);
2078}
2079EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
2080
2081void iommu_domain_free(struct iommu_domain *domain)
2082{
2083 switch (domain->cookie_type) {
2084 case IOMMU_COOKIE_DMA_IOVA:
2085 iommu_put_dma_cookie(domain);
2086 break;
2087 case IOMMU_COOKIE_DMA_MSI:
2088 iommu_put_msi_cookie(domain);
2089 break;
2090 case IOMMU_COOKIE_SVA:
2091 mmdrop(mm: domain->mm);
2092 break;
2093 default:
2094 break;
2095 }
2096 if (domain->ops->free)
2097 domain->ops->free(domain);
2098}
2099EXPORT_SYMBOL_GPL(iommu_domain_free);
2100
2101/*
2102 * Put the group's domain back to the appropriate core-owned domain - either the
2103 * standard kernel-mode DMA configuration or an all-DMA-blocked domain.
2104 */
2105static void __iommu_group_set_core_domain(struct iommu_group *group)
2106{
2107 struct iommu_domain *new_domain;
2108
2109 if (group->owner)
2110 new_domain = group->blocking_domain;
2111 else
2112 new_domain = group->default_domain;
2113
2114 __iommu_group_set_domain_nofail(group, new_domain);
2115}
2116
2117static int __iommu_attach_device(struct iommu_domain *domain,
2118 struct device *dev)
2119{
2120 int ret;
2121
2122 if (unlikely(domain->ops->attach_dev == NULL))
2123 return -ENODEV;
2124
2125 ret = domain->ops->attach_dev(domain, dev);
2126 if (ret)
2127 return ret;
2128 dev->iommu->attach_deferred = 0;
2129 trace_attach_device_to_domain(dev);
2130 return 0;
2131}
2132
2133/**
2134 * iommu_attach_device - Attach an IOMMU domain to a device
2135 * @domain: IOMMU domain to attach
2136 * @dev: Device that will be attached
2137 *
2138 * Returns 0 on success and error code on failure
2139 *
2140 * Note that EINVAL can be treated as a soft failure, indicating
2141 * that certain configuration of the domain is incompatible with
2142 * the device. In this case attaching a different domain to the
2143 * device may succeed.
2144 */
2145int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
2146{
2147 /* Caller must be a probed driver on dev */
2148 struct iommu_group *group = dev->iommu_group;
2149 int ret;
2150
2151 if (!group)
2152 return -ENODEV;
2153
2154 /*
2155 * Lock the group to make sure the device-count doesn't
2156 * change while we are attaching
2157 */
2158 mutex_lock(lock: &group->mutex);
2159 ret = -EINVAL;
2160 if (list_count_nodes(head: &group->devices) != 1)
2161 goto out_unlock;
2162
2163 ret = __iommu_attach_group(domain, group);
2164
2165out_unlock:
2166 mutex_unlock(lock: &group->mutex);
2167 return ret;
2168}
2169EXPORT_SYMBOL_GPL(iommu_attach_device);
2170
2171int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
2172{
2173 if (dev->iommu && dev->iommu->attach_deferred)
2174 return __iommu_attach_device(domain, dev);
2175
2176 return 0;
2177}
2178
2179void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
2180{
2181 /* Caller must be a probed driver on dev */
2182 struct iommu_group *group = dev->iommu_group;
2183
2184 if (!group)
2185 return;
2186
2187 mutex_lock(lock: &group->mutex);
2188 if (WARN_ON(domain != group->domain) ||
2189 WARN_ON(list_count_nodes(&group->devices) != 1))
2190 goto out_unlock;
2191 __iommu_group_set_core_domain(group);
2192
2193out_unlock:
2194 mutex_unlock(lock: &group->mutex);
2195}
2196EXPORT_SYMBOL_GPL(iommu_detach_device);
2197
2198struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
2199{
2200 /* Caller must be a probed driver on dev */
2201 struct iommu_group *group = dev->iommu_group;
2202
2203 if (!group)
2204 return NULL;
2205
2206 return group->domain;
2207}
2208EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2209
2210/*
2211 * For IOMMU_DOMAIN_DMA implementations which already provide their own
2212 * guarantees that the group and its default domain are valid and correct.
2213 */
2214struct iommu_domain *iommu_get_dma_domain(struct device *dev)
2215{
2216 return dev->iommu_group->default_domain;
2217}
2218
2219static void *iommu_make_pasid_array_entry(struct iommu_domain *domain,
2220 struct iommu_attach_handle *handle)
2221{
2222 if (handle) {
2223 handle->domain = domain;
2224 return xa_tag_pointer(p: handle, tag: IOMMU_PASID_ARRAY_HANDLE);
2225 }
2226
2227 return xa_tag_pointer(p: domain, tag: IOMMU_PASID_ARRAY_DOMAIN);
2228}
2229
2230static bool domain_iommu_ops_compatible(const struct iommu_ops *ops,
2231 struct iommu_domain *domain)
2232{
2233 if (domain->owner == ops)
2234 return true;
2235
2236 /* For static domains, owner isn't set. */
2237 if (domain == ops->blocked_domain || domain == ops->identity_domain)
2238 return true;
2239
2240 return false;
2241}
2242
2243static int __iommu_attach_group(struct iommu_domain *domain,
2244 struct iommu_group *group)
2245{
2246 struct device *dev;
2247
2248 if (group->domain && group->domain != group->default_domain &&
2249 group->domain != group->blocking_domain)
2250 return -EBUSY;
2251
2252 dev = iommu_group_first_dev(group);
2253 if (!dev_has_iommu(dev) ||
2254 !domain_iommu_ops_compatible(ops: dev_iommu_ops(dev), domain))
2255 return -EINVAL;
2256
2257 return __iommu_group_set_domain(group, new_domain: domain);
2258}
2259
2260/**
2261 * iommu_attach_group - Attach an IOMMU domain to an IOMMU group
2262 * @domain: IOMMU domain to attach
2263 * @group: IOMMU group that will be attached
2264 *
2265 * Returns 0 on success and error code on failure
2266 *
2267 * Note that EINVAL can be treated as a soft failure, indicating
2268 * that certain configuration of the domain is incompatible with
2269 * the group. In this case attaching a different domain to the
2270 * group may succeed.
2271 */
2272int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
2273{
2274 int ret;
2275
2276 mutex_lock(lock: &group->mutex);
2277 ret = __iommu_attach_group(domain, group);
2278 mutex_unlock(lock: &group->mutex);
2279
2280 return ret;
2281}
2282EXPORT_SYMBOL_GPL(iommu_attach_group);
2283
2284static int __iommu_device_set_domain(struct iommu_group *group,
2285 struct device *dev,
2286 struct iommu_domain *new_domain,
2287 unsigned int flags)
2288{
2289 int ret;
2290
2291 /*
2292 * If the device requires IOMMU_RESV_DIRECT then we cannot allow
2293 * the blocking domain to be attached as it does not contain the
2294 * required 1:1 mapping. This test effectively excludes the device
2295 * being used with iommu_group_claim_dma_owner() which will block
2296 * vfio and iommufd as well.
2297 */
2298 if (dev->iommu->require_direct &&
2299 (new_domain->type == IOMMU_DOMAIN_BLOCKED ||
2300 new_domain == group->blocking_domain)) {
2301 dev_warn(dev,
2302 "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n");
2303 return -EINVAL;
2304 }
2305
2306 if (dev->iommu->attach_deferred) {
2307 if (new_domain == group->default_domain)
2308 return 0;
2309 dev->iommu->attach_deferred = 0;
2310 }
2311
2312 ret = __iommu_attach_device(domain: new_domain, dev);
2313 if (ret) {
2314 /*
2315 * If we have a blocking domain then try to attach that in hopes
2316 * of avoiding a UAF. Modern drivers should implement blocking
2317 * domains as global statics that cannot fail.
2318 */
2319 if ((flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) &&
2320 group->blocking_domain &&
2321 group->blocking_domain != new_domain)
2322 __iommu_attach_device(domain: group->blocking_domain, dev);
2323 return ret;
2324 }
2325 return 0;
2326}
2327
2328/*
2329 * If 0 is returned the group's domain is new_domain. If an error is returned
2330 * then the group's domain will be set back to the existing domain unless
2331 * IOMMU_SET_DOMAIN_MUST_SUCCEED, otherwise an error is returned and the group's
2332 * domains is left inconsistent. This is a driver bug to fail attach with a
2333 * previously good domain. We try to avoid a kernel UAF because of this.
2334 *
2335 * IOMMU groups are really the natural working unit of the IOMMU, but the IOMMU
2336 * API works on domains and devices. Bridge that gap by iterating over the
2337 * devices in a group. Ideally we'd have a single device which represents the
2338 * requestor ID of the group, but we also allow IOMMU drivers to create policy
2339 * defined minimum sets, where the physical hardware may be able to distiguish
2340 * members, but we wish to group them at a higher level (ex. untrusted
2341 * multi-function PCI devices). Thus we attach each device.
2342 */
2343static int __iommu_group_set_domain_internal(struct iommu_group *group,
2344 struct iommu_domain *new_domain,
2345 unsigned int flags)
2346{
2347 struct group_device *last_gdev;
2348 struct group_device *gdev;
2349 int result;
2350 int ret;
2351
2352 lockdep_assert_held(&group->mutex);
2353
2354 if (group->domain == new_domain)
2355 return 0;
2356
2357 if (WARN_ON(!new_domain))
2358 return -EINVAL;
2359
2360 /*
2361 * Changing the domain is done by calling attach_dev() on the new
2362 * domain. This switch does not have to be atomic and DMA can be
2363 * discarded during the transition. DMA must only be able to access
2364 * either new_domain or group->domain, never something else.
2365 */
2366 result = 0;
2367 for_each_group_device(group, gdev) {
2368 ret = __iommu_device_set_domain(group, dev: gdev->dev, new_domain,
2369 flags);
2370 if (ret) {
2371 result = ret;
2372 /*
2373 * Keep trying the other devices in the group. If a
2374 * driver fails attach to an otherwise good domain, and
2375 * does not support blocking domains, it should at least
2376 * drop its reference on the current domain so we don't
2377 * UAF.
2378 */
2379 if (flags & IOMMU_SET_DOMAIN_MUST_SUCCEED)
2380 continue;
2381 goto err_revert;
2382 }
2383 }
2384 group->domain = new_domain;
2385 return result;
2386
2387err_revert:
2388 /*
2389 * This is called in error unwind paths. A well behaved driver should
2390 * always allow us to attach to a domain that was already attached.
2391 */
2392 last_gdev = gdev;
2393 for_each_group_device(group, gdev) {
2394 /*
2395 * A NULL domain can happen only for first probe, in which case
2396 * we leave group->domain as NULL and let release clean
2397 * everything up.
2398 */
2399 if (group->domain)
2400 WARN_ON(__iommu_device_set_domain(
2401 group, gdev->dev, group->domain,
2402 IOMMU_SET_DOMAIN_MUST_SUCCEED));
2403 if (gdev == last_gdev)
2404 break;
2405 }
2406 return ret;
2407}
2408
2409void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
2410{
2411 mutex_lock(lock: &group->mutex);
2412 __iommu_group_set_core_domain(group);
2413 mutex_unlock(lock: &group->mutex);
2414}
2415EXPORT_SYMBOL_GPL(iommu_detach_group);
2416
2417phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2418{
2419 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2420 return iova;
2421
2422 if (domain->type == IOMMU_DOMAIN_BLOCKED)
2423 return 0;
2424
2425 return domain->ops->iova_to_phys(domain, iova);
2426}
2427EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
2428
2429static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
2430 phys_addr_t paddr, size_t size, size_t *count)
2431{
2432 unsigned int pgsize_idx, pgsize_idx_next;
2433 unsigned long pgsizes;
2434 size_t offset, pgsize, pgsize_next;
2435 size_t offset_end;
2436 unsigned long addr_merge = paddr | iova;
2437
2438 /* Page sizes supported by the hardware and small enough for @size */
2439 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
2440
2441 /* Constrain the page sizes further based on the maximum alignment */
2442 if (likely(addr_merge))
2443 pgsizes &= GENMASK(__ffs(addr_merge), 0);
2444
2445 /* Make sure we have at least one suitable page size */
2446 BUG_ON(!pgsizes);
2447
2448 /* Pick the biggest page size remaining */
2449 pgsize_idx = __fls(word: pgsizes);
2450 pgsize = BIT(pgsize_idx);
2451 if (!count)
2452 return pgsize;
2453
2454 /* Find the next biggest support page size, if it exists */
2455 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
2456 if (!pgsizes)
2457 goto out_set_count;
2458
2459 pgsize_idx_next = __ffs(pgsizes);
2460 pgsize_next = BIT(pgsize_idx_next);
2461
2462 /*
2463 * There's no point trying a bigger page size unless the virtual
2464 * and physical addresses are similarly offset within the larger page.
2465 */
2466 if ((iova ^ paddr) & (pgsize_next - 1))
2467 goto out_set_count;
2468
2469 /* Calculate the offset to the next page size alignment boundary */
2470 offset = pgsize_next - (addr_merge & (pgsize_next - 1));
2471
2472 /*
2473 * If size is big enough to accommodate the larger page, reduce
2474 * the number of smaller pages.
2475 */
2476 if (!check_add_overflow(offset, pgsize_next, &offset_end) &&
2477 offset_end <= size)
2478 size = offset;
2479
2480out_set_count:
2481 *count = size >> pgsize_idx;
2482 return pgsize;
2483}
2484
2485int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
2486 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2487{
2488 const struct iommu_domain_ops *ops = domain->ops;
2489 unsigned long orig_iova = iova;
2490 unsigned int min_pagesz;
2491 size_t orig_size = size;
2492 phys_addr_t orig_paddr = paddr;
2493 int ret = 0;
2494
2495 might_sleep_if(gfpflags_allow_blocking(gfp));
2496
2497 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2498 return -EINVAL;
2499
2500 if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
2501 return -ENODEV;
2502
2503 /* Discourage passing strange GFP flags */
2504 if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
2505 __GFP_HIGHMEM)))
2506 return -EINVAL;
2507
2508 /* find out the minimum page size supported */
2509 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2510
2511 /*
2512 * both the virtual address and the physical one, as well as
2513 * the size of the mapping, must be aligned (at least) to the
2514 * size of the smallest page supported by the hardware
2515 */
2516 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
2517 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
2518 iova, &paddr, size, min_pagesz);
2519 return -EINVAL;
2520 }
2521
2522 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
2523
2524 while (size) {
2525 size_t pgsize, count, mapped = 0;
2526
2527 pgsize = iommu_pgsize(domain, iova, paddr, size, count: &count);
2528
2529 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
2530 iova, &paddr, pgsize, count);
2531 ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
2532 gfp, &mapped);
2533 /*
2534 * Some pages may have been mapped, even if an error occurred,
2535 * so we should account for those so they can be unmapped.
2536 */
2537 size -= mapped;
2538
2539 if (ret)
2540 break;
2541
2542 iova += mapped;
2543 paddr += mapped;
2544 }
2545
2546 /* unroll mapping in case something went wrong */
2547 if (ret)
2548 iommu_unmap(domain, iova: orig_iova, size: orig_size - size);
2549 else
2550 trace_map(iova: orig_iova, paddr: orig_paddr, size: orig_size);
2551
2552 return ret;
2553}
2554
2555int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size)
2556{
2557 const struct iommu_domain_ops *ops = domain->ops;
2558
2559 if (!ops->iotlb_sync_map)
2560 return 0;
2561 return ops->iotlb_sync_map(domain, iova, size);
2562}
2563
2564int iommu_map(struct iommu_domain *domain, unsigned long iova,
2565 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2566{
2567 int ret;
2568
2569 ret = iommu_map_nosync(domain, iova, paddr, size, prot, gfp);
2570 if (ret)
2571 return ret;
2572
2573 ret = iommu_sync_map(domain, iova, size);
2574 if (ret)
2575 iommu_unmap(domain, iova, size);
2576
2577 return ret;
2578}
2579EXPORT_SYMBOL_GPL(iommu_map);
2580
2581static size_t __iommu_unmap(struct iommu_domain *domain,
2582 unsigned long iova, size_t size,
2583 struct iommu_iotlb_gather *iotlb_gather)
2584{
2585 const struct iommu_domain_ops *ops = domain->ops;
2586 size_t unmapped_page, unmapped = 0;
2587 unsigned long orig_iova = iova;
2588 unsigned int min_pagesz;
2589
2590 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2591 return 0;
2592
2593 if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
2594 return 0;
2595
2596 /* find out the minimum page size supported */
2597 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2598
2599 /*
2600 * The virtual address, as well as the size of the mapping, must be
2601 * aligned (at least) to the size of the smallest page supported
2602 * by the hardware
2603 */
2604 if (!IS_ALIGNED(iova | size, min_pagesz)) {
2605 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2606 iova, size, min_pagesz);
2607 return 0;
2608 }
2609
2610 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2611
2612 /*
2613 * Keep iterating until we either unmap 'size' bytes (or more)
2614 * or we hit an area that isn't mapped.
2615 */
2616 while (unmapped < size) {
2617 size_t pgsize, count;
2618
2619 pgsize = iommu_pgsize(domain, iova, paddr: iova, size: size - unmapped, count: &count);
2620 unmapped_page = ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather);
2621 if (!unmapped_page)
2622 break;
2623
2624 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2625 iova, unmapped_page);
2626
2627 iova += unmapped_page;
2628 unmapped += unmapped_page;
2629 }
2630
2631 trace_unmap(iova: orig_iova, size, unmapped_size: unmapped);
2632 return unmapped;
2633}
2634
2635/**
2636 * iommu_unmap() - Remove mappings from a range of IOVA
2637 * @domain: Domain to manipulate
2638 * @iova: IO virtual address to start
2639 * @size: Length of the range starting from @iova
2640 *
2641 * iommu_unmap() will remove a translation created by iommu_map(). It cannot
2642 * subdivide a mapping created by iommu_map(), so it should be called with IOVA
2643 * ranges that match what was passed to iommu_map(). The range can aggregate
2644 * contiguous iommu_map() calls so long as no individual range is split.
2645 *
2646 * Returns: Number of bytes of IOVA unmapped. iova + res will be the point
2647 * unmapping stopped.
2648 */
2649size_t iommu_unmap(struct iommu_domain *domain,
2650 unsigned long iova, size_t size)
2651{
2652 struct iommu_iotlb_gather iotlb_gather;
2653 size_t ret;
2654
2655 iommu_iotlb_gather_init(gather: &iotlb_gather);
2656 ret = __iommu_unmap(domain, iova, size, iotlb_gather: &iotlb_gather);
2657 iommu_iotlb_sync(domain, iotlb_gather: &iotlb_gather);
2658
2659 return ret;
2660}
2661EXPORT_SYMBOL_GPL(iommu_unmap);
2662
2663/**
2664 * iommu_unmap_fast() - Remove mappings from a range of IOVA without IOTLB sync
2665 * @domain: Domain to manipulate
2666 * @iova: IO virtual address to start
2667 * @size: Length of the range starting from @iova
2668 * @iotlb_gather: range information for a pending IOTLB flush
2669 *
2670 * iommu_unmap_fast() will remove a translation created by iommu_map().
2671 * It can't subdivide a mapping created by iommu_map(), so it should be
2672 * called with IOVA ranges that match what was passed to iommu_map(). The
2673 * range can aggregate contiguous iommu_map() calls so long as no individual
2674 * range is split.
2675 *
2676 * Basically iommu_unmap_fast() is the same as iommu_unmap() but for callers
2677 * which manage the IOTLB flushing externally to perform a batched sync.
2678 *
2679 * Returns: Number of bytes of IOVA unmapped. iova + res will be the point
2680 * unmapping stopped.
2681 */
2682size_t iommu_unmap_fast(struct iommu_domain *domain,
2683 unsigned long iova, size_t size,
2684 struct iommu_iotlb_gather *iotlb_gather)
2685{
2686 return __iommu_unmap(domain, iova, size, iotlb_gather);
2687}
2688EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2689
2690ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2691 struct scatterlist *sg, unsigned int nents, int prot,
2692 gfp_t gfp)
2693{
2694 size_t len = 0, mapped = 0;
2695 phys_addr_t start;
2696 unsigned int i = 0;
2697 int ret;
2698
2699 while (i <= nents) {
2700 phys_addr_t s_phys = sg_phys(sg);
2701
2702 if (len && s_phys != start + len) {
2703 ret = iommu_map_nosync(domain, iova: iova + mapped, paddr: start,
2704 size: len, prot, gfp);
2705 if (ret)
2706 goto out_err;
2707
2708 mapped += len;
2709 len = 0;
2710 }
2711
2712 if (sg_dma_is_bus_address(sg))
2713 goto next;
2714
2715 if (len) {
2716 len += sg->length;
2717 } else {
2718 len = sg->length;
2719 start = s_phys;
2720 }
2721
2722next:
2723 if (++i < nents)
2724 sg = sg_next(sg);
2725 }
2726
2727 ret = iommu_sync_map(domain, iova, size: mapped);
2728 if (ret)
2729 goto out_err;
2730
2731 return mapped;
2732
2733out_err:
2734 /* undo mappings already done */
2735 iommu_unmap(domain, iova, mapped);
2736
2737 return ret;
2738}
2739EXPORT_SYMBOL_GPL(iommu_map_sg);
2740
2741/**
2742 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2743 * @domain: the iommu domain where the fault has happened
2744 * @dev: the device where the fault has happened
2745 * @iova: the faulting address
2746 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2747 *
2748 * This function should be called by the low-level IOMMU implementations
2749 * whenever IOMMU faults happen, to allow high-level users, that are
2750 * interested in such events, to know about them.
2751 *
2752 * This event may be useful for several possible use cases:
2753 * - mere logging of the event
2754 * - dynamic TLB/PTE loading
2755 * - if restarting of the faulting device is required
2756 *
2757 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2758 * PTE/TLB loading will one day be supported, implementations will be able
2759 * to tell whether it succeeded or not according to this return value).
2760 *
2761 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2762 * (though fault handlers can also return -ENOSYS, in case they want to
2763 * elicit the default behavior of the IOMMU drivers).
2764 */
2765int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2766 unsigned long iova, int flags)
2767{
2768 int ret = -ENOSYS;
2769
2770 /*
2771 * if upper layers showed interest and installed a fault handler,
2772 * invoke it.
2773 */
2774 if (domain->cookie_type == IOMMU_COOKIE_FAULT_HANDLER &&
2775 domain->handler)
2776 ret = domain->handler(domain, dev, iova, flags,
2777 domain->handler_token);
2778
2779 trace_io_page_fault(dev, iova, flags);
2780 return ret;
2781}
2782EXPORT_SYMBOL_GPL(report_iommu_fault);
2783
2784static int __init iommu_init(void)
2785{
2786 iommu_group_kset = kset_create_and_add(name: "iommu_groups",
2787 NULL, parent_kobj: kernel_kobj);
2788 BUG_ON(!iommu_group_kset);
2789
2790 iommu_debugfs_setup();
2791
2792 return 0;
2793}
2794core_initcall(iommu_init);
2795
2796int iommu_set_pgtable_quirks(struct iommu_domain *domain,
2797 unsigned long quirk)
2798{
2799 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2800 return -EINVAL;
2801 if (!domain->ops->set_pgtable_quirks)
2802 return -EINVAL;
2803 return domain->ops->set_pgtable_quirks(domain, quirk);
2804}
2805EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2806
2807/**
2808 * iommu_get_resv_regions - get reserved regions
2809 * @dev: device for which to get reserved regions
2810 * @list: reserved region list for device
2811 *
2812 * This returns a list of reserved IOVA regions specific to this device.
2813 * A domain user should not map IOVA in these ranges.
2814 */
2815void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2816{
2817 const struct iommu_ops *ops = dev_iommu_ops(dev);
2818
2819 if (ops->get_resv_regions)
2820 ops->get_resv_regions(dev, list);
2821}
2822EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
2823
2824/**
2825 * iommu_put_resv_regions - release reserved regions
2826 * @dev: device for which to free reserved regions
2827 * @list: reserved region list for device
2828 *
2829 * This releases a reserved region list acquired by iommu_get_resv_regions().
2830 */
2831void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2832{
2833 struct iommu_resv_region *entry, *next;
2834
2835 list_for_each_entry_safe(entry, next, list, list) {
2836 if (entry->free)
2837 entry->free(dev, entry);
2838 else
2839 kfree(objp: entry);
2840 }
2841}
2842EXPORT_SYMBOL(iommu_put_resv_regions);
2843
2844struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2845 size_t length, int prot,
2846 enum iommu_resv_type type,
2847 gfp_t gfp)
2848{
2849 struct iommu_resv_region *region;
2850
2851 region = kzalloc(sizeof(*region), gfp);
2852 if (!region)
2853 return NULL;
2854
2855 INIT_LIST_HEAD(list: &region->list);
2856 region->start = start;
2857 region->length = length;
2858 region->prot = prot;
2859 region->type = type;
2860 return region;
2861}
2862EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2863
2864void iommu_set_default_passthrough(bool cmd_line)
2865{
2866 if (cmd_line)
2867 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2868 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2869}
2870
2871void iommu_set_default_translated(bool cmd_line)
2872{
2873 if (cmd_line)
2874 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2875 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2876}
2877
2878bool iommu_default_passthrough(void)
2879{
2880 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2881}
2882EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2883
2884static const struct iommu_device *iommu_from_fwnode(const struct fwnode_handle *fwnode)
2885{
2886 const struct iommu_device *iommu, *ret = NULL;
2887
2888 spin_lock(lock: &iommu_device_lock);
2889 list_for_each_entry(iommu, &iommu_device_list, list)
2890 if (iommu->fwnode == fwnode) {
2891 ret = iommu;
2892 break;
2893 }
2894 spin_unlock(lock: &iommu_device_lock);
2895 return ret;
2896}
2897
2898const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode)
2899{
2900 const struct iommu_device *iommu = iommu_from_fwnode(fwnode);
2901
2902 return iommu ? iommu->ops : NULL;
2903}
2904
2905int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode)
2906{
2907 const struct iommu_device *iommu = iommu_from_fwnode(fwnode: iommu_fwnode);
2908 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2909
2910 if (!iommu)
2911 return driver_deferred_probe_check_state(dev);
2912 if (!dev->iommu && !READ_ONCE(iommu->ready))
2913 return -EPROBE_DEFER;
2914
2915 if (fwspec)
2916 return iommu->ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL;
2917
2918 if (!dev_iommu_get(dev))
2919 return -ENOMEM;
2920
2921 /* Preallocate for the overwhelmingly common case of 1 ID */
2922 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2923 if (!fwspec)
2924 return -ENOMEM;
2925
2926 fwnode_handle_get(fwnode: iommu_fwnode);
2927 fwspec->iommu_fwnode = iommu_fwnode;
2928 dev_iommu_fwspec_set(dev, fwspec);
2929 return 0;
2930}
2931EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2932
2933void iommu_fwspec_free(struct device *dev)
2934{
2935 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2936
2937 if (fwspec) {
2938 fwnode_handle_put(fwnode: fwspec->iommu_fwnode);
2939 kfree(objp: fwspec);
2940 dev_iommu_fwspec_set(dev, NULL);
2941 }
2942}
2943
2944int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids)
2945{
2946 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2947 int i, new_num;
2948
2949 if (!fwspec)
2950 return -EINVAL;
2951
2952 new_num = fwspec->num_ids + num_ids;
2953 if (new_num > 1) {
2954 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2955 GFP_KERNEL);
2956 if (!fwspec)
2957 return -ENOMEM;
2958
2959 dev_iommu_fwspec_set(dev, fwspec);
2960 }
2961
2962 for (i = 0; i < num_ids; i++)
2963 fwspec->ids[fwspec->num_ids + i] = ids[i];
2964
2965 fwspec->num_ids = new_num;
2966 return 0;
2967}
2968EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2969
2970/**
2971 * iommu_setup_default_domain - Set the default_domain for the group
2972 * @group: Group to change
2973 * @target_type: Domain type to set as the default_domain
2974 *
2975 * Allocate a default domain and set it as the current domain on the group. If
2976 * the group already has a default domain it will be changed to the target_type.
2977 * When target_type is 0 the default domain is selected based on driver and
2978 * system preferences.
2979 */
2980static int iommu_setup_default_domain(struct iommu_group *group,
2981 int target_type)
2982{
2983 struct iommu_domain *old_dom = group->default_domain;
2984 struct group_device *gdev;
2985 struct iommu_domain *dom;
2986 bool direct_failed;
2987 int req_type;
2988 int ret;
2989
2990 lockdep_assert_held(&group->mutex);
2991
2992 req_type = iommu_get_default_domain_type(group, target_type);
2993 if (req_type < 0)
2994 return -EINVAL;
2995
2996 dom = iommu_group_alloc_default_domain(group, req_type);
2997 if (IS_ERR(ptr: dom))
2998 return PTR_ERR(ptr: dom);
2999
3000 if (group->default_domain == dom)
3001 return 0;
3002
3003 if (iommu_is_dma_domain(domain: dom)) {
3004 ret = iommu_get_dma_cookie(domain: dom);
3005 if (ret) {
3006 iommu_domain_free(dom);
3007 return ret;
3008 }
3009 }
3010
3011 /*
3012 * IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
3013 * mapped before their device is attached, in order to guarantee
3014 * continuity with any FW activity
3015 */
3016 direct_failed = false;
3017 for_each_group_device(group, gdev) {
3018 if (iommu_create_device_direct_mappings(domain: dom, dev: gdev->dev)) {
3019 direct_failed = true;
3020 dev_warn_once(
3021 gdev->dev->iommu->iommu_dev->dev,
3022 "IOMMU driver was not able to establish FW requested direct mapping.");
3023 }
3024 }
3025
3026 /* We must set default_domain early for __iommu_device_set_domain */
3027 group->default_domain = dom;
3028 if (!group->domain) {
3029 /*
3030 * Drivers are not allowed to fail the first domain attach.
3031 * The only way to recover from this is to fail attaching the
3032 * iommu driver and call ops->release_device. Put the domain
3033 * in group->default_domain so it is freed after.
3034 */
3035 ret = __iommu_group_set_domain_internal(
3036 group, new_domain: dom, flags: IOMMU_SET_DOMAIN_MUST_SUCCEED);
3037 if (WARN_ON(ret))
3038 goto out_free_old;
3039 } else {
3040 ret = __iommu_group_set_domain(group, new_domain: dom);
3041 if (ret)
3042 goto err_restore_def_domain;
3043 }
3044
3045 /*
3046 * Drivers are supposed to allow mappings to be installed in a domain
3047 * before device attachment, but some don't. Hack around this defect by
3048 * trying again after attaching. If this happens it means the device
3049 * will not continuously have the IOMMU_RESV_DIRECT map.
3050 */
3051 if (direct_failed) {
3052 for_each_group_device(group, gdev) {
3053 ret = iommu_create_device_direct_mappings(domain: dom, dev: gdev->dev);
3054 if (ret)
3055 goto err_restore_domain;
3056 }
3057 }
3058
3059out_free_old:
3060 if (old_dom)
3061 iommu_domain_free(old_dom);
3062 return ret;
3063
3064err_restore_domain:
3065 if (old_dom)
3066 __iommu_group_set_domain_internal(
3067 group, new_domain: old_dom, flags: IOMMU_SET_DOMAIN_MUST_SUCCEED);
3068err_restore_def_domain:
3069 if (old_dom) {
3070 iommu_domain_free(dom);
3071 group->default_domain = old_dom;
3072 }
3073 return ret;
3074}
3075
3076/*
3077 * Changing the default domain through sysfs requires the users to unbind the
3078 * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
3079 * transition. Return failure if this isn't met.
3080 *
3081 * We need to consider the race between this and the device release path.
3082 * group->mutex is used here to guarantee that the device release path
3083 * will not be entered at the same time.
3084 */
3085static ssize_t iommu_group_store_type(struct iommu_group *group,
3086 const char *buf, size_t count)
3087{
3088 struct group_device *gdev;
3089 int ret, req_type;
3090
3091 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
3092 return -EACCES;
3093
3094 if (WARN_ON(!group) || !group->default_domain)
3095 return -EINVAL;
3096
3097 if (sysfs_streq(s1: buf, s2: "identity"))
3098 req_type = IOMMU_DOMAIN_IDENTITY;
3099 else if (sysfs_streq(s1: buf, s2: "DMA"))
3100 req_type = IOMMU_DOMAIN_DMA;
3101 else if (sysfs_streq(s1: buf, s2: "DMA-FQ"))
3102 req_type = IOMMU_DOMAIN_DMA_FQ;
3103 else if (sysfs_streq(s1: buf, s2: "auto"))
3104 req_type = 0;
3105 else
3106 return -EINVAL;
3107
3108 mutex_lock(lock: &group->mutex);
3109 /* We can bring up a flush queue without tearing down the domain. */
3110 if (req_type == IOMMU_DOMAIN_DMA_FQ &&
3111 group->default_domain->type == IOMMU_DOMAIN_DMA) {
3112 ret = iommu_dma_init_fq(domain: group->default_domain);
3113 if (ret)
3114 goto out_unlock;
3115
3116 group->default_domain->type = IOMMU_DOMAIN_DMA_FQ;
3117 ret = count;
3118 goto out_unlock;
3119 }
3120
3121 /* Otherwise, ensure that device exists and no driver is bound. */
3122 if (list_empty(head: &group->devices) || group->owner_cnt) {
3123 ret = -EPERM;
3124 goto out_unlock;
3125 }
3126
3127 ret = iommu_setup_default_domain(group, target_type: req_type);
3128 if (ret)
3129 goto out_unlock;
3130
3131 /* Make sure dma_ops is appropriatley set */
3132 for_each_group_device(group, gdev)
3133 iommu_setup_dma_ops(dev: gdev->dev);
3134
3135out_unlock:
3136 mutex_unlock(lock: &group->mutex);
3137 return ret ?: count;
3138}
3139
3140/**
3141 * iommu_device_use_default_domain() - Device driver wants to handle device
3142 * DMA through the kernel DMA API.
3143 * @dev: The device.
3144 *
3145 * The device driver about to bind @dev wants to do DMA through the kernel
3146 * DMA API. Return 0 if it is allowed, otherwise an error.
3147 */
3148int iommu_device_use_default_domain(struct device *dev)
3149{
3150 /* Caller is the driver core during the pre-probe path */
3151 struct iommu_group *group = dev->iommu_group;
3152 int ret = 0;
3153
3154 if (!group)
3155 return 0;
3156
3157 mutex_lock(lock: &group->mutex);
3158 /* We may race against bus_iommu_probe() finalising groups here */
3159 if (!group->default_domain) {
3160 ret = -EPROBE_DEFER;
3161 goto unlock_out;
3162 }
3163 if (group->owner_cnt) {
3164 if (group->domain != group->default_domain || group->owner ||
3165 !xa_empty(xa: &group->pasid_array)) {
3166 ret = -EBUSY;
3167 goto unlock_out;
3168 }
3169 }
3170
3171 group->owner_cnt++;
3172
3173unlock_out:
3174 mutex_unlock(lock: &group->mutex);
3175 return ret;
3176}
3177
3178/**
3179 * iommu_device_unuse_default_domain() - Device driver stops handling device
3180 * DMA through the kernel DMA API.
3181 * @dev: The device.
3182 *
3183 * The device driver doesn't want to do DMA through kernel DMA API anymore.
3184 * It must be called after iommu_device_use_default_domain().
3185 */
3186void iommu_device_unuse_default_domain(struct device *dev)
3187{
3188 /* Caller is the driver core during the post-probe path */
3189 struct iommu_group *group = dev->iommu_group;
3190
3191 if (!group)
3192 return;
3193
3194 mutex_lock(lock: &group->mutex);
3195 if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
3196 group->owner_cnt--;
3197
3198 mutex_unlock(lock: &group->mutex);
3199}
3200
3201static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
3202{
3203 struct device *dev = iommu_group_first_dev(group);
3204 const struct iommu_ops *ops = dev_iommu_ops(dev);
3205 struct iommu_domain *domain;
3206
3207 if (group->blocking_domain)
3208 return 0;
3209
3210 if (ops->blocked_domain) {
3211 group->blocking_domain = ops->blocked_domain;
3212 return 0;
3213 }
3214
3215 /*
3216 * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED create an
3217 * empty PAGING domain instead.
3218 */
3219 domain = iommu_paging_domain_alloc(dev);
3220 if (IS_ERR(ptr: domain))
3221 return PTR_ERR(ptr: domain);
3222 group->blocking_domain = domain;
3223 return 0;
3224}
3225
3226static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
3227{
3228 int ret;
3229
3230 if ((group->domain && group->domain != group->default_domain) ||
3231 !xa_empty(xa: &group->pasid_array))
3232 return -EBUSY;
3233
3234 ret = __iommu_group_alloc_blocking_domain(group);
3235 if (ret)
3236 return ret;
3237 ret = __iommu_group_set_domain(group, new_domain: group->blocking_domain);
3238 if (ret)
3239 return ret;
3240
3241 group->owner = owner;
3242 group->owner_cnt++;
3243 return 0;
3244}
3245
3246/**
3247 * iommu_group_claim_dma_owner() - Set DMA ownership of a group
3248 * @group: The group.
3249 * @owner: Caller specified pointer. Used for exclusive ownership.
3250 *
3251 * This is to support backward compatibility for vfio which manages the dma
3252 * ownership in iommu_group level. New invocations on this interface should be
3253 * prohibited. Only a single owner may exist for a group.
3254 */
3255int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
3256{
3257 int ret = 0;
3258
3259 if (WARN_ON(!owner))
3260 return -EINVAL;
3261
3262 mutex_lock(lock: &group->mutex);
3263 if (group->owner_cnt) {
3264 ret = -EPERM;
3265 goto unlock_out;
3266 }
3267
3268 ret = __iommu_take_dma_ownership(group, owner);
3269unlock_out:
3270 mutex_unlock(lock: &group->mutex);
3271
3272 return ret;
3273}
3274EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
3275
3276/**
3277 * iommu_device_claim_dma_owner() - Set DMA ownership of a device
3278 * @dev: The device.
3279 * @owner: Caller specified pointer. Used for exclusive ownership.
3280 *
3281 * Claim the DMA ownership of a device. Multiple devices in the same group may
3282 * concurrently claim ownership if they present the same owner value. Returns 0
3283 * on success and error code on failure
3284 */
3285int iommu_device_claim_dma_owner(struct device *dev, void *owner)
3286{
3287 /* Caller must be a probed driver on dev */
3288 struct iommu_group *group = dev->iommu_group;
3289 int ret = 0;
3290
3291 if (WARN_ON(!owner))
3292 return -EINVAL;
3293
3294 if (!group)
3295 return -ENODEV;
3296
3297 mutex_lock(lock: &group->mutex);
3298 if (group->owner_cnt) {
3299 if (group->owner != owner) {
3300 ret = -EPERM;
3301 goto unlock_out;
3302 }
3303 group->owner_cnt++;
3304 goto unlock_out;
3305 }
3306
3307 ret = __iommu_take_dma_ownership(group, owner);
3308unlock_out:
3309 mutex_unlock(lock: &group->mutex);
3310 return ret;
3311}
3312EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
3313
3314static void __iommu_release_dma_ownership(struct iommu_group *group)
3315{
3316 if (WARN_ON(!group->owner_cnt || !group->owner ||
3317 !xa_empty(&group->pasid_array)))
3318 return;
3319
3320 group->owner_cnt = 0;
3321 group->owner = NULL;
3322 __iommu_group_set_domain_nofail(group, new_domain: group->default_domain);
3323}
3324
3325/**
3326 * iommu_group_release_dma_owner() - Release DMA ownership of a group
3327 * @group: The group
3328 *
3329 * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3330 */
3331void iommu_group_release_dma_owner(struct iommu_group *group)
3332{
3333 mutex_lock(lock: &group->mutex);
3334 __iommu_release_dma_ownership(group);
3335 mutex_unlock(lock: &group->mutex);
3336}
3337EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
3338
3339/**
3340 * iommu_device_release_dma_owner() - Release DMA ownership of a device
3341 * @dev: The device.
3342 *
3343 * Release the DMA ownership claimed by iommu_device_claim_dma_owner().
3344 */
3345void iommu_device_release_dma_owner(struct device *dev)
3346{
3347 /* Caller must be a probed driver on dev */
3348 struct iommu_group *group = dev->iommu_group;
3349
3350 mutex_lock(lock: &group->mutex);
3351 if (group->owner_cnt > 1)
3352 group->owner_cnt--;
3353 else
3354 __iommu_release_dma_ownership(group);
3355 mutex_unlock(lock: &group->mutex);
3356}
3357EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
3358
3359/**
3360 * iommu_group_dma_owner_claimed() - Query group dma ownership status
3361 * @group: The group.
3362 *
3363 * This provides status query on a given group. It is racy and only for
3364 * non-binding status reporting.
3365 */
3366bool iommu_group_dma_owner_claimed(struct iommu_group *group)
3367{
3368 unsigned int user;
3369
3370 mutex_lock(lock: &group->mutex);
3371 user = group->owner_cnt;
3372 mutex_unlock(lock: &group->mutex);
3373
3374 return user;
3375}
3376EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3377
3378static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
3379 struct iommu_domain *domain)
3380{
3381 const struct iommu_ops *ops = dev_iommu_ops(dev);
3382 struct iommu_domain *blocked_domain = ops->blocked_domain;
3383
3384 WARN_ON(blocked_domain->ops->set_dev_pasid(blocked_domain,
3385 dev, pasid, domain));
3386}
3387
3388static int __iommu_set_group_pasid(struct iommu_domain *domain,
3389 struct iommu_group *group, ioasid_t pasid,
3390 struct iommu_domain *old)
3391{
3392 struct group_device *device, *last_gdev;
3393 int ret;
3394
3395 for_each_group_device(group, device) {
3396 if (device->dev->iommu->max_pasids > 0) {
3397 ret = domain->ops->set_dev_pasid(domain, device->dev,
3398 pasid, old);
3399 if (ret)
3400 goto err_revert;
3401 }
3402 }
3403
3404 return 0;
3405
3406err_revert:
3407 last_gdev = device;
3408 for_each_group_device(group, device) {
3409 if (device == last_gdev)
3410 break;
3411 if (device->dev->iommu->max_pasids > 0) {
3412 /*
3413 * If no old domain, undo the succeeded devices/pasid.
3414 * Otherwise, rollback the succeeded devices/pasid to
3415 * the old domain. And it is a driver bug to fail
3416 * attaching with a previously good domain.
3417 */
3418 if (!old ||
3419 WARN_ON(old->ops->set_dev_pasid(old, device->dev,
3420 pasid, domain)))
3421 iommu_remove_dev_pasid(dev: device->dev, pasid, domain);
3422 }
3423 }
3424 return ret;
3425}
3426
3427static void __iommu_remove_group_pasid(struct iommu_group *group,
3428 ioasid_t pasid,
3429 struct iommu_domain *domain)
3430{
3431 struct group_device *device;
3432
3433 for_each_group_device(group, device) {
3434 if (device->dev->iommu->max_pasids > 0)
3435 iommu_remove_dev_pasid(dev: device->dev, pasid, domain);
3436 }
3437}
3438
3439/*
3440 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3441 * @domain: the iommu domain.
3442 * @dev: the attached device.
3443 * @pasid: the pasid of the device.
3444 * @handle: the attach handle.
3445 *
3446 * Caller should always provide a new handle to avoid race with the paths
3447 * that have lockless reference to handle if it intends to pass a valid handle.
3448 *
3449 * Return: 0 on success, or an error.
3450 */
3451int iommu_attach_device_pasid(struct iommu_domain *domain,
3452 struct device *dev, ioasid_t pasid,
3453 struct iommu_attach_handle *handle)
3454{
3455 /* Caller must be a probed driver on dev */
3456 struct iommu_group *group = dev->iommu_group;
3457 struct group_device *device;
3458 const struct iommu_ops *ops;
3459 void *entry;
3460 int ret;
3461
3462 if (!group)
3463 return -ENODEV;
3464
3465 ops = dev_iommu_ops(dev);
3466
3467 if (!domain->ops->set_dev_pasid ||
3468 !ops->blocked_domain ||
3469 !ops->blocked_domain->ops->set_dev_pasid)
3470 return -EOPNOTSUPP;
3471
3472 if (!domain_iommu_ops_compatible(ops, domain) ||
3473 pasid == IOMMU_NO_PASID)
3474 return -EINVAL;
3475
3476 mutex_lock(lock: &group->mutex);
3477 for_each_group_device(group, device) {
3478 /*
3479 * Skip PASID validation for devices without PASID support
3480 * (max_pasids = 0). These devices cannot issue transactions
3481 * with PASID, so they don't affect group's PASID usage.
3482 */
3483 if ((device->dev->iommu->max_pasids > 0) &&
3484 (pasid >= device->dev->iommu->max_pasids)) {
3485 ret = -EINVAL;
3486 goto out_unlock;
3487 }
3488 }
3489
3490 entry = iommu_make_pasid_array_entry(domain, handle);
3491
3492 /*
3493 * Entry present is a failure case. Use xa_insert() instead of
3494 * xa_reserve().
3495 */
3496 ret = xa_insert(xa: &group->pasid_array, index: pasid, XA_ZERO_ENTRY, GFP_KERNEL);
3497 if (ret)
3498 goto out_unlock;
3499
3500 ret = __iommu_set_group_pasid(domain, group, pasid, NULL);
3501 if (ret) {
3502 xa_release(xa: &group->pasid_array, index: pasid);
3503 goto out_unlock;
3504 }
3505
3506 /*
3507 * The xa_insert() above reserved the memory, and the group->mutex is
3508 * held, this cannot fail. The new domain cannot be visible until the
3509 * operation succeeds as we cannot tolerate PRIs becoming concurrently
3510 * queued and then failing attach.
3511 */
3512 WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3513 pasid, entry, GFP_KERNEL)));
3514
3515out_unlock:
3516 mutex_unlock(lock: &group->mutex);
3517 return ret;
3518}
3519EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3520
3521/**
3522 * iommu_replace_device_pasid - Replace the domain that a specific pasid
3523 * of the device is attached to
3524 * @domain: the new iommu domain
3525 * @dev: the attached device.
3526 * @pasid: the pasid of the device.
3527 * @handle: the attach handle.
3528 *
3529 * This API allows the pasid to switch domains. The @pasid should have been
3530 * attached. Otherwise, this fails. The pasid will keep the old configuration
3531 * if replacement failed.
3532 *
3533 * Caller should always provide a new handle to avoid race with the paths
3534 * that have lockless reference to handle if it intends to pass a valid handle.
3535 *
3536 * Return 0 on success, or an error.
3537 */
3538int iommu_replace_device_pasid(struct iommu_domain *domain,
3539 struct device *dev, ioasid_t pasid,
3540 struct iommu_attach_handle *handle)
3541{
3542 /* Caller must be a probed driver on dev */
3543 struct iommu_group *group = dev->iommu_group;
3544 struct iommu_attach_handle *entry;
3545 struct iommu_domain *curr_domain;
3546 void *curr;
3547 int ret;
3548
3549 if (!group)
3550 return -ENODEV;
3551
3552 if (!domain->ops->set_dev_pasid)
3553 return -EOPNOTSUPP;
3554
3555 if (!domain_iommu_ops_compatible(ops: dev_iommu_ops(dev), domain) ||
3556 pasid == IOMMU_NO_PASID || !handle)
3557 return -EINVAL;
3558
3559 mutex_lock(lock: &group->mutex);
3560 entry = iommu_make_pasid_array_entry(domain, handle);
3561 curr = xa_cmpxchg(xa: &group->pasid_array, index: pasid, NULL,
3562 XA_ZERO_ENTRY, GFP_KERNEL);
3563 if (xa_is_err(entry: curr)) {
3564 ret = xa_err(entry: curr);
3565 goto out_unlock;
3566 }
3567
3568 /*
3569 * No domain (with or without handle) attached, hence not
3570 * a replace case.
3571 */
3572 if (!curr) {
3573 xa_release(xa: &group->pasid_array, index: pasid);
3574 ret = -EINVAL;
3575 goto out_unlock;
3576 }
3577
3578 /*
3579 * Reusing handle is problematic as there are paths that refers
3580 * the handle without lock. To avoid race, reject the callers that
3581 * attempt it.
3582 */
3583 if (curr == entry) {
3584 WARN_ON(1);
3585 ret = -EINVAL;
3586 goto out_unlock;
3587 }
3588
3589 curr_domain = pasid_array_entry_to_domain(entry: curr);
3590 ret = 0;
3591
3592 if (curr_domain != domain) {
3593 ret = __iommu_set_group_pasid(domain, group,
3594 pasid, old: curr_domain);
3595 if (ret)
3596 goto out_unlock;
3597 }
3598
3599 /*
3600 * The above xa_cmpxchg() reserved the memory, and the
3601 * group->mutex is held, this cannot fail.
3602 */
3603 WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3604 pasid, entry, GFP_KERNEL)));
3605
3606out_unlock:
3607 mutex_unlock(lock: &group->mutex);
3608 return ret;
3609}
3610EXPORT_SYMBOL_NS_GPL(iommu_replace_device_pasid, "IOMMUFD_INTERNAL");
3611
3612/*
3613 * iommu_detach_device_pasid() - Detach the domain from pasid of device
3614 * @domain: the iommu domain.
3615 * @dev: the attached device.
3616 * @pasid: the pasid of the device.
3617 *
3618 * The @domain must have been attached to @pasid of the @dev with
3619 * iommu_attach_device_pasid().
3620 */
3621void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3622 ioasid_t pasid)
3623{
3624 /* Caller must be a probed driver on dev */
3625 struct iommu_group *group = dev->iommu_group;
3626
3627 mutex_lock(lock: &group->mutex);
3628 __iommu_remove_group_pasid(group, pasid, domain);
3629 xa_erase(&group->pasid_array, index: pasid);
3630 mutex_unlock(lock: &group->mutex);
3631}
3632EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3633
3634ioasid_t iommu_alloc_global_pasid(struct device *dev)
3635{
3636 int ret;
3637
3638 /* max_pasids == 0 means that the device does not support PASID */
3639 if (!dev->iommu->max_pasids)
3640 return IOMMU_PASID_INVALID;
3641
3642 /*
3643 * max_pasids is set up by vendor driver based on number of PASID bits
3644 * supported but the IDA allocation is inclusive.
3645 */
3646 ret = ida_alloc_range(&iommu_global_pasid_ida, IOMMU_FIRST_GLOBAL_PASID,
3647 max: dev->iommu->max_pasids - 1, GFP_KERNEL);
3648 return ret < 0 ? IOMMU_PASID_INVALID : ret;
3649}
3650EXPORT_SYMBOL_GPL(iommu_alloc_global_pasid);
3651
3652void iommu_free_global_pasid(ioasid_t pasid)
3653{
3654 if (WARN_ON(pasid == IOMMU_PASID_INVALID))
3655 return;
3656
3657 ida_free(&iommu_global_pasid_ida, id: pasid);
3658}
3659EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
3660
3661/**
3662 * iommu_attach_handle_get - Return the attach handle
3663 * @group: the iommu group that domain was attached to
3664 * @pasid: the pasid within the group
3665 * @type: matched domain type, 0 for any match
3666 *
3667 * Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
3668 *
3669 * Return the attach handle to the caller. The life cycle of an iommu attach
3670 * handle is from the time when the domain is attached to the time when the
3671 * domain is detached. Callers are required to synchronize the call of
3672 * iommu_attach_handle_get() with domain attachment and detachment. The attach
3673 * handle can only be used during its life cycle.
3674 */
3675struct iommu_attach_handle *
3676iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
3677{
3678 struct iommu_attach_handle *handle;
3679 void *entry;
3680
3681 xa_lock(&group->pasid_array);
3682 entry = xa_load(&group->pasid_array, index: pasid);
3683 if (!entry || xa_pointer_tag(entry) != IOMMU_PASID_ARRAY_HANDLE) {
3684 handle = ERR_PTR(error: -ENOENT);
3685 } else {
3686 handle = xa_untag_pointer(entry);
3687 if (type && handle->domain->type != type)
3688 handle = ERR_PTR(error: -EBUSY);
3689 }
3690 xa_unlock(&group->pasid_array);
3691
3692 return handle;
3693}
3694EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, "IOMMUFD_INTERNAL");
3695
3696/**
3697 * iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
3698 * @domain: IOMMU domain to attach
3699 * @group: IOMMU group that will be attached
3700 * @handle: attach handle
3701 *
3702 * Returns 0 on success and error code on failure.
3703 *
3704 * This is a variant of iommu_attach_group(). It allows the caller to provide
3705 * an attach handle and use it when the domain is attached. This is currently
3706 * used by IOMMUFD to deliver the I/O page faults.
3707 *
3708 * Caller should always provide a new handle to avoid race with the paths
3709 * that have lockless reference to handle.
3710 */
3711int iommu_attach_group_handle(struct iommu_domain *domain,
3712 struct iommu_group *group,
3713 struct iommu_attach_handle *handle)
3714{
3715 void *entry;
3716 int ret;
3717
3718 if (!handle)
3719 return -EINVAL;
3720
3721 mutex_lock(lock: &group->mutex);
3722 entry = iommu_make_pasid_array_entry(domain, handle);
3723 ret = xa_insert(xa: &group->pasid_array,
3724 IOMMU_NO_PASID, XA_ZERO_ENTRY, GFP_KERNEL);
3725 if (ret)
3726 goto out_unlock;
3727
3728 ret = __iommu_attach_group(domain, group);
3729 if (ret) {
3730 xa_release(xa: &group->pasid_array, IOMMU_NO_PASID);
3731 goto out_unlock;
3732 }
3733
3734 /*
3735 * The xa_insert() above reserved the memory, and the group->mutex is
3736 * held, this cannot fail. The new domain cannot be visible until the
3737 * operation succeeds as we cannot tolerate PRIs becoming concurrently
3738 * queued and then failing attach.
3739 */
3740 WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3741 IOMMU_NO_PASID, entry, GFP_KERNEL)));
3742
3743out_unlock:
3744 mutex_unlock(lock: &group->mutex);
3745 return ret;
3746}
3747EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, "IOMMUFD_INTERNAL");
3748
3749/**
3750 * iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
3751 * @domain: IOMMU domain to attach
3752 * @group: IOMMU group that will be attached
3753 *
3754 * Detach the specified IOMMU domain from the specified IOMMU group.
3755 * It must be used in conjunction with iommu_attach_group_handle().
3756 */
3757void iommu_detach_group_handle(struct iommu_domain *domain,
3758 struct iommu_group *group)
3759{
3760 mutex_lock(lock: &group->mutex);
3761 __iommu_group_set_core_domain(group);
3762 xa_erase(&group->pasid_array, IOMMU_NO_PASID);
3763 mutex_unlock(lock: &group->mutex);
3764}
3765EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, "IOMMUFD_INTERNAL");
3766
3767/**
3768 * iommu_replace_group_handle - replace the domain that a group is attached to
3769 * @group: IOMMU group that will be attached to the new domain
3770 * @new_domain: new IOMMU domain to replace with
3771 * @handle: attach handle
3772 *
3773 * This API allows the group to switch domains without being forced to go to
3774 * the blocking domain in-between. It allows the caller to provide an attach
3775 * handle for the new domain and use it when the domain is attached.
3776 *
3777 * If the currently attached domain is a core domain (e.g. a default_domain),
3778 * it will act just like the iommu_attach_group_handle().
3779 *
3780 * Caller should always provide a new handle to avoid race with the paths
3781 * that have lockless reference to handle.
3782 */
3783int iommu_replace_group_handle(struct iommu_group *group,
3784 struct iommu_domain *new_domain,
3785 struct iommu_attach_handle *handle)
3786{
3787 void *curr, *entry;
3788 int ret;
3789
3790 if (!new_domain || !handle)
3791 return -EINVAL;
3792
3793 mutex_lock(lock: &group->mutex);
3794 entry = iommu_make_pasid_array_entry(domain: new_domain, handle);
3795 ret = xa_reserve(xa: &group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
3796 if (ret)
3797 goto err_unlock;
3798
3799 ret = __iommu_group_set_domain(group, new_domain);
3800 if (ret)
3801 goto err_release;
3802
3803 curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, entry, GFP_KERNEL);
3804 WARN_ON(xa_is_err(curr));
3805
3806 mutex_unlock(lock: &group->mutex);
3807
3808 return 0;
3809err_release:
3810 xa_release(xa: &group->pasid_array, IOMMU_NO_PASID);
3811err_unlock:
3812 mutex_unlock(lock: &group->mutex);
3813 return ret;
3814}
3815EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
3816
3817#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
3818/**
3819 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
3820 * @desc: MSI descriptor, will store the MSI page
3821 * @msi_addr: MSI target address to be mapped
3822 *
3823 * The implementation of sw_msi() should take msi_addr and map it to
3824 * an IOVA in the domain and call msi_desc_set_iommu_msi_iova() with the
3825 * mapping information.
3826 *
3827 * Return: 0 on success or negative error code if the mapping failed.
3828 */
3829int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
3830{
3831 struct device *dev = msi_desc_to_dev(desc);
3832 struct iommu_group *group = dev->iommu_group;
3833 int ret = 0;
3834
3835 if (!group)
3836 return 0;
3837
3838 mutex_lock(&group->mutex);
3839 /* An IDENTITY domain must pass through */
3840 if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
3841 switch (group->domain->cookie_type) {
3842 case IOMMU_COOKIE_DMA_MSI:
3843 case IOMMU_COOKIE_DMA_IOVA:
3844 ret = iommu_dma_sw_msi(group->domain, desc, msi_addr);
3845 break;
3846 case IOMMU_COOKIE_IOMMUFD:
3847 ret = iommufd_sw_msi(group->domain, desc, msi_addr);
3848 break;
3849 default:
3850 ret = -EOPNOTSUPP;
3851 break;
3852 }
3853 }
3854 mutex_unlock(&group->mutex);
3855 return ret;
3856}
3857#endif /* CONFIG_IRQ_MSI_IOMMU */
3858