1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2024 Advanced Micro Devices, Inc.
4 */
5
6#define pr_fmt(fmt) "AMD-Vi: " fmt
7#define dev_fmt(fmt) pr_fmt(fmt)
8
9#include <linux/iommu.h>
10#include <linux/mm_types.h>
11
12#include "amd_iommu.h"
13
14static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data)
15{
16 if (dev_data->pasid_enabled && dev_data->max_pasids &&
17 dev_data->gcr3_info.gcr3_tbl != NULL)
18 return true;
19
20 return false;
21}
22
23static inline bool is_pasid_valid(struct iommu_dev_data *dev_data,
24 ioasid_t pasid)
25{
26 if (pasid > 0 && pasid < dev_data->max_pasids)
27 return true;
28
29 return false;
30}
31
32static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data)
33{
34 /* Update GCR3 table and flush IOTLB */
35 amd_iommu_clear_gcr3(dev_data: pdom_dev_data->dev_data, pasid: pdom_dev_data->pasid);
36
37 list_del(entry: &pdom_dev_data->list);
38 kfree(objp: pdom_dev_data);
39}
40
41/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */
42static void remove_pdom_dev_pasid(struct protection_domain *pdom,
43 struct device *dev, ioasid_t pasid)
44{
45 struct pdom_dev_data *pdom_dev_data;
46 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
47
48 lockdep_assert_held(&pdom->lock);
49
50 for_each_pdom_dev_data(pdom_dev_data, pdom) {
51 if (pdom_dev_data->dev_data == dev_data &&
52 pdom_dev_data->pasid == pasid) {
53 remove_dev_pasid(pdom_dev_data);
54 break;
55 }
56 }
57}
58
59static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
60 struct mm_struct *mm,
61 unsigned long start, unsigned long end)
62{
63 struct pdom_dev_data *pdom_dev_data;
64 struct protection_domain *sva_pdom;
65 unsigned long flags;
66
67 sva_pdom = container_of(mn, struct protection_domain, mn);
68
69 spin_lock_irqsave(&sva_pdom->lock, flags);
70
71 for_each_pdom_dev_data(pdom_dev_data, sva_pdom) {
72 amd_iommu_dev_flush_pasid_pages(dev_data: pdom_dev_data->dev_data,
73 pasid: pdom_dev_data->pasid,
74 address: start, size: end - start);
75 }
76
77 spin_unlock_irqrestore(lock: &sva_pdom->lock, flags);
78}
79
80static void sva_mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
81{
82 struct pdom_dev_data *pdom_dev_data, *next;
83 struct protection_domain *sva_pdom;
84 unsigned long flags;
85
86 sva_pdom = container_of(mn, struct protection_domain, mn);
87
88 spin_lock_irqsave(&sva_pdom->lock, flags);
89
90 /* Assume dev_data_list contains same PASID with different devices */
91 for_each_pdom_dev_data_safe(pdom_dev_data, next, sva_pdom)
92 remove_dev_pasid(pdom_dev_data);
93
94 spin_unlock_irqrestore(lock: &sva_pdom->lock, flags);
95}
96
97static const struct mmu_notifier_ops sva_mn = {
98 .arch_invalidate_secondary_tlbs = sva_arch_invalidate_secondary_tlbs,
99 .release = sva_mn_release,
100};
101
102int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
103 struct device *dev, ioasid_t pasid,
104 struct iommu_domain *old)
105{
106 struct pdom_dev_data *pdom_dev_data;
107 struct protection_domain *sva_pdom = to_pdomain(dom: domain);
108 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
109 unsigned long flags;
110 int ret = -EINVAL;
111
112 if (old)
113 return -EOPNOTSUPP;
114
115 /* PASID zero is used for requests from the I/O device without PASID */
116 if (!is_pasid_valid(dev_data, pasid))
117 return ret;
118
119 /* Make sure PASID is enabled */
120 if (!is_pasid_enabled(dev_data))
121 return ret;
122
123 /* Add PASID to protection domain pasid list */
124 pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL);
125 if (pdom_dev_data == NULL)
126 return ret;
127
128 pdom_dev_data->pasid = pasid;
129 pdom_dev_data->dev_data = dev_data;
130
131 spin_lock_irqsave(&sva_pdom->lock, flags);
132
133 /* Setup GCR3 table */
134 ret = amd_iommu_set_gcr3(dev_data, pasid,
135 gcr3: iommu_virt_to_phys(vaddr: domain->mm->pgd));
136 if (ret) {
137 kfree(objp: pdom_dev_data);
138 goto out_unlock;
139 }
140
141 list_add(new: &pdom_dev_data->list, head: &sva_pdom->dev_data_list);
142
143out_unlock:
144 spin_unlock_irqrestore(lock: &sva_pdom->lock, flags);
145 return ret;
146}
147
148void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
149 struct iommu_domain *domain)
150{
151 struct protection_domain *sva_pdom;
152 unsigned long flags;
153
154 if (!is_pasid_valid(dev_data: dev_iommu_priv_get(dev), pasid))
155 return;
156
157 sva_pdom = to_pdomain(dom: domain);
158
159 spin_lock_irqsave(&sva_pdom->lock, flags);
160
161 /* Remove PASID from dev_data_list */
162 remove_pdom_dev_pasid(pdom: sva_pdom, dev, pasid);
163
164 spin_unlock_irqrestore(lock: &sva_pdom->lock, flags);
165}
166
167static void iommu_sva_domain_free(struct iommu_domain *domain)
168{
169 struct protection_domain *sva_pdom = to_pdomain(dom: domain);
170
171 if (sva_pdom->mn.ops)
172 mmu_notifier_unregister(subscription: &sva_pdom->mn, mm: domain->mm);
173
174 amd_iommu_domain_free(dom: domain);
175}
176
177static const struct iommu_domain_ops amd_sva_domain_ops = {
178 .set_dev_pasid = iommu_sva_set_dev_pasid,
179 .free = iommu_sva_domain_free
180};
181
182struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
183 struct mm_struct *mm)
184{
185 struct protection_domain *pdom;
186 int ret;
187
188 pdom = protection_domain_alloc();
189 if (!pdom)
190 return ERR_PTR(error: -ENOMEM);
191
192 pdom->domain.ops = &amd_sva_domain_ops;
193 pdom->mn.ops = &sva_mn;
194 pdom->domain.type = IOMMU_DOMAIN_SVA;
195
196 ret = mmu_notifier_register(subscription: &pdom->mn, mm);
197 if (ret) {
198 amd_iommu_domain_free(dom: &pdom->domain);
199 return ERR_PTR(error: ret);
200 }
201
202 return &pdom->domain;
203}
204