1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2023 Advanced Micro Devices, Inc.
4 */
5
6#define pr_fmt(fmt) "AMD-Vi: " fmt
7#define dev_fmt(fmt) pr_fmt(fmt)
8
9#include <linux/amd-iommu.h>
10#include <linux/delay.h>
11#include <linux/mmu_notifier.h>
12
13#include <asm/iommu.h>
14
15#include "amd_iommu.h"
16#include "amd_iommu_types.h"
17
18#include "../iommu-pages.h"
19
20int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)
21{
22 iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
23 PPR_LOG_SIZE);
24 return iommu->ppr_log ? 0 : -ENOMEM;
25}
26
27void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)
28{
29 u64 entry;
30
31 if (iommu->ppr_log == NULL)
32 return;
33
34 iommu_feature_enable(iommu, CONTROL_PPR_EN);
35
36 entry = iommu_virt_to_phys(vaddr: iommu->ppr_log) | PPR_LOG_SIZE_512;
37
38 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
39 &entry, sizeof(entry));
40
41 /* set head and tail to zero manually */
42 writel(val: 0x00, addr: iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
43 writel(val: 0x00, addr: iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
44
45 iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
46 iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
47}
48
49void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu)
50{
51 iommu_free_pages(virt: iommu->ppr_log);
52}
53
54/*
55 * This function restarts ppr logging in case the IOMMU experienced
56 * PPR log overflow.
57 */
58void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)
59{
60 amd_iommu_restart_log(iommu, evt_type: "PPR", CONTROL_PPRINT_EN,
61 CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK,
62 MMIO_STATUS_PPR_OVERFLOW_MASK);
63}
64
65static inline u32 ppr_flag_to_fault_perm(u16 flag)
66{
67 int perm = 0;
68
69 if (flag & PPR_FLAG_READ)
70 perm |= IOMMU_FAULT_PERM_READ;
71 if (flag & PPR_FLAG_WRITE)
72 perm |= IOMMU_FAULT_PERM_WRITE;
73 if (flag & PPR_FLAG_EXEC)
74 perm |= IOMMU_FAULT_PERM_EXEC;
75 if (!(flag & PPR_FLAG_US))
76 perm |= IOMMU_FAULT_PERM_PRIV;
77
78 return perm;
79}
80
81static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw)
82{
83 struct device *dev = iommu->iommu.dev;
84 u16 devid = PPR_DEVID(raw[0]);
85
86 if (!(PPR_FLAGS(raw[0]) & PPR_FLAG_GN)) {
87 dev_dbg(dev, "PPR logged [Request ignored due to GN=0 (device=%04x:%02x:%02x.%x "
88 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n",
89 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
90 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0]));
91 return false;
92 }
93
94 if (PPR_FLAGS(raw[0]) & PPR_FLAG_RVSD) {
95 dev_dbg(dev, "PPR logged [Invalid request format (device=%04x:%02x:%02x.%x "
96 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n",
97 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
98 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0]));
99 return false;
100 }
101
102 return true;
103}
104
105static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
106{
107 struct iommu_dev_data *dev_data;
108 struct iopf_fault event;
109 struct pci_dev *pdev;
110 u16 devid = PPR_DEVID(raw[0]);
111
112 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
113 pr_info_ratelimited("Unknown PPR request received\n");
114 return;
115 }
116
117 pdev = pci_get_domain_bus_and_slot(domain: iommu->pci_seg->id,
118 PCI_BUS_NUM(devid), devfn: devid & 0xff);
119 if (!pdev)
120 return;
121
122 if (!ppr_is_valid(iommu, raw))
123 goto out;
124
125 memset(s: &event, c: 0, n: sizeof(struct iopf_fault));
126
127 event.fault.type = IOMMU_FAULT_PAGE_REQ;
128 event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0]));
129 event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK);
130 event.fault.prm.pasid = PPR_PASID(raw[0]);
131 event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF;
132
133 /*
134 * PASID zero is used for requests from the I/O device without
135 * a PASID
136 */
137 dev_data = dev_iommu_priv_get(dev: &pdev->dev);
138 if (event.fault.prm.pasid == 0 ||
139 event.fault.prm.pasid >= dev_data->max_pasids) {
140 pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n",
141 event.fault.prm.pasid, pdev->dev.id);
142 goto out;
143 }
144
145 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
146 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
147 if (PPR_TAG(raw[0]) & 0x200)
148 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
149
150 /* Submit event */
151 iommu_report_device_fault(dev: &pdev->dev, evt: &event);
152
153 return;
154
155out:
156 /* Nobody cared, abort */
157 amd_iommu_complete_ppr(dev: &pdev->dev, PPR_PASID(raw[0]),
158 status: IOMMU_PAGE_RESP_FAILURE,
159 PPR_TAG(raw[0]) & 0x1FF);
160}
161
162void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
163{
164 u32 head, tail;
165
166 if (iommu->ppr_log == NULL)
167 return;
168
169 head = readl(addr: iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
170 tail = readl(addr: iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
171
172 while (head != tail) {
173 volatile u64 *raw;
174 u64 entry[2];
175 int i;
176
177 raw = (u64 *)(iommu->ppr_log + head);
178
179 /*
180 * Hardware bug: Interrupt may arrive before the entry is
181 * written to memory. If this happens we need to wait for the
182 * entry to arrive.
183 */
184 for (i = 0; i < LOOP_TIMEOUT; ++i) {
185 if (PPR_REQ_TYPE(raw[0]) != 0)
186 break;
187 udelay(usec: 1);
188 }
189
190 /* Avoid memcpy function-call overhead */
191 entry[0] = raw[0];
192 entry[1] = raw[1];
193
194 /*
195 * To detect the hardware errata 733 we need to clear the
196 * entry back to zero. This issue does not exist on SNP
197 * enabled system. Also this buffer is not writeable on
198 * SNP enabled system.
199 */
200 if (!amd_iommu_snp_en)
201 raw[0] = raw[1] = 0UL;
202
203 /* Update head pointer of hardware ring-buffer */
204 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
205 writel(val: head, addr: iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
206
207 /* Handle PPR entry */
208 iommu_call_iopf_notifier(iommu, raw: entry);
209 }
210}
211
212/**************************************************************
213 *
214 * IOPF handling stuff
215 */
216
217/* Setup per-IOMMU IOPF queue if not exist. */
218int amd_iommu_iopf_init(struct amd_iommu *iommu)
219{
220 int ret = 0;
221
222 if (iommu->iopf_queue)
223 return ret;
224
225 snprintf(buf: iommu->iopfq_name, size: sizeof(iommu->iopfq_name), fmt: "amdvi-%#x",
226 PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid));
227
228 iommu->iopf_queue = iopf_queue_alloc(name: iommu->iopfq_name);
229 if (!iommu->iopf_queue)
230 ret = -ENOMEM;
231
232 return ret;
233}
234
235/* Destroy per-IOMMU IOPF queue if no longer needed. */
236void amd_iommu_iopf_uninit(struct amd_iommu *iommu)
237{
238 iopf_queue_free(queue: iommu->iopf_queue);
239 iommu->iopf_queue = NULL;
240}
241
242void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt,
243 struct iommu_page_response *resp)
244{
245 amd_iommu_complete_ppr(dev, pasid: resp->pasid, status: resp->code, tag: resp->grpid);
246}
247
248int amd_iommu_iopf_add_device(struct amd_iommu *iommu,
249 struct iommu_dev_data *dev_data)
250{
251 int ret = 0;
252
253 if (!dev_data->pri_enabled)
254 return ret;
255
256 if (!iommu->iopf_queue)
257 return -EINVAL;
258
259 ret = iopf_queue_add_device(queue: iommu->iopf_queue, dev: dev_data->dev);
260 if (ret)
261 return ret;
262
263 dev_data->ppr = true;
264 return 0;
265}
266
267/* Its assumed that caller has verified that device was added to iopf queue */
268void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,
269 struct iommu_dev_data *dev_data)
270{
271 iopf_queue_remove_device(queue: iommu->iopf_queue, dev: dev_data->dev);
272 dev_data->ppr = false;
273}
274