1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8#ifndef _ASM_X86_AMD_IOMMU_TYPES_H
9#define _ASM_X86_AMD_IOMMU_TYPES_H
10
11#include <linux/bitfield.h>
12#include <linux/iommu.h>
13#include <linux/types.h>
14#include <linux/mmu_notifier.h>
15#include <linux/mutex.h>
16#include <linux/msi.h>
17#include <linux/list.h>
18#include <linux/spinlock.h>
19#include <linux/pci.h>
20#include <linux/irqreturn.h>
21#include <linux/io-pgtable.h>
22
23/*
24 * Maximum number of IOMMUs supported
25 */
26#define MAX_IOMMUS 32
27
28/*
29 * some size calculation constants
30 */
31#define DEV_TABLE_ENTRY_SIZE 32
32
33/* Capability offsets used by the driver */
34#define MMIO_CAP_HDR_OFFSET 0x00
35#define MMIO_RANGE_OFFSET 0x0c
36#define MMIO_MISC_OFFSET 0x10
37
38/* Masks, shifts and macros to parse the device range capability */
39#define MMIO_RANGE_LD_MASK 0xff000000
40#define MMIO_RANGE_FD_MASK 0x00ff0000
41#define MMIO_RANGE_BUS_MASK 0x0000ff00
42#define MMIO_RANGE_LD_SHIFT 24
43#define MMIO_RANGE_FD_SHIFT 16
44#define MMIO_RANGE_BUS_SHIFT 8
45#define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT)
46#define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT)
47#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
48#define MMIO_MSI_NUM(x) ((x) & 0x1f)
49
50/* Flag masks for the AMD IOMMU exclusion range */
51#define MMIO_EXCL_ENABLE_MASK 0x01ULL
52#define MMIO_EXCL_ALLOW_MASK 0x02ULL
53
54/* Used offsets into the MMIO space */
55#define MMIO_DEV_TABLE_OFFSET 0x0000
56#define MMIO_CMD_BUF_OFFSET 0x0008
57#define MMIO_EVT_BUF_OFFSET 0x0010
58#define MMIO_CONTROL_OFFSET 0x0018
59#define MMIO_EXCL_BASE_OFFSET 0x0020
60#define MMIO_EXCL_LIMIT_OFFSET 0x0028
61#define MMIO_EXT_FEATURES 0x0030
62#define MMIO_PPR_LOG_OFFSET 0x0038
63#define MMIO_GA_LOG_BASE_OFFSET 0x00e0
64#define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
65#define MMIO_MSI_ADDR_LO_OFFSET 0x015C
66#define MMIO_MSI_ADDR_HI_OFFSET 0x0160
67#define MMIO_MSI_DATA_OFFSET 0x0164
68#define MMIO_INTCAPXT_EVT_OFFSET 0x0170
69#define MMIO_INTCAPXT_PPR_OFFSET 0x0178
70#define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
71#define MMIO_EXT_FEATURES2 0x01A0
72#define MMIO_CMD_HEAD_OFFSET 0x2000
73#define MMIO_CMD_TAIL_OFFSET 0x2008
74#define MMIO_EVT_HEAD_OFFSET 0x2010
75#define MMIO_EVT_TAIL_OFFSET 0x2018
76#define MMIO_STATUS_OFFSET 0x2020
77#define MMIO_PPR_HEAD_OFFSET 0x2030
78#define MMIO_PPR_TAIL_OFFSET 0x2038
79#define MMIO_GA_HEAD_OFFSET 0x2040
80#define MMIO_GA_TAIL_OFFSET 0x2048
81#define MMIO_CNTR_CONF_OFFSET 0x4000
82#define MMIO_CNTR_REG_OFFSET 0x40000
83#define MMIO_REG_END_OFFSET 0x80000
84
85
86
87/* Extended Feature Bits */
88#define FEATURE_PREFETCH BIT_ULL(0)
89#define FEATURE_PPR BIT_ULL(1)
90#define FEATURE_X2APIC BIT_ULL(2)
91#define FEATURE_NX BIT_ULL(3)
92#define FEATURE_GT BIT_ULL(4)
93#define FEATURE_IA BIT_ULL(6)
94#define FEATURE_GA BIT_ULL(7)
95#define FEATURE_HE BIT_ULL(8)
96#define FEATURE_PC BIT_ULL(9)
97#define FEATURE_HATS GENMASK_ULL(11, 10)
98#define FEATURE_GATS GENMASK_ULL(13, 12)
99#define FEATURE_GLX GENMASK_ULL(15, 14)
100#define FEATURE_GAM_VAPIC BIT_ULL(21)
101#define FEATURE_PASMAX GENMASK_ULL(36, 32)
102#define FEATURE_GIOSUP BIT_ULL(48)
103#define FEATURE_HASUP BIT_ULL(49)
104#define FEATURE_EPHSUP BIT_ULL(50)
105#define FEATURE_HDSUP BIT_ULL(52)
106#define FEATURE_SNP BIT_ULL(63)
107
108
109/* Extended Feature 2 Bits */
110#define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
111#define FEATURE_SNPAVICSUP_GAM(x) \
112 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
113#define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
114
115#define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
116#define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
117 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
118
119/* Note:
120 * The current driver only support 16-bit PASID.
121 * Currently, hardware only implement upto 16-bit PASID
122 * even though the spec says it could have upto 20 bits.
123 */
124#define PASID_MASK 0x0000ffff
125
126/* MMIO status bits */
127#define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0)
128#define MMIO_STATUS_EVT_INT_MASK BIT(1)
129#define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2)
130#define MMIO_STATUS_EVT_RUN_MASK BIT(3)
131#define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5)
132#define MMIO_STATUS_PPR_INT_MASK BIT(6)
133#define MMIO_STATUS_PPR_RUN_MASK BIT(7)
134#define MMIO_STATUS_GALOG_RUN_MASK BIT(8)
135#define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9)
136#define MMIO_STATUS_GALOG_INT_MASK BIT(10)
137
138/* event logging constants */
139#define EVENT_ENTRY_SIZE 0x10
140#define EVENT_TYPE_SHIFT 28
141#define EVENT_TYPE_MASK 0xf
142#define EVENT_TYPE_ILL_DEV 0x1
143#define EVENT_TYPE_IO_FAULT 0x2
144#define EVENT_TYPE_DEV_TAB_ERR 0x3
145#define EVENT_TYPE_PAGE_TAB_ERR 0x4
146#define EVENT_TYPE_ILL_CMD 0x5
147#define EVENT_TYPE_CMD_HARD_ERR 0x6
148#define EVENT_TYPE_IOTLB_INV_TO 0x7
149#define EVENT_TYPE_INV_DEV_REQ 0x8
150#define EVENT_TYPE_INV_PPR_REQ 0x9
151#define EVENT_TYPE_RMP_FAULT 0xd
152#define EVENT_TYPE_RMP_HW_ERR 0xe
153#define EVENT_DEVID_MASK 0xffff
154#define EVENT_DEVID_SHIFT 0
155#define EVENT_DOMID_MASK_LO 0xffff
156#define EVENT_DOMID_MASK_HI 0xf0000
157#define EVENT_FLAGS_MASK 0xfff
158#define EVENT_FLAGS_SHIFT 0x10
159#define EVENT_FLAG_RW 0x020
160#define EVENT_FLAG_I 0x008
161
162/* feature control bits */
163#define CONTROL_IOMMU_EN 0
164#define CONTROL_HT_TUN_EN 1
165#define CONTROL_EVT_LOG_EN 2
166#define CONTROL_EVT_INT_EN 3
167#define CONTROL_COMWAIT_EN 4
168#define CONTROL_INV_TIMEOUT 5
169#define CONTROL_PASSPW_EN 8
170#define CONTROL_RESPASSPW_EN 9
171#define CONTROL_COHERENT_EN 10
172#define CONTROL_ISOC_EN 11
173#define CONTROL_CMDBUF_EN 12
174#define CONTROL_PPRLOG_EN 13
175#define CONTROL_PPRINT_EN 14
176#define CONTROL_PPR_EN 15
177#define CONTROL_GT_EN 16
178#define CONTROL_GA_EN 17
179#define CONTROL_GAM_EN 25
180#define CONTROL_GALOG_EN 28
181#define CONTROL_GAINT_EN 29
182#define CONTROL_NUM_INT_REMAP_MODE 43
183#define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03
184#define CONTROL_NUM_INT_REMAP_MODE_2K 0x01
185#define CONTROL_EPH_EN 45
186#define CONTROL_XT_EN 50
187#define CONTROL_INTCAPXT_EN 51
188#define CONTROL_IRTCACHEDIS 59
189#define CONTROL_SNPAVIC_EN 61
190
191#define CTRL_INV_TO_MASK 7
192#define CTRL_INV_TO_NONE 0
193#define CTRL_INV_TO_1MS 1
194#define CTRL_INV_TO_10MS 2
195#define CTRL_INV_TO_100MS 3
196#define CTRL_INV_TO_1S 4
197#define CTRL_INV_TO_10S 5
198#define CTRL_INV_TO_100S 6
199
200/* command specific defines */
201#define CMD_COMPL_WAIT 0x01
202#define CMD_INV_DEV_ENTRY 0x02
203#define CMD_INV_IOMMU_PAGES 0x03
204#define CMD_INV_IOTLB_PAGES 0x04
205#define CMD_INV_IRT 0x05
206#define CMD_COMPLETE_PPR 0x07
207#define CMD_INV_ALL 0x08
208
209#define CMD_COMPL_WAIT_STORE_MASK 0x01
210#define CMD_COMPL_WAIT_INT_MASK 0x02
211#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
212#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
213#define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
214
215#define PPR_STATUS_MASK 0xf
216#define PPR_STATUS_SHIFT 12
217
218#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
219
220/* macros and definitions for device table entries */
221#define DEV_ENTRY_VALID 0x00
222#define DEV_ENTRY_TRANSLATION 0x01
223#define DEV_ENTRY_HAD 0x07
224#define DEV_ENTRY_PPR 0x34
225#define DEV_ENTRY_IR 0x3d
226#define DEV_ENTRY_IW 0x3e
227#define DEV_ENTRY_NO_PAGE_FAULT 0x62
228#define DEV_ENTRY_EX 0x67
229#define DEV_ENTRY_SYSMGT1 0x68
230#define DEV_ENTRY_SYSMGT2 0x69
231#define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
232
233#define DEV_ENTRY_IRQ_TBL_EN 0x80
234#define DEV_ENTRY_INIT_PASS 0xb8
235#define DEV_ENTRY_EINT_PASS 0xb9
236#define DEV_ENTRY_NMI_PASS 0xba
237#define DEV_ENTRY_LINT0_PASS 0xbe
238#define DEV_ENTRY_LINT1_PASS 0xbf
239#define DEV_ENTRY_MODE_MASK 0x07
240#define DEV_ENTRY_MODE_SHIFT 0x09
241
242#define MAX_DEV_TABLE_ENTRIES 0xffff
243
244/* constants to configure the command buffer */
245#define CMD_BUFFER_SIZE 8192
246#define CMD_BUFFER_UNINITIALIZED 1
247#define CMD_BUFFER_ENTRIES 512
248#define MMIO_CMD_SIZE_SHIFT 56
249#define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
250
251/* constants for event buffer handling */
252#define EVT_BUFFER_SIZE 8192 /* 512 entries */
253#define EVT_LEN_MASK (0x9ULL << 56)
254
255/* Constants for PPR Log handling */
256#define PPR_LOG_ENTRIES 512
257#define PPR_LOG_SIZE_SHIFT 56
258#define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
259#define PPR_ENTRY_SIZE 16
260#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
261
262/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
263#define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
264#define PPR_FLAG_READ 0x004 /* Read permission requested */
265#define PPR_FLAG_WRITE 0x020 /* Write permission requested */
266#define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */
267#define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */
268#define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */
269
270#define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
271#define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
272#define PPR_DEVID(x) ((x) & 0xffffULL)
273#define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
274#define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
275#define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
276#define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
277
278#define PPR_REQ_FAULT 0x01
279
280/* Constants for GA Log handling */
281#define GA_LOG_ENTRIES 512
282#define GA_LOG_SIZE_SHIFT 56
283#define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
284#define GA_ENTRY_SIZE 8
285#define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)
286
287#define GA_TAG(x) (u32)(x & 0xffffffffULL)
288#define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
289#define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
290
291#define GA_GUEST_NR 0x1
292
293#define IOMMU_IN_ADDR_BIT_SIZE 52
294#define IOMMU_OUT_ADDR_BIT_SIZE 52
295
296/*
297 * This bitmap is used to advertise the page sizes our hardware support
298 * to the IOMMU core, which will then use this information to split
299 * physically contiguous memory regions it is mapping into page sizes
300 * that we support.
301 *
302 * 512GB Pages are not supported due to a hardware bug
303 * Page sizes >= the 52 bit max physical address of the CPU are not supported.
304 */
305#define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G)
306
307/* Special mode where page-sizes are limited to 4 KiB */
308#define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE)
309
310/* 4K, 2MB, 1G page sizes are supported */
311#define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30))
312
313/* Bit value definition for dte irq remapping fields*/
314#define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6)
315#define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
316#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
317#define DTE_IRQ_REMAP_ENABLE 1ULL
318
319#define DTE_INTTAB_ALIGNMENT 128
320#define DTE_INTTABLEN_MASK (0xfULL << 1)
321#define DTE_INTTABLEN_VALUE_512 9ULL
322#define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1)
323#define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512)
324#define DTE_INTTABLEN_VALUE_2K 11ULL
325#define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1)
326#define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K)
327
328#define PAGE_MODE_NONE 0x00
329#define PAGE_MODE_1_LEVEL 0x01
330#define PAGE_MODE_2_LEVEL 0x02
331#define PAGE_MODE_3_LEVEL 0x03
332#define PAGE_MODE_4_LEVEL 0x04
333#define PAGE_MODE_5_LEVEL 0x05
334#define PAGE_MODE_6_LEVEL 0x06
335#define PAGE_MODE_7_LEVEL 0x07
336
337#define GUEST_PGTABLE_4_LEVEL 0x00
338#define GUEST_PGTABLE_5_LEVEL 0x01
339
340#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9))
341#define PM_LEVEL_SIZE(x) (((x) < 6) ? \
342 ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
343 (0xffffffffffffffffULL))
344#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
345#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL)
346#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \
347 IOMMU_PTE_PR | IOMMU_PTE_IR | IOMMU_PTE_IW)
348#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL)
349
350#define PM_MAP_4k 0
351#define PM_ADDR_MASK 0x000ffffffffff000ULL
352#define PM_MAP_MASK(lvl) (PM_ADDR_MASK & \
353 (~((1ULL << (12 + ((lvl) * 9))) - 1)))
354#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr))
355
356/*
357 * Returns the page table level to use for a given page size
358 * Pagesize is expected to be a power-of-two
359 */
360#define PAGE_SIZE_LEVEL(pagesize) \
361 ((__ffs(pagesize) - 12) / 9)
362/*
363 * Returns the number of ptes to use for a given page size
364 * Pagesize is expected to be a power-of-two
365 */
366#define PAGE_SIZE_PTE_COUNT(pagesize) \
367 (1ULL << ((__ffs(pagesize) - 12) % 9))
368
369/*
370 * Aligns a given io-virtual address to a given page size
371 * Pagesize is expected to be a power-of-two
372 */
373#define PAGE_SIZE_ALIGN(address, pagesize) \
374 ((address) & ~((pagesize) - 1))
375/*
376 * Creates an IOMMU PTE for an address and a given pagesize
377 * The PTE has no permission bits set
378 * Pagesize is expected to be a power-of-two larger than 4096
379 */
380#define PAGE_SIZE_PTE(address, pagesize) \
381 (((address) | ((pagesize) - 1)) & \
382 (~(pagesize >> 1)) & PM_ADDR_MASK)
383
384/*
385 * Takes a PTE value with mode=0x07 and returns the page size it maps
386 */
387#define PTE_PAGE_SIZE(pte) \
388 (1ULL << (1 + ffz(((pte) | 0xfffULL))))
389
390/*
391 * Takes a page-table level and returns the default page-size for this level
392 */
393#define PTE_LEVEL_PAGE_SIZE(level) \
394 (1ULL << (12 + (9 * (level))))
395
396/*
397 * The IOPTE dirty bit
398 */
399#define IOMMU_PTE_HD_BIT (6)
400
401/*
402 * Bit value definition for I/O PTE fields
403 */
404#define IOMMU_PTE_PR BIT_ULL(0)
405#define IOMMU_PTE_HD BIT_ULL(IOMMU_PTE_HD_BIT)
406#define IOMMU_PTE_U BIT_ULL(59)
407#define IOMMU_PTE_FC BIT_ULL(60)
408#define IOMMU_PTE_IR BIT_ULL(61)
409#define IOMMU_PTE_IW BIT_ULL(62)
410
411/*
412 * Bit value definition for DTE fields
413 */
414#define DTE_FLAG_V BIT_ULL(0)
415#define DTE_FLAG_TV BIT_ULL(1)
416#define DTE_FLAG_HAD (3ULL << 7)
417#define DTE_FLAG_GIOV BIT_ULL(54)
418#define DTE_FLAG_GV BIT_ULL(55)
419#define DTE_GLX GENMASK_ULL(57, 56)
420#define DTE_FLAG_IR BIT_ULL(61)
421#define DTE_FLAG_IW BIT_ULL(62)
422
423#define DTE_FLAG_IOTLB BIT_ULL(32)
424#define DTE_FLAG_MASK (0x3ffULL << 32)
425#define DEV_DOMID_MASK 0xffffULL
426
427#define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
428#define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
429#define DTE_GCR3_51_31 GENMASK_ULL(63, 43)
430
431#define DTE_GPT_LEVEL_SHIFT 54
432#define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
433
434#define GCR3_VALID 0x01ULL
435
436/* DTE[128:179] | DTE[184:191] */
437#define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52)
438
439#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
440#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR)
441#define IOMMU_PTE_DIRTY(pte) ((pte) & IOMMU_PTE_HD)
442#define IOMMU_PTE_PAGE(pte) (iommu_phys_to_virt((pte) & IOMMU_PAGE_MASK))
443#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
444
445#define IOMMU_PROT_MASK 0x03
446#define IOMMU_PROT_IR 0x01
447#define IOMMU_PROT_IW 0x02
448
449#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE (1 << 2)
450
451/* IOMMU capabilities */
452#define IOMMU_CAP_IOTLB 24
453#define IOMMU_CAP_NPCACHE 26
454#define IOMMU_CAP_EFR 27
455
456/* IOMMU IVINFO */
457#define IOMMU_IVINFO_OFFSET 36
458#define IOMMU_IVINFO_EFRSUP BIT(0)
459#define IOMMU_IVINFO_DMA_REMAP BIT(1)
460
461/* IOMMU Feature Reporting Field (for IVHD type 10h */
462#define IOMMU_FEAT_GASUP_SHIFT 6
463
464/* IOMMU HATDIS for IVHD type 11h and 40h */
465#define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0
466
467/* IOMMU Extended Feature Register (EFR) */
468#define IOMMU_EFR_XTSUP_SHIFT 2
469#define IOMMU_EFR_GASUP_SHIFT 7
470#define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
471
472#define MAX_DOMAIN_ID 65536
473
474/* Timeout stuff */
475#define LOOP_TIMEOUT 100000
476#define MMIO_STATUS_TIMEOUT 2000000
477
478extern bool amd_iommu_dump;
479#define DUMP_printk(format, arg...) \
480 do { \
481 if (amd_iommu_dump) \
482 pr_info(format, ## arg); \
483 } while(0);
484
485/* global flag if IOMMUs cache non-present entries */
486extern bool amd_iommu_np_cache;
487/* Only true if all IOMMUs support device IOTLBs */
488extern bool amd_iommu_iotlb_sup;
489
490struct irq_remap_table {
491 raw_spinlock_t lock;
492 unsigned min_index;
493 u32 *table;
494};
495
496/* Interrupt remapping feature used? */
497extern bool amd_iommu_irq_remap;
498
499extern const struct iommu_ops amd_iommu_ops;
500
501/* IVRS indicates that pre-boot remapping was enabled */
502extern bool amdr_ivrs_remap_support;
503
504#define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff)
505#define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff)
506#define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \
507 ((devid) & 0xffff))
508
509/* Make iterating over all pci segment easier */
510#define for_each_pci_segment(pci_seg) \
511 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list)
512#define for_each_pci_segment_safe(pci_seg, next) \
513 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list)
514/*
515 * Make iterating over all IOMMUs easier
516 */
517#define for_each_iommu(iommu) \
518 list_for_each_entry((iommu), &amd_iommu_list, list)
519#define for_each_iommu_safe(iommu, next) \
520 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
521/* Making iterating over protection_domain->dev_data_list easier */
522#define for_each_pdom_dev_data(pdom_dev_data, pdom) \
523 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
524#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
525 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
526
527#define for_each_ivhd_dte_flags(entry) \
528 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list)
529
530struct amd_iommu;
531struct iommu_domain;
532struct irq_domain;
533struct amd_irte_ops;
534
535#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
536
537#define io_pgtable_to_data(x) \
538 container_of((x), struct amd_io_pgtable, pgtbl)
539
540#define io_pgtable_ops_to_data(x) \
541 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
542
543#define io_pgtable_ops_to_domain(x) \
544 container_of(io_pgtable_ops_to_data(x), \
545 struct protection_domain, iop)
546
547#define io_pgtable_cfg_to_data(x) \
548 container_of((x), struct amd_io_pgtable, pgtbl.cfg)
549
550struct gcr3_tbl_info {
551 u64 *gcr3_tbl; /* Guest CR3 table */
552 int glx; /* Number of levels for GCR3 table */
553 u32 pasid_cnt; /* Track attached PASIDs */
554 u16 domid; /* Per device domain ID */
555};
556
557struct amd_io_pgtable {
558 seqcount_t seqcount; /* Protects root/mode update */
559 struct io_pgtable pgtbl;
560 int mode;
561 u64 *root;
562 u64 *pgd; /* v2 pgtable pgd pointer */
563};
564
565enum protection_domain_mode {
566 PD_MODE_NONE,
567 PD_MODE_V1,
568 PD_MODE_V2,
569};
570
571/* Track dev_data/PASID list for the protection domain */
572struct pdom_dev_data {
573 /* Points to attached device data */
574 struct iommu_dev_data *dev_data;
575 /* PASID attached to the protection domain */
576 ioasid_t pasid;
577 /* For protection_domain->dev_data_list */
578 struct list_head list;
579};
580
581/* Keeps track of the IOMMUs attached to protection domain */
582struct pdom_iommu_info {
583 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
584 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
585};
586
587/*
588 * This structure contains generic data for IOMMU protection domains
589 * independent of their use.
590 */
591struct protection_domain {
592 struct list_head dev_list; /* List of all devices in this domain */
593 struct iommu_domain domain; /* generic domain handle used by
594 iommu core code */
595 struct amd_io_pgtable iop;
596 spinlock_t lock; /* mostly used to lock the page table*/
597 u16 id; /* the domain id written to the device table */
598 enum protection_domain_mode pd_mode; /* Track page table type */
599 bool dirty_tracking; /* dirty tracking is enabled in the domain */
600 struct xarray iommu_array; /* per-IOMMU reference count */
601
602 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
603 struct list_head dev_data_list; /* List of pdom_dev_data */
604};
605
606/*
607 * This structure contains information about one PCI segment in the system.
608 */
609struct amd_iommu_pci_seg {
610 /* List with all PCI segments in the system */
611 struct list_head list;
612
613 /* List of all available dev_data structures */
614 struct llist_head dev_data_list;
615
616 /* PCI segment number */
617 u16 id;
618
619 /* Largest PCI device id we expect translation requests for */
620 u16 last_bdf;
621
622 /* Size of the device table */
623 u32 dev_table_size;
624
625 /*
626 * device table virtual address
627 *
628 * Pointer to the per PCI segment device table.
629 * It is indexed by the PCI device id or the HT unit id and contains
630 * information about the domain the device belongs to as well as the
631 * page table root pointer.
632 */
633 struct dev_table_entry *dev_table;
634
635 /*
636 * The rlookup iommu table is used to find the IOMMU which is
637 * responsible for a specific device. It is indexed by the PCI
638 * device id.
639 */
640 struct amd_iommu **rlookup_table;
641
642 /*
643 * This table is used to find the irq remapping table for a given
644 * device id quickly.
645 */
646 struct irq_remap_table **irq_lookup_table;
647
648 /*
649 * Pointer to a device table which the content of old device table
650 * will be copied to. It's only be used in kdump kernel.
651 */
652 struct dev_table_entry *old_dev_tbl_cpy;
653
654 /*
655 * The alias table is a driver specific data structure which contains the
656 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
657 * More than one device can share the same requestor id.
658 */
659 u16 *alias_table;
660
661 /*
662 * A list of required unity mappings we find in ACPI. It is not locked
663 * because as runtime it is only read. It is created at ACPI table
664 * parsing time.
665 */
666 struct list_head unity_map;
667};
668
669/*
670 * Structure where we save information about one hardware AMD IOMMU in the
671 * system.
672 */
673struct amd_iommu {
674 struct list_head list;
675
676 /* Index within the IOMMU array */
677 int index;
678
679 /* locks the accesses to the hardware */
680 raw_spinlock_t lock;
681
682 /* Pointer to PCI device of this IOMMU */
683 struct pci_dev *dev;
684
685 /* Cache pdev to root device for resume quirks */
686 struct pci_dev *root_pdev;
687
688 /* physical address of MMIO space */
689 u64 mmio_phys;
690
691 /* physical end address of MMIO space */
692 u64 mmio_phys_end;
693
694 /* virtual address of MMIO space */
695 u8 __iomem *mmio_base;
696
697 /* capabilities of that IOMMU read from ACPI */
698 u32 cap;
699
700 /* flags read from acpi table */
701 u8 acpi_flags;
702
703 /* Extended features */
704 u64 features;
705
706 /* Extended features 2 */
707 u64 features2;
708
709 /* PCI device id of the IOMMU device */
710 u16 devid;
711
712 /*
713 * Capability pointer. There could be more than one IOMMU per PCI
714 * device function if there are more than one AMD IOMMU capability
715 * pointers.
716 */
717 u16 cap_ptr;
718
719 /* pci domain of this IOMMU */
720 struct amd_iommu_pci_seg *pci_seg;
721
722 /* start of exclusion range of that IOMMU */
723 u64 exclusion_start;
724 /* length of exclusion range of that IOMMU */
725 u64 exclusion_length;
726
727 /* command buffer virtual address */
728 u8 *cmd_buf;
729 u32 cmd_buf_head;
730 u32 cmd_buf_tail;
731
732 /* event buffer virtual address */
733 u8 *evt_buf;
734
735 /* Name for event log interrupt */
736 unsigned char evt_irq_name[16];
737
738 /* Base of the PPR log, if present */
739 u8 *ppr_log;
740
741 /* Name for PPR log interrupt */
742 unsigned char ppr_irq_name[16];
743
744 /* Base of the GA log, if present */
745 u8 *ga_log;
746
747 /* Name for GA log interrupt */
748 unsigned char ga_irq_name[16];
749
750 /* Tail of the GA log, if present */
751 u8 *ga_log_tail;
752
753 /* true if interrupts for this IOMMU are already enabled */
754 bool int_enabled;
755
756 /* if one, we need to send a completion wait command */
757 bool need_sync;
758
759 /* true if disable irte caching */
760 bool irtcachedis_enabled;
761
762 /* Handle for IOMMU core code */
763 struct iommu_device iommu;
764
765 /*
766 * We can't rely on the BIOS to restore all values on reinit, so we
767 * need to stash them
768 */
769
770 /* The iommu BAR */
771 u32 stored_addr_lo;
772 u32 stored_addr_hi;
773
774 /*
775 * Each iommu has 6 l1s, each of which is documented as having 0x12
776 * registers
777 */
778 u32 stored_l1[6][0x12];
779
780 /* The l2 indirect registers */
781 u32 stored_l2[0x83];
782
783 /* The maximum PC banks and counters/bank (PCSup=1) */
784 u8 max_banks;
785 u8 max_counters;
786#ifdef CONFIG_IRQ_REMAP
787 struct irq_domain *ir_domain;
788
789 struct amd_irte_ops *irte_ops;
790#endif
791
792 u32 flags;
793 volatile u64 *cmd_sem;
794 atomic64_t cmd_sem_val;
795 /*
796 * Track physical address to directly use it in build_completion_wait()
797 * and avoid adding any special checks and handling for kdump.
798 */
799 u64 cmd_sem_paddr;
800
801#ifdef CONFIG_AMD_IOMMU_DEBUGFS
802 /* DebugFS Info */
803 struct dentry *debugfs;
804 int dbg_mmio_offset;
805 int dbg_cap_offset;
806#endif
807
808 /* IOPF support */
809 struct iopf_queue *iopf_queue;
810 unsigned char iopfq_name[32];
811};
812
813static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
814{
815 struct iommu_device *iommu = dev_to_iommu_device(dev);
816
817 return container_of(iommu, struct amd_iommu, iommu);
818}
819
820#define ACPIHID_UID_LEN 256
821#define ACPIHID_HID_LEN 9
822
823struct acpihid_map_entry {
824 struct list_head list;
825 u8 uid[ACPIHID_UID_LEN];
826 u8 hid[ACPIHID_HID_LEN];
827 u32 devid;
828 u32 root_devid;
829 bool cmd_line;
830 struct iommu_group *group;
831};
832
833struct devid_map {
834 struct list_head list;
835 u8 id;
836 u32 devid;
837 bool cmd_line;
838};
839
840#define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
841#define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
842#define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
843/* Device may request execution on memory pages */
844#define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
845/* Device may request super-user privileges */
846#define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
847
848/*
849 * This struct contains device specific data for the IOMMU
850 */
851struct iommu_dev_data {
852 /*Protect against attach/detach races */
853 struct mutex mutex;
854 spinlock_t dte_lock; /* DTE lock for 256-bit access */
855
856 struct list_head list; /* For domain->dev_list */
857 struct llist_node dev_data_list; /* For global dev_data_list */
858 struct protection_domain *domain; /* Domain the device is bound to */
859 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */
860 struct device *dev;
861 u16 devid; /* PCI Device ID */
862
863 unsigned int max_irqs; /* Maximum IRQs supported by device */
864 u32 max_pasids; /* Max supported PASIDs */
865 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
866 int ats_qdep;
867 u8 ats_enabled :1; /* ATS state */
868 u8 pri_enabled :1; /* PRI state */
869 u8 pasid_enabled:1; /* PASID state */
870 u8 pri_tlp :1; /* PASID TLB required for
871 PPR completions */
872 u8 ppr :1; /* Enable device PPR support */
873 bool use_vapic; /* Enable device to use vapic mode */
874 bool defer_attach;
875
876 struct ratelimit_state rs; /* Ratelimit IOPF messages */
877};
878
879/* Map HPET and IOAPIC ids to the devid used by the IOMMU */
880extern struct list_head ioapic_map;
881extern struct list_head hpet_map;
882extern struct list_head acpihid_map;
883
884/*
885 * List with all PCI segments in the system. This list is not locked because
886 * it is only written at driver initialization time
887 */
888extern struct list_head amd_iommu_pci_seg_list;
889
890/*
891 * List with all IOMMUs in the system. This list is not locked because it is
892 * only written and read at driver initialization or suspend time
893 */
894extern struct list_head amd_iommu_list;
895
896/*
897 * Structure defining one entry in the device table
898 */
899struct dev_table_entry {
900 union {
901 u64 data[4];
902 u128 data128[2];
903 };
904};
905
906/*
907 * Structure defining one entry in the command buffer
908 */
909struct iommu_cmd {
910 u32 data[4];
911};
912
913/*
914 * Structure to sture persistent DTE flags from IVHD
915 */
916struct ivhd_dte_flags {
917 struct list_head list;
918 u16 segid;
919 u16 devid_first;
920 u16 devid_last;
921 struct dev_table_entry dte;
922};
923
924/*
925 * One entry for unity mappings parsed out of the ACPI table.
926 */
927struct unity_map_entry {
928 struct list_head list;
929
930 /* starting device id this entry is used for (including) */
931 u16 devid_start;
932 /* end device id this entry is used for (including) */
933 u16 devid_end;
934
935 /* start address to unity map (including) */
936 u64 address_start;
937 /* end address to unity map (including) */
938 u64 address_end;
939
940 /* required protection */
941 int prot;
942};
943
944/*
945 * Data structures for device handling
946 */
947
948extern bool amd_iommu_force_isolation;
949
950/* Max levels of glxval supported */
951extern int amd_iommu_max_glx_val;
952
953/* IDA to track protection domain IDs */
954extern struct ida pdom_ids;
955
956/* Global EFR and EFR2 registers */
957extern u64 amd_iommu_efr;
958extern u64 amd_iommu_efr2;
959
960static inline int get_ioapic_devid(int id)
961{
962 struct devid_map *entry;
963
964 list_for_each_entry(entry, &ioapic_map, list) {
965 if (entry->id == id)
966 return entry->devid;
967 }
968
969 return -EINVAL;
970}
971
972static inline int get_hpet_devid(int id)
973{
974 struct devid_map *entry;
975
976 list_for_each_entry(entry, &hpet_map, list) {
977 if (entry->id == id)
978 return entry->devid;
979 }
980
981 return -EINVAL;
982}
983
984enum amd_iommu_intr_mode_type {
985 AMD_IOMMU_GUEST_IR_LEGACY,
986
987 /* This mode is not visible to users. It is used when
988 * we cannot fully enable vAPIC and fallback to only support
989 * legacy interrupt remapping via 128-bit IRTE.
990 */
991 AMD_IOMMU_GUEST_IR_LEGACY_GA,
992 AMD_IOMMU_GUEST_IR_VAPIC,
993};
994
995#define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \
996 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
997
998#define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)
999
1000union irte {
1001 u32 val;
1002 struct {
1003 u32 valid : 1,
1004 no_fault : 1,
1005 int_type : 3,
1006 rq_eoi : 1,
1007 dm : 1,
1008 rsvd_1 : 1,
1009 destination : 8,
1010 vector : 8,
1011 rsvd_2 : 8;
1012 } fields;
1013};
1014
1015#define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
1016#define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
1017
1018union irte_ga_lo {
1019 u64 val;
1020
1021 /* For int remapping */
1022 struct {
1023 u64 valid : 1,
1024 no_fault : 1,
1025 /* ------ */
1026 int_type : 3,
1027 rq_eoi : 1,
1028 dm : 1,
1029 /* ------ */
1030 guest_mode : 1,
1031 destination : 24,
1032 ga_tag : 32;
1033 } fields_remap;
1034
1035 /* For guest vAPIC */
1036 struct {
1037 u64 valid : 1,
1038 no_fault : 1,
1039 /* ------ */
1040 ga_log_intr : 1,
1041 rsvd1 : 3,
1042 is_run : 1,
1043 /* ------ */
1044 guest_mode : 1,
1045 destination : 24,
1046 ga_tag : 32;
1047 } fields_vapic;
1048};
1049
1050union irte_ga_hi {
1051 u64 val;
1052 struct {
1053 u64 vector : 8,
1054 rsvd_1 : 4,
1055 ga_root_ptr : 40,
1056 rsvd_2 : 4,
1057 destination : 8;
1058 } fields;
1059};
1060
1061struct irte_ga {
1062 union {
1063 struct {
1064 union irte_ga_lo lo;
1065 union irte_ga_hi hi;
1066 };
1067 u128 irte;
1068 };
1069};
1070
1071struct irq_2_irte {
1072 u16 devid; /* Device ID for IRTE table */
1073 u16 index; /* Index into IRTE table*/
1074};
1075
1076struct amd_ir_data {
1077 struct amd_iommu *iommu;
1078 struct irq_2_irte irq_2_irte;
1079 struct msi_msg msi_entry;
1080 void *entry; /* Pointer to union irte or struct irte_ga */
1081
1082 /**
1083 * Store information for activate/de-activate
1084 * Guest virtual APIC mode during runtime.
1085 */
1086 struct irq_cfg *cfg;
1087 int ga_vector;
1088 u64 ga_root_ptr;
1089 u32 ga_tag;
1090};
1091
1092struct amd_irte_ops {
1093 void (*prepare)(void *, u32, bool, u8, u32, int);
1094 void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
1095 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
1096 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
1097 void *(*get)(struct irq_remap_table *, int);
1098 void (*set_allocated)(struct irq_remap_table *, int);
1099 bool (*is_allocated)(struct irq_remap_table *, int);
1100 void (*clear_allocated)(struct irq_remap_table *, int);
1101};
1102
1103#ifdef CONFIG_IRQ_REMAP
1104extern struct amd_irte_ops irte_32_ops;
1105extern struct amd_irte_ops irte_128_ops;
1106#endif
1107
1108#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
1109