probe.c source code [Linux/drivers/pci/probe.c]

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* PCI detection and setup code
4	*/
5
6	#include <linux/array_size.h>
7	#include <linux/kernel.h>
8	#include <linux/delay.h>
9	#include <linux/init.h>
10	#include <linux/pci.h>
11	#include <linux/msi.h>
12	#include <linux/of_pci.h>
13	#include <linux/of_platform.h>
14	#include <linux/platform_device.h>
15	#include <linux/pci_hotplug.h>
16	#include <linux/slab.h>
17	#include <linux/module.h>
18	#include <linux/cpumask.h>
19	#include <linux/aer.h>
20	#include <linux/acpi.h>
21	#include <linux/hypervisor.h>
22	#include <linux/irqdomain.h>
23	#include <linux/pm_runtime.h>
24	#include <linux/bitfield.h>
25	#include "pci.h"
26
27	#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
28	#define CARDBUS_RESERVE_BUSNR 3
29
30	static struct resource busn_resource = {
31	.name = "PCI busn",
32	.start = `0`,
33	.end = `255`,
34	.flags = IORESOURCE_BUS,
35	};
36
37	/ Ugh. Need to stop exporting this to modules. /
38	LIST_HEAD(pci_root_buses);
39	EXPORT_SYMBOL(pci_root_buses);
40
41	static LIST_HEAD(pci_domain_busn_res_list);
42
43	struct pci_domain_busn_res {
44	struct list_head list;
45	struct resource res;
46	int domain_nr;
47	};
48
49	static struct resource get_pci_domain_busn_res(int* domain_nr)
50	{
51	struct pci_domain_busn_res *r;
52
53	list_for_each_entry(r, &pci_domain_busn_res_list, list)
54	if (r->domain_nr == domain_nr)
55	return &r->res;
56
57	r = kzalloc(sizeof(*r), GFP_KERNEL);
58	if (!r)
59	return NULL;
60
61	r->domain_nr = domain_nr;
62	r->res.start = `0`;
63	r->res.end = `0xff`;
64	r->res.flags = IORESOURCE_BUS \| IORESOURCE_PCI_FIXED;
65
66	list_add_tail(new: &r->list, head: &pci_domain_busn_res_list);
67
68	return &r->res;
69	}
70
71	/*
72	* Some device drivers need know if PCI is initiated.
73	* Basically, we think PCI is not initiated when there
74	* is no device to be found on the pci_bus_type.
75	*/
76	int no_pci_devices(void)
77	{
78	struct device *dev;
79	int no_devices;
80
81	dev = bus_find_next_device(bus: &pci_bus_type, NULL);
82	no_devices = (dev == NULL);
83	put_device(dev);
84	return no_devices;
85	}
86	EXPORT_SYMBOL(no_pci_devices);
87
88	/*
89	* PCI Bus Class
90	*/
91	static void release_pcibus_dev(struct device *dev)
92	{
93	struct pci_bus *pci_bus = to_pci_bus(dev);
94
95	put_device(dev: pci_bus->bridge);
96	pci_bus_remove_resources(bus: pci_bus);
97	pci_release_bus_of_node(bus: pci_bus);
98	kfree(objp: pci_bus);
99	}
100
101	static const struct class pcibus_class = {
102	.name = "pci_bus",
103	.dev_release = &release_pcibus_dev,
104	.dev_groups = pcibus_groups,
105	};
106
107	static int __init pcibus_class_init(void)
108	{
109	return class_register(class: &pcibus_class);
110	}
111	postcore_initcall(pcibus_class_init);
112
113	static u64 pci_size(u64 base, u64 maxbase, u64 mask)
114	{
115	u64 size = mask & maxbase; / Find the significant bits /
116	if (!size)
117	return `0`;
118
119	/*
120	* Get the lowest of them to find the decode size, and from that
121	* the extent.
122	*/
123	size = size & ~(size-`1`);
124
125	/*
126	* base == maxbase can be valid only if the BAR has already been
127	* programmed with all 1s.
128	*/
129	if (base == maxbase && ((base \| (size - `1`)) & mask) != mask)
130	return `0`;
131
132	return size;
133	}
134
135	static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
136	{
137	u32 mem_type;
138	unsigned long flags;
139
140	if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
141	flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
142	flags \|= IORESOURCE_IO;
143	return flags;
144	}
145
146	flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
147	flags \|= IORESOURCE_MEM;
148	if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
149	flags \|= IORESOURCE_PREFETCH;
150
151	mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
152	switch (mem_type) {
153	case PCI_BASE_ADDRESS_MEM_TYPE_32:
154	break;
155	case PCI_BASE_ADDRESS_MEM_TYPE_1M:
156	/ 1M mem BAR treated as 32-bit BAR /
157	break;
158	case PCI_BASE_ADDRESS_MEM_TYPE_64:
159	flags \|= IORESOURCE_MEM_64;
160	break;
161	default:
162	/ mem unknown type treated as 32-bit BAR /
163	break;
164	}
165	return flags;
166	}
167
168	#define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY \| PCI_COMMAND_IO)
169
170	/**
171	* __pci_size_bars - Read the raw BAR mask for a range of PCI BARs
172	* @dev: the PCI device
173	* @count: number of BARs to size
174	* @pos: starting config space position
175	* @sizes: array to store mask values
176	* @rom: indicate whether to use ROM mask, which avoids enabling ROM BARs
177	*
178	* Provided @sizes array must be sufficiently sized to store results for
179	* @count u32 BARs. Caller is responsible for disabling decode to specified
180	* BAR range around calling this function. This function is intended to avoid
181	* disabling decode around sizing each BAR individually, which can result in
182	* non-trivial overhead in virtualized environments with very large PCI BARs.
183	*/
184	static void __pci_size_bars(struct pci_dev dev, int* count,
185	unsigned int pos, u32 *sizes, bool rom)
186	{
187	u32 orig, mask = rom ? PCI_ROM_ADDRESS_MASK : ~`0`;
188	int i;
189
190	for (i = `0`; i < count; i++, pos += `4`, sizes++) {
191	pci_read_config_dword(dev, where: pos, val: &orig);
192	pci_write_config_dword(dev, where: pos, val: mask);
193	pci_read_config_dword(dev, where: pos, val: sizes);
194	pci_write_config_dword(dev, where: pos, val: orig);
195	}
196	}
197
198	void __pci_size_stdbars(struct pci_dev dev, int* count,
199	unsigned int pos, u32 *sizes)
200	{
201	__pci_size_bars(dev, count, pos, sizes, rom: false);
202	}
203
204	static void __pci_size_rom(struct pci_dev dev, unsigned* int pos, u32 *sizes)
205	{
206	__pci_size_bars(dev, count: `1`, pos, sizes, rom: true);
207	}
208
209	/**
210	* __pci_read_base - Read a PCI BAR
211	* @dev: the PCI device
212	* @type: type of the BAR
213	* @res: resource buffer to be filled in
214	* @pos: BAR position in the config space
215	* @sizes: array of one or more pre-read BAR masks
216	*
217	* Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
218	*/
219	int __pci_read_base(struct pci_dev dev, enum* pci_bar_type type,
220	struct resource res, unsigned* int pos, u32 *sizes)
221	{
222	u32 l = `0`, sz;
223	u64 l64, sz64, mask64;
224	struct pci_bus_region region, inverted_region;
225	const char *res_name = pci_resource_name(dev, i: res - dev->resource);
226
227	res->name = pci_name(pdev: dev);
228
229	pci_read_config_dword(dev, where: pos, val: &l);
230	sz = sizes[`0`];
231
232	/*
233	* All bits set in sz means the device isn't working properly.
234	* If the BAR isn't implemented, all bits must be 0. If it's a
235	* memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
236	* 1 must be clear.
237	*/
238	if (PCI_POSSIBLE_ERROR(sz))
239	sz = `0`;
240
241	/*
242	* I don't know how l can have all bits set. Copied from old code.
243	* Maybe it fixes a bug on some ancient platform.
244	*/
245	if (PCI_POSSIBLE_ERROR(l))
246	l = `0`;
247
248	if (type == pci_bar_unknown) {
249	res->flags = decode_bar(dev, bar: l);
250	res->flags \|= IORESOURCE_SIZEALIGN;
251	if (res->flags & IORESOURCE_IO) {
252	l64 = l & PCI_BASE_ADDRESS_IO_MASK;
253	sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
254	mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
255	} else {
256	l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
257	sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
258	mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
259	}
260	} else {
261	if (l & PCI_ROM_ADDRESS_ENABLE)
262	res->flags \|= IORESOURCE_ROM_ENABLE;
263	l64 = l & PCI_ROM_ADDRESS_MASK;
264	sz64 = sz & PCI_ROM_ADDRESS_MASK;
265	mask64 = PCI_ROM_ADDRESS_MASK;
266	}
267
268	if (res->flags & IORESOURCE_MEM_64) {
269	pci_read_config_dword(dev, where: pos + `4`, val: &l);
270	sz = sizes[`1`];
271
272	l64 \|= ((u64)l << `32`);
273	sz64 \|= ((u64)sz << `32`);
274	mask64 \|= ((u64)~`0` << `32`);
275	}
276
277	if (!sz64)
278	goto fail;
279
280	sz64 = pci_size(base: l64, maxbase: sz64, mask: mask64);
281	if (!sz64) {
282	pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name);
283	goto fail;
284	}
285
286	if (res->flags & IORESOURCE_MEM_64) {
287	if ((sizeof(pci_bus_addr_t) < `8` \|\| sizeof(resource_size_t) < `8`)
288	&& sz64 > `0x100000000ULL`) {
289	res->flags \|= IORESOURCE_UNSET \| IORESOURCE_DISABLED;
290	res->start = `0`;
291	res->end = `0`;
292	pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n",
293	res_name, (unsigned long long)sz64);
294	goto out;
295	}
296
297	if ((sizeof(pci_bus_addr_t) < `8`) && l) {
298	/ Above 32-bit boundary; try to reallocate /
299	res->flags \|= IORESOURCE_UNSET;
300	res->start = `0`;
301	res->end = sz64 - `1`;
302	pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n",
303	res_name, (unsigned long long)l64);
304	goto out;
305	}
306	}
307
308	region.start = l64;
309	region.end = l64 + sz64 - `1`;
310
311	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
312	pcibios_resource_to_bus(bus: dev->bus, region: &inverted_region, res);
313
314	/*
315	* If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
316	* the corresponding resource address (the physical address used by
317	* the CPU. Converting that resource address back to a bus address
318	* should yield the original BAR value:
319	*
320	* resource_to_bus(bus_to_resource(A)) == A
321	*
322	* If it doesn't, CPU accesses to "bus_to_resource(A)" will not
323	* be claimed by the device.
324	*/
325	if (inverted_region.start != region.start) {
326	res->flags \|= IORESOURCE_UNSET;
327	res->start = `0`;
328	res->end = region.end - region.start;
329	pci_info(dev, "%s: initial BAR value %#010llx invalid\n",
330	res_name, (unsigned long long)region.start);
331	}
332
333	goto out;
334
335
336	fail:
337	res->flags = `0`;
338	out:
339	if (res->flags)
340	pci_info(dev, "%s %pR\n", res_name, res);
341
342	return (res->flags & IORESOURCE_MEM_64) ? `1` : `0`;
343	}
344
345	static __always_inline void pci_read_bases(struct pci_dev *dev,
346	unsigned int howmany, int rom)
347	{
348	u32 rombar, stdbars[PCI_STD_NUM_BARS];
349	unsigned int pos, reg;
350	u16 orig_cmd;
351
352	BUILD_BUG_ON(statically_true(howmany > PCI_STD_NUM_BARS));
353
354	if (dev->non_compliant_bars)
355	return;
356
357	/ Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero /
358	if (dev->is_virtfn)
359	return;
360
361	/ No printks while decoding is disabled! /
362	if (!dev->mmio_always_on) {
363	pci_read_config_word(dev, PCI_COMMAND, val: &orig_cmd);
364	if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
365	pci_write_config_word(dev, PCI_COMMAND,
366	val: orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
367	}
368	}
369
370	__pci_size_stdbars(dev, count: howmany, PCI_BASE_ADDRESS_0, sizes: stdbars);
371	if (rom)
372	__pci_size_rom(dev, pos: rom, sizes: &rombar);
373
374	if (!dev->mmio_always_on &&
375	(orig_cmd & PCI_COMMAND_DECODE_ENABLE))
376	pci_write_config_word(dev, PCI_COMMAND, val: orig_cmd);
377
378	for (pos = `0`; pos < howmany; pos++) {
379	struct resource *res = &dev->resource[pos];
380	reg = PCI_BASE_ADDRESS_0 + (pos << `2`);
381	pos += __pci_read_base(dev, type: pci_bar_unknown,
382	res, pos: reg, sizes: &stdbars[pos]);
383	}
384
385	if (rom) {
386	struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
387	dev->rom_base_reg = rom;
388	res->flags = IORESOURCE_MEM \| IORESOURCE_PREFETCH \|
389	IORESOURCE_READONLY \| IORESOURCE_SIZEALIGN;
390	__pci_read_base(dev, type: pci_bar_mem32, res, pos: rom, sizes: &rombar);
391	}
392	}
393
394	static void pci_read_bridge_io(struct pci_dev dev, struct* resource *res,
395	bool log)
396	{
397	u8 io_base_lo, io_limit_lo;
398	unsigned long io_mask, io_granularity, base, limit;
399	struct pci_bus_region region;
400
401	io_mask = PCI_IO_RANGE_MASK;
402	io_granularity = `0x1000`;
403	if (dev->io_window_1k) {
404	/ Support 1K I/O space granularity /
405	io_mask = PCI_IO_1K_RANGE_MASK;
406	io_granularity = `0x400`;
407	}
408
409	pci_read_config_byte(dev, PCI_IO_BASE, val: &io_base_lo);
410	pci_read_config_byte(dev, PCI_IO_LIMIT, val: &io_limit_lo);
411	base = (io_base_lo & io_mask) << `8`;
412	limit = (io_limit_lo & io_mask) << `8`;
413
414	if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
415	u16 io_base_hi, io_limit_hi;
416
417	pci_read_config_word(dev, PCI_IO_BASE_UPPER16, val: &io_base_hi);
418	pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, val: &io_limit_hi);
419	base \|= ((unsigned long) io_base_hi << `16`);
420	limit \|= ((unsigned long) io_limit_hi << `16`);
421	}
422
423	res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) \| IORESOURCE_IO;
424
425	if (base <= limit) {
426	region.start = base;
427	region.end = limit + io_granularity - `1`;
428	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
429	if (log)
430	pci_info(dev, " bridge window %pR\n", res);
431	} else {
432	resource_set_range(res, start: `0`, size: `0`);
433	res->flags \|= IORESOURCE_UNSET \| IORESOURCE_DISABLED;
434	}
435	}
436
437	static void pci_read_bridge_mmio(struct pci_dev dev, struct* resource *res,
438	bool log)
439	{
440	u16 mem_base_lo, mem_limit_lo;
441	unsigned long base, limit;
442	struct pci_bus_region region;
443
444	pci_read_config_word(dev, PCI_MEMORY_BASE, val: &mem_base_lo);
445	pci_read_config_word(dev, PCI_MEMORY_LIMIT, val: &mem_limit_lo);
446	base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << `16`;
447	limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << `16`;
448
449	res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) \| IORESOURCE_MEM;
450
451	if (base <= limit) {
452	region.start = base;
453	region.end = limit + `0xfffff`;
454	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
455	if (log)
456	pci_info(dev, " bridge window %pR\n", res);
457	} else {
458	resource_set_range(res, start: `0`, size: `0`);
459	res->flags \|= IORESOURCE_UNSET \| IORESOURCE_DISABLED;
460	}
461	}
462
463	static void pci_read_bridge_mmio_pref(struct pci_dev dev, struct* resource *res,
464	bool log)
465	{
466	u16 mem_base_lo, mem_limit_lo;
467	u64 base64, limit64;
468	pci_bus_addr_t base, limit;
469	struct pci_bus_region region;
470
471	pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, val: &mem_base_lo);
472	pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, val: &mem_limit_lo);
473	base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << `16`;
474	limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << `16`;
475
476	if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
477	u32 mem_base_hi, mem_limit_hi;
478
479	pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, val: &mem_base_hi);
480	pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, val: &mem_limit_hi);
481
482	/*
483	* Some bridges set the base > limit by default, and some
484	* (broken) BIOSes do not initialize them. If we find
485	* this, just assume they are not being used.
486	*/
487	if (mem_base_hi <= mem_limit_hi) {
488	base64 \|= (u64) mem_base_hi << `32`;
489	limit64 \|= (u64) mem_limit_hi << `32`;
490	}
491	}
492
493	base = (pci_bus_addr_t) base64;
494	limit = (pci_bus_addr_t) limit64;
495
496	if (base != base64) {
497	pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
498	(unsigned long long) base64);
499	return;
500	}
501
502	res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) \| IORESOURCE_MEM \|
503	IORESOURCE_PREFETCH;
504	if (res->flags & PCI_PREF_RANGE_TYPE_64)
505	res->flags \|= IORESOURCE_MEM_64;
506
507	if (base <= limit) {
508	region.start = base;
509	region.end = limit + `0xfffff`;
510	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
511	if (log)
512	pci_info(dev, " bridge window %pR\n", res);
513	} else {
514	resource_set_range(res, start: `0`, size: `0`);
515	res->flags \|= IORESOURCE_UNSET \| IORESOURCE_DISABLED;
516	}
517	}
518
519	static void pci_read_bridge_windows(struct pci_dev *bridge)
520	{
521	u32 buses;
522	u16 io;
523	u32 pmem, tmp;
524	struct resource res;
525
526	pci_read_config_dword(dev: bridge, PCI_PRIMARY_BUS, val: &buses);
527	res.flags = IORESOURCE_BUS;
528	res.start = (buses >> `8`) & `0xff`;
529	res.end = (buses >> `16`) & `0xff`;
530	pci_info(bridge, "PCI bridge to %pR%s\n", &res,
531	bridge->transparent ? " (subtractive decode)" : "");
532
533	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
534	if (!io) {
535	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: `0xe0f0`);
536	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
537	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: `0x0`);
538	}
539	if (io) {
540	bridge->io_window = `1`;
541	pci_read_bridge_io(dev: bridge,
542	pci_resource_n(bridge, PCI_BRIDGE_IO_WINDOW),
543	log: true);
544	}
545
546	pci_read_bridge_mmio(dev: bridge,
547	pci_resource_n(bridge, PCI_BRIDGE_MEM_WINDOW),
548	log: true);
549
550	/*
551	* DECchip 21050 pass 2 errata: the bridge may miss an address
552	* disconnect boundary by one PCI data phase. Workaround: do not
553	* use prefetching on this device.
554	*/
555	if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == `0x0001`)
556	return;
557
558	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
559	if (!pmem) {
560	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE,
561	val: `0xffe0fff0`);
562	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
563	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: `0x0`);
564	}
565	if (!pmem)
566	return;
567
568	bridge->pref_window = `1`;
569
570	if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
571
572	/*
573	* Bridge claims to have a 64-bit prefetchable memory
574	* window; verify that the upper bits are actually
575	* writable.
576	*/
577	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &pmem);
578	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32,
579	val: `0xffffffff`);
580	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &tmp);
581	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: pmem);
582	if (tmp)
583	bridge->pref_64_window = `1`;
584	}
585
586	pci_read_bridge_mmio_pref(dev: bridge,
587	pci_resource_n(bridge,
588	PCI_BRIDGE_PREF_MEM_WINDOW),
589	log: true);
590	}
591
592	void pci_read_bridge_bases(struct pci_bus *child)
593	{
594	struct pci_dev *dev = child->self;
595	struct resource *res;
596	int i;
597
598	if (pci_is_root_bus(pbus: child)) / It's a host bus, nothing to read /
599	return;
600
601	pci_info(dev, "PCI bridge to %pR%s\n",
602	&child->busn_res,
603	dev->transparent ? " (subtractive decode)" : "");
604
605	pci_bus_remove_resources(bus: child);
606	for (i = `0`; i < PCI_BRIDGE_RESOURCE_NUM; i++)
607	child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
608
609	pci_read_bridge_io(dev: child->self,
610	res: child->resource[PCI_BUS_BRIDGE_IO_WINDOW], log: false);
611	pci_read_bridge_mmio(dev: child->self,
612	res: child->resource[PCI_BUS_BRIDGE_MEM_WINDOW], log: false);
613	pci_read_bridge_mmio_pref(dev: child->self,
614	res: child->resource[PCI_BUS_BRIDGE_PREF_MEM_WINDOW],
615	log: false);
616
617	if (!dev->transparent)
618	return;
619
620	pci_bus_for_each_resource(child->parent, res) {
621	if (!res \|\| !res->flags)
622	continue;
623
624	pci_bus_add_resource(bus: child, res);
625	pci_info(dev, " bridge window %pR (subtractive decode)\n", res);
626	}
627	}
628
629	static struct pci_bus pci_alloc_bus(struct* pci_bus *parent)
630	{
631	struct pci_bus *b;
632
633	b = kzalloc(sizeof(*b), GFP_KERNEL);
634	if (!b)
635	return NULL;
636
637	INIT_LIST_HEAD(list: &b->node);
638	INIT_LIST_HEAD(list: &b->children);
639	INIT_LIST_HEAD(list: &b->devices);
640	INIT_LIST_HEAD(list: &b->slots);
641	INIT_LIST_HEAD(list: &b->resources);
642	b->max_bus_speed = PCI_SPEED_UNKNOWN;
643	b->cur_bus_speed = PCI_SPEED_UNKNOWN;
644	#ifdef CONFIG_PCI_DOMAINS_GENERIC
645	if (parent)
646	b->domain_nr = parent->domain_nr;
647	#endif
648	return b;
649	}
650
651	static void pci_release_host_bridge_dev(struct device *dev)
652	{
653	struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
654
655	if (bridge->release_fn)
656	bridge->release_fn(bridge);
657
658	pci_free_resource_list(resources: &bridge->windows);
659	pci_free_resource_list(resources: &bridge->dma_ranges);
660	kfree(objp: bridge);
661	}
662
663	static void pci_init_host_bridge(struct pci_host_bridge *bridge)
664	{
665	INIT_LIST_HEAD(list: &bridge->windows);
666	INIT_LIST_HEAD(list: &bridge->dma_ranges);
667
668	/*
669	* We assume we can manage these PCIe features. Some systems may
670	* reserve these for use by the platform itself, e.g., an ACPI BIOS
671	* may implement its own AER handling and use _OSC to prevent the
672	* OS from interfering.
673	*/
674	bridge->native_aer = `1`;
675	bridge->native_pcie_hotplug = `1`;
676	bridge->native_shpc_hotplug = `1`;
677	bridge->native_pme = `1`;
678	bridge->native_ltr = `1`;
679	bridge->native_dpc = `1`;
680	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
681	bridge->native_cxl_error = `1`;
682
683	device_initialize(dev: &bridge->dev);
684	}
685
686	struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
687	{
688	struct pci_host_bridge *bridge;
689
690	bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
691	if (!bridge)
692	return NULL;
693
694	pci_init_host_bridge(bridge);
695	bridge->dev.release = pci_release_host_bridge_dev;
696
697	return bridge;
698	}
699	EXPORT_SYMBOL(pci_alloc_host_bridge);
700
701	static void devm_pci_alloc_host_bridge_release(void *data)
702	{
703	pci_free_host_bridge(bridge: data);
704	}
705
706	struct pci_host_bridge devm_pci_alloc_host_bridge(struct* device *dev,
707	size_t priv)
708	{
709	int ret;
710	struct pci_host_bridge *bridge;
711
712	bridge = pci_alloc_host_bridge(priv);
713	if (!bridge)
714	return NULL;
715
716	bridge->dev.parent = dev;
717
718	ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
719	bridge);
720	if (ret)
721	return NULL;
722
723	ret = devm_of_pci_bridge_init(dev, bridge);
724	if (ret)
725	return NULL;
726
727	return bridge;
728	}
729	EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
730
731	void pci_free_host_bridge(struct pci_host_bridge *bridge)
732	{
733	put_device(dev: &bridge->dev);
734	}
735	EXPORT_SYMBOL(pci_free_host_bridge);
736
737	/ Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) /
738	static const unsigned char pcix_bus_speed[] = {
739	PCI_SPEED_UNKNOWN, / 0 /
740	PCI_SPEED_66MHz_PCIX, / 1 /
741	PCI_SPEED_100MHz_PCIX, / 2 /
742	PCI_SPEED_133MHz_PCIX, / 3 /
743	PCI_SPEED_UNKNOWN, / 4 /
744	PCI_SPEED_66MHz_PCIX_ECC, / 5 /
745	PCI_SPEED_100MHz_PCIX_ECC, / 6 /
746	PCI_SPEED_133MHz_PCIX_ECC, / 7 /
747	PCI_SPEED_UNKNOWN, / 8 /
748	PCI_SPEED_66MHz_PCIX_266, / 9 /
749	PCI_SPEED_100MHz_PCIX_266, / A /
750	PCI_SPEED_133MHz_PCIX_266, / B /
751	PCI_SPEED_UNKNOWN, / C /
752	PCI_SPEED_66MHz_PCIX_533, / D /
753	PCI_SPEED_100MHz_PCIX_533, / E /
754	PCI_SPEED_133MHz_PCIX_533 / F /
755	};
756
757	/ Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS /
758	const unsigned char pcie_link_speed[] = {
759	PCI_SPEED_UNKNOWN, / 0 /
760	PCIE_SPEED_2_5GT, / 1 /
761	PCIE_SPEED_5_0GT, / 2 /
762	PCIE_SPEED_8_0GT, / 3 /
763	PCIE_SPEED_16_0GT, / 4 /
764	PCIE_SPEED_32_0GT, / 5 /
765	PCIE_SPEED_64_0GT, / 6 /
766	PCI_SPEED_UNKNOWN, / 7 /
767	PCI_SPEED_UNKNOWN, / 8 /
768	PCI_SPEED_UNKNOWN, / 9 /
769	PCI_SPEED_UNKNOWN, / A /
770	PCI_SPEED_UNKNOWN, / B /
771	PCI_SPEED_UNKNOWN, / C /
772	PCI_SPEED_UNKNOWN, / D /
773	PCI_SPEED_UNKNOWN, / E /
774	PCI_SPEED_UNKNOWN / F /
775	};
776	EXPORT_SYMBOL_GPL(pcie_link_speed);
777
778	const char pci_speed_string(enum* pci_bus_speed speed)
779	{
780	/ Indexed by the pci_bus_speed enum /
781	static const char *speed_strings[] = {
782	"33 MHz PCI", / 0x00 /
783	"66 MHz PCI", / 0x01 /
784	"66 MHz PCI-X", / 0x02 /
785	"100 MHz PCI-X", / 0x03 /
786	"133 MHz PCI-X", / 0x04 /
787	NULL, / 0x05 /
788	NULL, / 0x06 /
789	NULL, / 0x07 /
790	NULL, / 0x08 /
791	"66 MHz PCI-X 266", / 0x09 /
792	"100 MHz PCI-X 266", / 0x0a /
793	"133 MHz PCI-X 266", / 0x0b /
794	"Unknown AGP", / 0x0c /
795	"1x AGP", / 0x0d /
796	"2x AGP", / 0x0e /
797	"4x AGP", / 0x0f /
798	"8x AGP", / 0x10 /
799	"66 MHz PCI-X 533", / 0x11 /
800	"100 MHz PCI-X 533", / 0x12 /
801	"133 MHz PCI-X 533", / 0x13 /
802	"2.5 GT/s PCIe", / 0x14 /
803	"5.0 GT/s PCIe", / 0x15 /
804	"8.0 GT/s PCIe", / 0x16 /
805	"16.0 GT/s PCIe", / 0x17 /
806	"32.0 GT/s PCIe", / 0x18 /
807	"64.0 GT/s PCIe", / 0x19 /
808	};
809
810	if (speed < ARRAY_SIZE(speed_strings))
811	return speed_strings[speed];
812	return "Unknown";
813	}
814	EXPORT_SYMBOL_GPL(pci_speed_string);
815
816	void pcie_update_link_speed(struct pci_bus *bus)
817	{
818	struct pci_dev *bridge = bus->self;
819	u16 linksta, linksta2;
820
821	pcie_capability_read_word(dev: bridge, PCI_EXP_LNKSTA, val: &linksta);
822	pcie_capability_read_word(dev: bridge, PCI_EXP_LNKSTA2, val: &linksta2);
823	__pcie_update_link_speed(bus, linksta, linksta2);
824	}
825	EXPORT_SYMBOL_GPL(pcie_update_link_speed);
826
827	static unsigned char agp_speeds[] = {
828	AGP_UNKNOWN,
829	AGP_1X,
830	AGP_2X,
831	AGP_4X,
832	AGP_8X
833	};
834
835	static enum pci_bus_speed agp_speed(int agp3, int agpstat)
836	{
837	int index = `0`;
838
839	if (agpstat & `4`)
840	index = `3`;
841	else if (agpstat & `2`)
842	index = `2`;
843	else if (agpstat & `1`)
844	index = `1`;
845	else
846	goto out;
847
848	if (agp3) {
849	index += `2`;
850	if (index == `5`)
851	index = `0`;
852	}
853
854	out:
855	return agp_speeds[index];
856	}
857
858	static void pci_set_bus_speed(struct pci_bus *bus)
859	{
860	struct pci_dev *bridge = bus->self;
861	int pos;
862
863	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP);
864	if (!pos)
865	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP3);
866	if (pos) {
867	u32 agpstat, agpcmd;
868
869	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_STATUS, val: &agpstat);
870	bus->max_bus_speed = agp_speed(agp3: agpstat & `8`, agpstat: agpstat & `7`);
871
872	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_COMMAND, val: &agpcmd);
873	bus->cur_bus_speed = agp_speed(agp3: agpstat & `8`, agpstat: agpcmd & `7`);
874	}
875
876	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
877	if (pos) {
878	u16 status;
879	enum pci_bus_speed max;
880
881	pci_read_config_word(dev: bridge, where: pos + PCI_X_BRIDGE_SSTATUS,
882	val: &status);
883
884	if (status & PCI_X_SSTATUS_533MHZ) {
885	max = PCI_SPEED_133MHz_PCIX_533;
886	} else if (status & PCI_X_SSTATUS_266MHZ) {
887	max = PCI_SPEED_133MHz_PCIX_266;
888	} else if (status & PCI_X_SSTATUS_133MHZ) {
889	if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
890	max = PCI_SPEED_133MHz_PCIX_ECC;
891	else
892	max = PCI_SPEED_133MHz_PCIX;
893	} else {
894	max = PCI_SPEED_66MHz_PCIX;
895	}
896
897	bus->max_bus_speed = max;
898	bus->cur_bus_speed =
899	pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
900
901	return;
902	}
903
904	if (pci_is_pcie(dev: bridge)) {
905	u32 linkcap;
906
907	pcie_capability_read_dword(dev: bridge, PCI_EXP_LNKCAP, val: &linkcap);
908	bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
909
910	pcie_update_link_speed(bus);
911	}
912	}
913
914	static struct irq_domain pci_host_bridge_msi_domain(struct* pci_bus *bus)
915	{
916	struct irq_domain *d;
917
918	/ If the host bridge driver sets a MSI domain of the bridge, use it /
919	d = dev_get_msi_domain(dev: bus->bridge);
920
921	/*
922	* Any firmware interface that can resolve the msi_domain
923	* should be called from here.
924	*/
925	if (!d)
926	d = pci_host_bridge_of_msi_domain(bus);
927	if (!d)
928	d = pci_host_bridge_acpi_msi_domain(bus);
929
930	/*
931	* If no IRQ domain was found via the OF tree, try looking it up
932	* directly through the fwnode_handle.
933	*/
934	if (!d) {
935	struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
936
937	if (fwnode)
938	d = irq_find_matching_fwnode(fwnode,
939	bus_token: DOMAIN_BUS_PCI_MSI);
940	}
941
942	return d;
943	}
944
945	static void pci_set_bus_msi_domain(struct pci_bus *bus)
946	{
947	struct irq_domain *d;
948	struct pci_bus *b;
949
950	/*
951	* The bus can be a root bus, a subordinate bus, or a virtual bus
952	* created by an SR-IOV device. Walk up to the first bridge device
953	* found or derive the domain from the host bridge.
954	*/
955	for (b = bus, d = NULL; !d && !pci_is_root_bus(pbus: b); b = b->parent) {
956	if (b->self)
957	d = dev_get_msi_domain(dev: &b->self->dev);
958	}
959
960	if (!d)
961	d = pci_host_bridge_msi_domain(bus: b);
962
963	dev_set_msi_domain(dev: &bus->dev, d);
964	}
965
966	static bool pci_preserve_config(struct pci_host_bridge *host_bridge)
967	{
968	if (pci_acpi_preserve_config(bridge: host_bridge))
969	return true;
970
971	if (host_bridge->dev.parent && host_bridge->dev.parent->of_node)
972	return of_pci_preserve_config(node: host_bridge->dev.parent->of_node);
973
974	return false;
975	}
976
977	static int pci_register_host_bridge(struct pci_host_bridge *bridge)
978	{
979	struct device *parent = bridge->dev.parent;
980	struct resource_entry window, next, *n;
981	struct pci_bus bus, b;
982	resource_size_t offset, next_offset;
983	LIST_HEAD(resources);
984	struct resource res, next_res;
985	bool bus_registered = false;
986	char addr[`64`], *fmt;
987	const char *name;
988	int err;
989
990	bus = pci_alloc_bus(NULL);
991	if (!bus)
992	return -ENOMEM;
993
994	bridge->bus = bus;
995
996	bus->sysdata = bridge->sysdata;
997	bus->ops = bridge->ops;
998	bus->number = bus->busn_res.start = bridge->busnr;
999	#ifdef CONFIG_PCI_DOMAINS_GENERIC
1000	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
1001	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
1002	else
1003	bus->domain_nr = bridge->domain_nr;
1004	if (bus->domain_nr < `0`) {
1005	err = bus->domain_nr;
1006	goto free;
1007	}
1008	#endif
1009
1010	b = pci_find_bus(domain: pci_domain_nr(bus), busnr: bridge->busnr);
1011	if (b) {
1012	/ Ignore it if we already got here via a different bridge /
1013	dev_dbg(&b->dev, "bus already known\n");
1014	err = -EEXIST;
1015	goto free;
1016	}
1017
1018	dev_set_name(dev: &bridge->dev, name: "pci%04x:%02x", pci_domain_nr(bus),
1019	bridge->busnr);
1020
1021	err = pcibios_root_bridge_prepare(bridge);
1022	if (err)
1023	goto free;
1024
1025	/ Temporarily move resources off the list /
1026	list_splice_init(list: &bridge->windows, head: &resources);
1027	err = device_add(dev: &bridge->dev);
1028	if (err)
1029	goto free;
1030
1031	bus->bridge = get_device(dev: &bridge->dev);
1032	device_enable_async_suspend(dev: bus->bridge);
1033	pci_set_bus_of_node(bus);
1034	pci_set_bus_msi_domain(bus);
1035	if (bridge->msi_domain && !dev_get_msi_domain(dev: &bus->dev) &&
1036	!pci_host_of_has_msi_map(dev: parent))
1037	bus->bus_flags \|= PCI_BUS_FLAGS_NO_MSI;
1038
1039	if (!parent)
1040	set_dev_node(dev: bus->bridge, pcibus_to_node(bus));
1041
1042	bus->dev.class = &pcibus_class;
1043	bus->dev.parent = bus->bridge;
1044
1045	dev_set_name(dev: &bus->dev, name: "%04x:%02x", pci_domain_nr(bus), bus->number);
1046	name = dev_name(dev: &bus->dev);
1047
1048	err = device_register(dev: &bus->dev);
1049	bus_registered = true;
1050	if (err)
1051	goto unregister;
1052
1053	pcibios_add_bus(bus);
1054
1055	if (bus->ops->add_bus) {
1056	err = bus->ops->add_bus(bus);
1057	if (WARN_ON(err < `0`))
1058	dev_err(&bus->dev, "failed to add bus: %d\n", err);
1059	}
1060
1061	/ Create legacy_io and legacy_mem files for this bus /
1062	pci_create_legacy_files(bus);
1063
1064	if (parent)
1065	dev_info(parent, "PCI host bridge to bus %s\n", name);
1066	else
1067	pr_info("PCI host bridge to bus %s\n", name);
1068
1069	if (nr_node_ids > `1` && pcibus_to_node(bus) == NUMA_NO_NODE)
1070	dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
1071
1072	/ Check if the boot configuration by FW needs to be preserved /
1073	bridge->preserve_config = pci_preserve_config(host_bridge: bridge);
1074
1075	/ Coalesce contiguous windows /
1076	resource_list_for_each_entry_safe(window, n, &resources) {
1077	if (list_is_last(list: &window->node, head: &resources))
1078	break;
1079
1080	next = list_next_entry(window, node);
1081	offset = window->offset;
1082	res = window->res;
1083	next_offset = next->offset;
1084	next_res = next->res;
1085
1086	if (res->flags != next_res->flags \|\| offset != next_offset)
1087	continue;
1088
1089	if (res->end + `1` == next_res->start) {
1090	next_res->start = res->start;
1091	res->flags = res->start = res->end = `0`;
1092	}
1093	}
1094
1095	/ Add initial resources to the bus /
1096	resource_list_for_each_entry_safe(window, n, &resources) {
1097	offset = window->offset;
1098	res = window->res;
1099	if (!res->flags && !res->start && !res->end) {
1100	release_resource(new: res);
1101	resource_list_destroy_entry(entry: window);
1102	continue;
1103	}
1104
1105	list_move_tail(list: &window->node, head: &bridge->windows);
1106
1107	if (res->flags & IORESOURCE_BUS)
1108	pci_bus_insert_busn_res(b: bus, bus: bus->number, busmax: res->end);
1109	else
1110	pci_bus_add_resource(bus, res);
1111
1112	if (offset) {
1113	if (resource_type(res) == IORESOURCE_IO)
1114	fmt = " (bus address [%#06llx-%#06llx])";
1115	else
1116	fmt = " (bus address [%#010llx-%#010llx])";
1117
1118	snprintf(buf: addr, size: sizeof(addr), fmt,
1119	(unsigned long long)(res->start - offset),
1120	(unsigned long long)(res->end - offset));
1121	} else
1122	addr[`0`] = `'\0'`;
1123
1124	dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1125	}
1126
1127	of_pci_make_host_bridge_node(bridge);
1128
1129	down_write(sem: &pci_bus_sem);
1130	list_add_tail(new: &bus->node, head: &pci_root_buses);
1131	up_write(sem: &pci_bus_sem);
1132
1133	return `0`;
1134
1135	unregister:
1136	put_device(dev: &bridge->dev);
1137	device_del(dev: &bridge->dev);
1138	free:
1139	#ifdef CONFIG_PCI_DOMAINS_GENERIC
1140	pci_bus_release_domain_nr(parent, bus->domain_nr);
1141	#endif
1142	if (bus_registered)
1143	put_device(dev: &bus->dev);
1144	else
1145	kfree(objp: bus);
1146
1147	return err;
1148	}
1149
1150	static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1151	{
1152	int pos;
1153	u32 status;
1154
1155	/*
1156	* If extended config space isn't accessible on a bridge's primary
1157	* bus, we certainly can't access it on the secondary bus.
1158	*/
1159	if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1160	return false;
1161
1162	/*
1163	* PCIe Root Ports and switch ports are PCIe on both sides, so if
1164	* extended config space is accessible on the primary, it's also
1165	* accessible on the secondary.
1166	*/
1167	if (pci_is_pcie(dev: bridge) &&
1168	(pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT \|\|
1169	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_UPSTREAM \|\|
1170	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1171	return true;
1172
1173	/*
1174	* For the other bridge types:
1175	* - PCI-to-PCI bridges
1176	* - PCIe-to-PCI/PCI-X forward bridges
1177	* - PCI/PCI-X-to-PCIe reverse bridges
1178	* extended config space on the secondary side is only accessible
1179	* if the bridge supports PCI-X Mode 2.
1180	*/
1181	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
1182	if (!pos)
1183	return false;
1184
1185	pci_read_config_dword(dev: bridge, where: pos + PCI_X_STATUS, val: &status);
1186	return status & (PCI_X_STATUS_266MHZ \| PCI_X_STATUS_533MHZ);
1187	}
1188
1189	static struct pci_bus pci_alloc_child_bus(struct* pci_bus *parent,
1190	struct pci_dev bridge, int* busnr)
1191	{
1192	struct pci_bus *child;
1193	struct pci_host_bridge *host;
1194	int i;
1195	int ret;
1196
1197	/ Allocate a new bus and inherit stuff from the parent /
1198	child = pci_alloc_bus(parent);
1199	if (!child)
1200	return NULL;
1201
1202	child->parent = parent;
1203	child->sysdata = parent->sysdata;
1204	child->bus_flags = parent->bus_flags;
1205
1206	host = pci_find_host_bridge(bus: parent);
1207	if (host->child_ops)
1208	child->ops = host->child_ops;
1209	else
1210	child->ops = parent->ops;
1211
1212	/*
1213	* Initialize some portions of the bus device, but don't register
1214	* it now as the parent is not properly set up yet.
1215	*/
1216	child->dev.class = &pcibus_class;
1217	dev_set_name(dev: &child->dev, name: "%04x:%02x", pci_domain_nr(bus: child), busnr);
1218
1219	/ Set up the primary, secondary and subordinate bus numbers /
1220	child->number = child->busn_res.start = busnr;
1221	child->primary = parent->busn_res.start;
1222	child->busn_res.end = `0xff`;
1223
1224	if (!bridge) {
1225	child->dev.parent = parent->bridge;
1226	goto add_dev;
1227	}
1228
1229	child->self = bridge;
1230	child->bridge = get_device(dev: &bridge->dev);
1231	child->dev.parent = child->bridge;
1232	pci_set_bus_of_node(bus: child);
1233	pci_set_bus_speed(bus: child);
1234
1235	/*
1236	* Check whether extended config space is accessible on the child
1237	* bus. Note that we currently assume it is always accessible on
1238	* the root bus.
1239	*/
1240	if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1241	child->bus_flags \|= PCI_BUS_FLAGS_NO_EXTCFG;
1242	pci_info(child, "extended config space not accessible\n");
1243	}
1244
1245	/ Set up default resource pointers and names /
1246	for (i = `0`; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1247	child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1248	child->resource[i]->name = child->name;
1249	}
1250	bridge->subordinate = child;
1251
1252	add_dev:
1253	pci_set_bus_msi_domain(bus: child);
1254	ret = device_register(dev: &child->dev);
1255	if (WARN_ON(ret < `0`)) {
1256	put_device(dev: &child->dev);
1257	return NULL;
1258	}
1259
1260	pcibios_add_bus(bus: child);
1261
1262	if (child->ops->add_bus) {
1263	ret = child->ops->add_bus(child);
1264	if (WARN_ON(ret < `0`))
1265	dev_err(&child->dev, "failed to add bus: %d\n", ret);
1266	}
1267
1268	/ Create legacy_io and legacy_mem files for this bus /
1269	pci_create_legacy_files(bus: child);
1270
1271	return child;
1272	}
1273
1274	struct pci_bus pci_add_new_bus(struct* pci_bus parent, struct* pci_dev *dev,
1275	int busnr)
1276	{
1277	struct pci_bus *child;
1278
1279	child = pci_alloc_child_bus(parent, bridge: dev, busnr);
1280	if (child) {
1281	down_write(sem: &pci_bus_sem);
1282	list_add_tail(new: &child->node, head: &parent->children);
1283	up_write(sem: &pci_bus_sem);
1284	}
1285	return child;
1286	}
1287	EXPORT_SYMBOL(pci_add_new_bus);
1288
1289	static void pci_enable_rrs_sv(struct pci_dev *pdev)
1290	{
1291	u16 root_cap = `0`;
1292
1293	/ Enable Configuration RRS Software Visibility if supported /
1294	pcie_capability_read_word(dev: pdev, PCI_EXP_RTCAP, val: &root_cap);
1295	if (root_cap & PCI_EXP_RTCAP_RRS_SV) {
1296	pcie_capability_set_word(dev: pdev, PCI_EXP_RTCTL,
1297	PCI_EXP_RTCTL_RRS_SVE);
1298	pdev->config_rrs_sv = `1`;
1299	}
1300	}
1301
1302	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1303	unsigned int available_buses);
1304	/**
1305	* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1306	* numbers from EA capability.
1307	* @dev: Bridge
1308	* @sec: updated with secondary bus number from EA
1309	* @sub: updated with subordinate bus number from EA
1310	*
1311	* If @dev is a bridge with EA capability that specifies valid secondary
1312	* and subordinate bus numbers, return true with the bus numbers in @sec
1313	* and @sub. Otherwise return false.
1314	*/
1315	static bool pci_ea_fixed_busnrs(struct pci_dev dev, u8 sec, u8 *sub)
1316	{
1317	int ea, offset;
1318	u32 dw;
1319	u8 ea_sec, ea_sub;
1320
1321	if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1322	return false;
1323
1324	/ find PCI EA capability in list /
1325	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1326	if (!ea)
1327	return false;
1328
1329	offset = ea + PCI_EA_FIRST_ENT;
1330	pci_read_config_dword(dev, where: offset, val: &dw);
1331	ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1332	ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1333	if (ea_sec == `0` \|\| ea_sub < ea_sec)
1334	return false;
1335
1336	*sec = ea_sec;
1337	*sub = ea_sub;
1338	return true;
1339	}
1340
1341	/*
1342	* pci_scan_bridge_extend() - Scan buses behind a bridge
1343	* @bus: Parent bus the bridge is on
1344	* @dev: Bridge itself
1345	* @max: Starting subordinate number of buses behind this bridge
1346	* @available_buses: Total number of buses available for this bridge and
1347	* the devices below. After the minimal bus space has
1348	* been allocated the remaining buses will be
1349	* distributed equally between hotplug-capable bridges.
1350	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1351	* that need to be reconfigured.
1352	*
1353	* If it's a bridge, configure it and scan the bus behind it.
1354	* For CardBus bridges, we don't scan behind as the devices will
1355	* be handled by the bridge driver itself.
1356	*
1357	* We need to process bridges in two passes -- first we scan those
1358	* already configured by the BIOS and after we are done with all of
1359	* them, we proceed to assigning numbers to the remaining buses in
1360	* order to avoid overlaps between old and new bus numbers.
1361	*
1362	* Return: New subordinate number covering all buses behind this bridge.
1363	*/
1364	static int pci_scan_bridge_extend(struct pci_bus bus, struct* pci_dev *dev,
1365	int max, unsigned int available_buses,
1366	int pass)
1367	{
1368	struct pci_bus *child;
1369	int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1370	u32 buses, i, j = `0`;
1371	u16 bctl;
1372	u8 primary, secondary, subordinate;
1373	int broken = `0`;
1374	bool fixed_buses;
1375	u8 fixed_sec, fixed_sub;
1376	int next_busnr;
1377
1378	/*
1379	* Make sure the bridge is powered on to be able to access config
1380	* space of devices below it.
1381	*/
1382	pm_runtime_get_sync(dev: &dev->dev);
1383
1384	pci_read_config_dword(dev, PCI_PRIMARY_BUS, val: &buses);
1385	primary = buses & `0xFF`;
1386	secondary = (buses >> `8`) & `0xFF`;
1387	subordinate = (buses >> `16`) & `0xFF`;
1388
1389	pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1390	secondary, subordinate, pass);
1391
1392	if (!primary && (primary != bus->number) && secondary && subordinate) {
1393	pci_warn(dev, "Primary bus is hard wired to 0\n");
1394	primary = bus->number;
1395	}
1396
1397	/ Check if setup is sensible at all /
1398	if (!pass &&
1399	(primary != bus->number \|\| secondary <= bus->number \|\|
1400	secondary > subordinate)) {
1401	pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1402	secondary, subordinate);
1403	broken = `1`;
1404	}
1405
1406	/*
1407	* Disable Master-Abort Mode during probing to avoid reporting of
1408	* bus errors in some architectures.
1409	*/
1410	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &bctl);
1411	pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1412	val: bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1413
1414	if ((secondary \|\| subordinate) && !pcibios_assign_all_busses() &&
1415	!is_cardbus && !broken) {
1416	unsigned int cmax, buses;
1417
1418	/*
1419	* Bus already configured by firmware, process it in the
1420	* first pass and just note the configuration.
1421	*/
1422	if (pass)
1423	goto out;
1424
1425	/*
1426	* The bus might already exist for two reasons: Either we
1427	* are rescanning the bus or the bus is reachable through
1428	* more than one bridge. The second case can happen with
1429	* the i450NX chipset.
1430	*/
1431	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: secondary);
1432	if (!child) {
1433	child = pci_add_new_bus(bus, dev, secondary);
1434	if (!child)
1435	goto out;
1436	child->primary = primary;
1437	pci_bus_insert_busn_res(b: child, bus: secondary, busmax: subordinate);
1438	child->bridge_ctl = bctl;
1439	}
1440
1441	buses = subordinate - secondary;
1442	cmax = pci_scan_child_bus_extend(bus: child, available_buses: buses);
1443	if (cmax > subordinate)
1444	pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1445	subordinate, cmax);
1446
1447	/ Subordinate should equal child->busn_res.end /
1448	if (subordinate > max)
1449	max = subordinate;
1450	} else {
1451
1452	/*
1453	* We need to assign a number to this bus which we always
1454	* do in the second pass.
1455	*/
1456	if (!pass) {
1457	if (pcibios_assign_all_busses() \|\| broken \|\| is_cardbus)
1458
1459	/*
1460	* Temporarily disable forwarding of the
1461	* configuration cycles on all bridges in
1462	* this bus segment to avoid possible
1463	* conflicts in the second pass between two
1464	* bridges programmed with overlapping bus
1465	* ranges.
1466	*/
1467	pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1468	val: buses & ~`0xffffff`);
1469	goto out;
1470	}
1471
1472	/ Clear errors /
1473	pci_write_config_word(dev, PCI_STATUS, val: `0xffff`);
1474
1475	/ Read bus numbers from EA Capability (if present) /
1476	fixed_buses = pci_ea_fixed_busnrs(dev, sec: &fixed_sec, sub: &fixed_sub);
1477	if (fixed_buses)
1478	next_busnr = fixed_sec;
1479	else
1480	next_busnr = max + `1`;
1481
1482	/*
1483	* Prevent assigning a bus number that already exists.
1484	* This can happen when a bridge is hot-plugged, so in this
1485	* case we only re-scan this bus.
1486	*/
1487	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: next_busnr);
1488	if (!child) {
1489	child = pci_add_new_bus(bus, dev, next_busnr);
1490	if (!child)
1491	goto out;
1492	pci_bus_insert_busn_res(b: child, bus: next_busnr,
1493	busmax: bus->busn_res.end);
1494	}
1495	max++;
1496	if (available_buses)
1497	available_buses--;
1498
1499	buses = (buses & `0xff000000`)
1500	\| ((unsigned int)(child->primary) << `0`)
1501	\| ((unsigned int)(child->busn_res.start) << `8`)
1502	\| ((unsigned int)(child->busn_res.end) << `16`);
1503
1504	/*
1505	* yenta.c forces a secondary latency timer of 176.
1506	* Copy that behaviour here.
1507	*/
1508	if (is_cardbus) {
1509	buses &= ~`0xff000000`;
1510	buses \|= CARDBUS_LATENCY_TIMER << `24`;
1511	}
1512
1513	/ We need to blast all three values with a single write /
1514	pci_write_config_dword(dev, PCI_PRIMARY_BUS, val: buses);
1515
1516	if (!is_cardbus) {
1517	child->bridge_ctl = bctl;
1518	max = pci_scan_child_bus_extend(bus: child, available_buses);
1519	} else {
1520
1521	/*
1522	* For CardBus bridges, we leave 4 bus numbers as
1523	* cards with a PCI-to-PCI bridge can be inserted
1524	* later.
1525	*/
1526	for (i = `0`; i < CARDBUS_RESERVE_BUSNR; i++) {
1527	struct pci_bus *parent = bus;
1528	if (pci_find_bus(domain: pci_domain_nr(bus),
1529	busnr: max+i+`1`))
1530	break;
1531	while (parent->parent) {
1532	if ((!pcibios_assign_all_busses()) &&
1533	(parent->busn_res.end > max) &&
1534	(parent->busn_res.end <= max+i)) {
1535	j = `1`;
1536	}
1537	parent = parent->parent;
1538	}
1539	if (j) {
1540
1541	/*
1542	* Often, there are two CardBus
1543	* bridges -- try to leave one
1544	* valid bus number for each one.
1545	*/
1546	i /= `2`;
1547	break;
1548	}
1549	}
1550	max += i;
1551	}
1552
1553	/*
1554	* Set subordinate bus number to its real value.
1555	* If fixed subordinate bus number exists from EA
1556	* capability then use it.
1557	*/
1558	if (fixed_buses)
1559	max = fixed_sub;
1560	pci_bus_update_busn_res_end(b: child, busmax: max);
1561	pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, val: max);
1562	}
1563
1564	sprintf(buf: child->name,
1565	fmt: (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1566	pci_domain_nr(bus), child->number);
1567
1568	/ Check that all devices are accessible /
1569	while (bus->parent) {
1570	if ((child->busn_res.end > bus->busn_res.end) \|\|
1571	(child->number > bus->busn_res.end) \|\|
1572	(child->number < bus->number) \|\|
1573	(child->busn_res.end < bus->number)) {
1574	dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1575	&child->busn_res);
1576	break;
1577	}
1578	bus = bus->parent;
1579	}
1580
1581	out:
1582	/ Clear errors in the Secondary Status Register /
1583	pci_write_config_word(dev, PCI_SEC_STATUS, val: `0xffff`);
1584
1585	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: bctl);
1586
1587	pm_runtime_put(dev: &dev->dev);
1588
1589	return max;
1590	}
1591
1592	/*
1593	* pci_scan_bridge() - Scan buses behind a bridge
1594	* @bus: Parent bus the bridge is on
1595	* @dev: Bridge itself
1596	* @max: Starting subordinate number of buses behind this bridge
1597	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1598	* that need to be reconfigured.
1599	*
1600	* If it's a bridge, configure it and scan the bus behind it.
1601	* For CardBus bridges, we don't scan behind as the devices will
1602	* be handled by the bridge driver itself.
1603	*
1604	* We need to process bridges in two passes -- first we scan those
1605	* already configured by the BIOS and after we are done with all of
1606	* them, we proceed to assigning numbers to the remaining buses in
1607	* order to avoid overlaps between old and new bus numbers.
1608	*
1609	* Return: New subordinate number covering all buses behind this bridge.
1610	*/
1611	int pci_scan_bridge(struct pci_bus bus, struct* pci_dev dev, int* max, int pass)
1612	{
1613	return pci_scan_bridge_extend(bus, dev, max, available_buses: `0`, pass);
1614	}
1615	EXPORT_SYMBOL(pci_scan_bridge);
1616
1617	/*
1618	* Read interrupt line and base address registers.
1619	* The architecture-dependent code can tweak these, of course.
1620	*/
1621	static void pci_read_irq(struct pci_dev *dev)
1622	{
1623	unsigned char irq;
1624
1625	/ VFs are not allowed to use INTx, so skip the config reads /
1626	if (dev->is_virtfn) {
1627	dev->pin = `0`;
1628	dev->irq = `0`;
1629	return;
1630	}
1631
1632	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, val: &irq);
1633	dev->pin = irq;
1634	if (irq)
1635	pci_read_config_byte(dev, PCI_INTERRUPT_LINE, val: &irq);
1636	dev->irq = irq;
1637	}
1638
1639	void set_pcie_port_type(struct pci_dev *pdev)
1640	{
1641	int pos;
1642	u16 reg16;
1643	u32 reg32;
1644	int type;
1645	struct pci_dev *parent;
1646
1647	pos = pci_find_capability(dev: pdev, PCI_CAP_ID_EXP);
1648	if (!pos)
1649	return;
1650
1651	pdev->pcie_cap = pos;
1652	pci_read_config_word(dev: pdev, where: pos + PCI_EXP_FLAGS, val: &reg16);
1653	pdev->pcie_flags_reg = reg16;
1654
1655	type = pci_pcie_type(dev: pdev);
1656	if (type == PCI_EXP_TYPE_ROOT_PORT)
1657	pci_enable_rrs_sv(pdev);
1658
1659	pci_read_config_dword(dev: pdev, where: pos + PCI_EXP_DEVCAP, val: &pdev->devcap);
1660	pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1661
1662	pcie_capability_read_dword(dev: pdev, PCI_EXP_LNKCAP, val: &reg32);
1663	if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1664	pdev->link_active_reporting = `1`;
1665
1666	parent = pci_upstream_bridge(dev: pdev);
1667	if (!parent)
1668	return;
1669
1670	/*
1671	* Some systems do not identify their upstream/downstream ports
1672	* correctly so detect impossible configurations here and correct
1673	* the port type accordingly.
1674	*/
1675	if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1676	/*
1677	* If pdev claims to be downstream port but the parent
1678	* device is also downstream port assume pdev is actually
1679	* upstream port.
1680	*/
1681	if (pcie_downstream_port(dev: parent)) {
1682	pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1683	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1684	pdev->pcie_flags_reg \|= PCI_EXP_TYPE_UPSTREAM;
1685	}
1686	} else if (type == PCI_EXP_TYPE_UPSTREAM) {
1687	/*
1688	* If pdev claims to be upstream port but the parent
1689	* device is also upstream port assume pdev is actually
1690	* downstream port.
1691	*/
1692	if (pci_pcie_type(dev: parent) == PCI_EXP_TYPE_UPSTREAM) {
1693	pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1694	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1695	pdev->pcie_flags_reg \|= PCI_EXP_TYPE_DOWNSTREAM;
1696	}
1697	}
1698	}
1699
1700	void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1701	{
1702	u32 reg32;
1703
1704	pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP, val: &reg32);
1705	if (reg32 & PCI_EXP_SLTCAP_HPC)
1706	pdev->is_hotplug_bridge = pdev->is_pciehp = `1`;
1707	}
1708
1709	static void set_pcie_thunderbolt(struct pci_dev *dev)
1710	{
1711	u16 vsec;
1712
1713	/ Is the device part of a Thunderbolt controller? /
1714	vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1715	if (vsec)
1716	dev->is_thunderbolt = `1`;
1717	}
1718
1719	static void set_pcie_untrusted(struct pci_dev *dev)
1720	{
1721	struct pci_dev *parent = pci_upstream_bridge(dev);
1722
1723	if (!parent)
1724	return;
1725	/*
1726	* If the upstream bridge is untrusted we treat this device as
1727	* untrusted as well.
1728	*/
1729	if (parent->untrusted) {
1730	dev->untrusted = true;
1731	return;
1732	}
1733
1734	if (arch_pci_dev_is_removable(pdev: dev)) {
1735	pci_dbg(dev, "marking as untrusted\n");
1736	dev->untrusted = true;
1737	}
1738	}
1739
1740	static void pci_set_removable(struct pci_dev *dev)
1741	{
1742	struct pci_dev *parent = pci_upstream_bridge(dev);
1743
1744	if (!parent)
1745	return;
1746	/*
1747	* We (only) consider everything tunneled below an external_facing
1748	* device to be removable by the user. We're mainly concerned with
1749	* consumer platforms with user accessible thunderbolt ports that are
1750	* vulnerable to DMA attacks, and we expect those ports to be marked by
1751	* the firmware as external_facing. Devices in traditional hotplug
1752	* slots can technically be removed, but the expectation is that unless
1753	* the port is marked with external_facing, such devices are less
1754	* accessible to user / may not be removed by end user, and thus not
1755	* exposed as "removable" to userspace.
1756	*/
1757	if (dev_is_removable(dev: &parent->dev)) {
1758	dev_set_removable(dev: &dev->dev, removable: DEVICE_REMOVABLE);
1759	return;
1760	}
1761
1762	if (arch_pci_dev_is_removable(pdev: dev)) {
1763	pci_dbg(dev, "marking as removable\n");
1764	dev_set_removable(dev: &dev->dev, removable: DEVICE_REMOVABLE);
1765	}
1766	}
1767
1768	/**
1769	* pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1770	* @dev: PCI device
1771	*
1772	* PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1773	* when forwarding a type1 configuration request the bridge must check that
1774	* the extended register address field is zero. The bridge is not permitted
1775	* to forward the transactions and must handle it as an Unsupported Request.
1776	* Some bridges do not follow this rule and simply drop the extended register
1777	* bits, resulting in the standard config space being aliased, every 256
1778	* bytes across the entire configuration space. Test for this condition by
1779	* comparing the first dword of each potential alias to the vendor/device ID.
1780	* Known offenders:
1781	* ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1782	* AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1783	*/
1784	static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1785	{
1786	#ifdef CONFIG_PCI_QUIRKS
1787	int pos, ret;
1788	u32 header, tmp;
1789
1790	pci_read_config_dword(dev, PCI_VENDOR_ID, val: &header);
1791
1792	for (pos = PCI_CFG_SPACE_SIZE;
1793	pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1794	ret = pci_read_config_dword(dev, where: pos, val: &tmp);
1795	if ((ret != PCIBIOS_SUCCESSFUL) \|\| (header != tmp))
1796	return false;
1797	}
1798
1799	return true;
1800	#else
1801	return false;
1802	#endif
1803	}
1804
1805	/**
1806	* pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1807	* @dev: PCI device
1808	*
1809	* Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1810	* have 4096 bytes. Even if the device is capable, that doesn't mean we can
1811	* access it. Maybe we don't have a way to generate extended config space
1812	* accesses, or the device is behind a reverse Express bridge. So we try
1813	* reading the dword at 0x100 which must either be 0 or a valid extended
1814	* capability header.
1815	*/
1816	static int pci_cfg_space_size_ext(struct pci_dev *dev)
1817	{
1818	u32 status;
1819	int pos = PCI_CFG_SPACE_SIZE;
1820
1821	if (pci_read_config_dword(dev, where: pos, val: &status) != PCIBIOS_SUCCESSFUL)
1822	return PCI_CFG_SPACE_SIZE;
1823	if (PCI_POSSIBLE_ERROR(status) \|\| pci_ext_cfg_is_aliased(dev))
1824	return PCI_CFG_SPACE_SIZE;
1825
1826	return PCI_CFG_SPACE_EXP_SIZE;
1827	}
1828
1829	int pci_cfg_space_size(struct pci_dev *dev)
1830	{
1831	int pos;
1832	u32 status;
1833	u16 class;
1834
1835	#ifdef CONFIG_PCI_IOV
1836	/*
1837	* Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1838	* implement a PCIe capability and therefore must implement extended
1839	* config space. We can skip the NO_EXTCFG test below and the
1840	* reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1841	* the fact that the SR-IOV capability on the PF resides in extended
1842	* config space and must be accessible and non-aliased to have enabled
1843	* support for this VF. This is a micro performance optimization for
1844	* systems supporting many VFs.
1845	*/
1846	if (dev->is_virtfn)
1847	return PCI_CFG_SPACE_EXP_SIZE;
1848	#endif
1849
1850	if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1851	return PCI_CFG_SPACE_SIZE;
1852
1853	class = dev->class >> `8`;
1854	if (class == PCI_CLASS_BRIDGE_HOST)
1855	return pci_cfg_space_size_ext(dev);
1856
1857	if (pci_is_pcie(dev))
1858	return pci_cfg_space_size_ext(dev);
1859
1860	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1861	if (!pos)
1862	return PCI_CFG_SPACE_SIZE;
1863
1864	pci_read_config_dword(dev, where: pos + PCI_X_STATUS, val: &status);
1865	if (status & (PCI_X_STATUS_266MHZ \| PCI_X_STATUS_533MHZ))
1866	return pci_cfg_space_size_ext(dev);
1867
1868	return PCI_CFG_SPACE_SIZE;
1869	}
1870
1871	static u32 pci_class(struct pci_dev *dev)
1872	{
1873	u32 class;
1874
1875	#ifdef CONFIG_PCI_IOV
1876	if (dev->is_virtfn)
1877	return dev->physfn->sriov->class;
1878	#endif
1879	pci_read_config_dword(dev, PCI_CLASS_REVISION, val: &class);
1880	return class;
1881	}
1882
1883	static void pci_subsystem_ids(struct pci_dev dev, u16 vendor, u16 *device)
1884	{
1885	#ifdef CONFIG_PCI_IOV
1886	if (dev->is_virtfn) {
1887	*vendor = dev->physfn->sriov->subsystem_vendor;
1888	*device = dev->physfn->sriov->subsystem_device;
1889	return;
1890	}
1891	#endif
1892	pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, val: vendor);
1893	pci_read_config_word(dev, PCI_SUBSYSTEM_ID, val: device);
1894	}
1895
1896	static u8 pci_hdr_type(struct pci_dev *dev)
1897	{
1898	u8 hdr_type;
1899
1900	#ifdef CONFIG_PCI_IOV
1901	if (dev->is_virtfn)
1902	return dev->physfn->sriov->hdr_type;
1903	#endif
1904	pci_read_config_byte(dev, PCI_HEADER_TYPE, val: &hdr_type);
1905	return hdr_type;
1906	}
1907
1908	#define LEGACY_IO_RESOURCE (IORESOURCE_IO \| IORESOURCE_PCI_FIXED)
1909
1910	/**
1911	* pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1912	* @dev: PCI device
1913	*
1914	* Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev. Check this
1915	* at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1916	*/
1917	static int pci_intx_mask_broken(struct pci_dev *dev)
1918	{
1919	u16 orig, toggle, new;
1920
1921	pci_read_config_word(dev, PCI_COMMAND, val: &orig);
1922	toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1923	pci_write_config_word(dev, PCI_COMMAND, val: toggle);
1924	pci_read_config_word(dev, PCI_COMMAND, val: &new);
1925
1926	pci_write_config_word(dev, PCI_COMMAND, val: orig);
1927
1928	/*
1929	* PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1930	* r2.3, so strictly speaking, a device is not broken if it's not
1931	* writable. But we'll live with the misnomer for now.
1932	*/
1933	if (new != toggle)
1934	return `1`;
1935	return `0`;
1936	}
1937
1938	static void early_dump_pci_device(struct pci_dev *pdev)
1939	{
1940	u32 value[PCI_CFG_SPACE_SIZE / sizeof(u32)];
1941	int i;
1942
1943	pci_info(pdev, "config space:\n");
1944
1945	for (i = `0`; i < ARRAY_SIZE(value); i++)
1946	pci_read_config_dword(dev: pdev, where: i * sizeof(u32), val: &value[i]);
1947
1948	print_hex_dump(KERN_INFO, prefix_str: "", prefix_type: DUMP_PREFIX_OFFSET, rowsize: `16`, groupsize: `1`,
1949	buf: value, ARRAY_SIZE(value) * sizeof(u32), ascii: false);
1950	}
1951
1952	static const char pci_type_str(struct* pci_dev *dev)
1953	{
1954	static const char * const str[] = {
1955	"PCIe Endpoint",
1956	"PCIe Legacy Endpoint",
1957	"PCIe unknown",
1958	"PCIe unknown",
1959	"PCIe Root Port",
1960	"PCIe Switch Upstream Port",
1961	"PCIe Switch Downstream Port",
1962	"PCIe to PCI/PCI-X bridge",
1963	"PCI/PCI-X to PCIe bridge",
1964	"PCIe Root Complex Integrated Endpoint",
1965	"PCIe Root Complex Event Collector",
1966	};
1967	int type;
1968
1969	if (pci_is_pcie(dev)) {
1970	type = pci_pcie_type(dev);
1971	if (type < ARRAY_SIZE(str))
1972	return str[type];
1973
1974	return "PCIe unknown";
1975	}
1976
1977	switch (dev->hdr_type) {
1978	case PCI_HEADER_TYPE_NORMAL:
1979	return "conventional PCI endpoint";
1980	case PCI_HEADER_TYPE_BRIDGE:
1981	return "conventional PCI bridge";
1982	case PCI_HEADER_TYPE_CARDBUS:
1983	return "CardBus bridge";
1984	default:
1985	return "conventional PCI";
1986	}
1987	}
1988
1989	/**
1990	* pci_setup_device - Fill in class and map information of a device
1991	* @dev: the device structure to fill
1992	*
1993	* Initialize the device structure with information about the device's
1994	* vendor,class,memory and IO-space addresses, IRQ lines etc.
1995	* Called at initialisation of the PCI subsystem and by CardBus services.
1996	* Returns 0 on success and negative if unknown type of device (not normal,
1997	* bridge or CardBus).
1998	*/
1999	int pci_setup_device(struct pci_dev *dev)
2000	{
2001	u32 class;
2002	u16 cmd;
2003	u8 hdr_type;
2004	int err, pos = `0`;
2005	struct pci_bus_region region;
2006	struct resource *res;
2007
2008	hdr_type = pci_hdr_type(dev);
2009
2010	dev->sysdata = dev->bus->sysdata;
2011	dev->dev.parent = dev->bus->bridge;
2012	dev->dev.bus = &pci_bus_type;
2013	dev->hdr_type = FIELD_GET(PCI_HEADER_TYPE_MASK, hdr_type);
2014	dev->multifunction = FIELD_GET(PCI_HEADER_TYPE_MFD, hdr_type);
2015	dev->error_state = pci_channel_io_normal;
2016	set_pcie_port_type(dev);
2017
2018	err = pci_set_of_node(dev);
2019	if (err)
2020	return err;
2021	pci_set_acpi_fwnode(dev);
2022
2023	pci_dev_assign_slot(dev);
2024
2025	/*
2026	* Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
2027	* set this higher, assuming the system even supports it.
2028	*/
2029	dev->dma_mask = `0xffffffff`;
2030
2031	dev_set_name(dev: &dev->dev, name: "%04x:%02x:%02x.%d", pci_domain_nr(bus: dev->bus),
2032	dev->bus->number, PCI_SLOT(dev->devfn),
2033	PCI_FUNC(dev->devfn));
2034
2035	class = pci_class(dev);
2036
2037	dev->revision = class & `0xff`;
2038	dev->class = class >> `8`; / upper 3 bytes /
2039
2040	if (pci_early_dump)
2041	early_dump_pci_device(pdev: dev);
2042
2043	/ Need to have dev->class ready /
2044	dev->cfg_size = pci_cfg_space_size(dev);
2045
2046	/ Need to have dev->cfg_size ready /
2047	set_pcie_thunderbolt(dev);
2048
2049	set_pcie_untrusted(dev);
2050
2051	if (pci_is_pcie(dev))
2052	dev->supported_speeds = pcie_get_supported_speeds(dev);
2053
2054	/ "Unknown power state" /
2055	dev->current_state = PCI_UNKNOWN;
2056
2057	/ Early fixups, before probing the BARs /
2058	pci_fixup_device(pass: pci_fixup_early, dev);
2059
2060	pci_set_removable(dev);
2061
2062	pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n",
2063	dev->vendor, dev->device, dev->hdr_type, dev->class,
2064	pci_type_str(dev));
2065
2066	/ Device class may be changed after fixup /
2067	class = dev->class >> `8`;
2068
2069	if (dev->non_compliant_bars && !dev->mmio_always_on) {
2070	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
2071	if (cmd & (PCI_COMMAND_IO \| PCI_COMMAND_MEMORY)) {
2072	pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
2073	cmd &= ~PCI_COMMAND_IO;
2074	cmd &= ~PCI_COMMAND_MEMORY;
2075	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
2076	}
2077	}
2078
2079	dev->broken_intx_masking = pci_intx_mask_broken(dev);
2080
2081	switch (dev->hdr_type) { / header type /
2082	case PCI_HEADER_TYPE_NORMAL: / standard header /
2083	if (class == PCI_CLASS_BRIDGE_PCI)
2084	goto bad;
2085	pci_read_irq(dev);
2086	pci_read_bases(dev, PCI_STD_NUM_BARS, PCI_ROM_ADDRESS);
2087
2088	pci_subsystem_ids(dev, vendor: &dev->subsystem_vendor, device: &dev->subsystem_device);
2089
2090	/*
2091	* Do the ugly legacy mode stuff here rather than broken chip
2092	* quirk code. Legacy mode ATA controllers have fixed
2093	* addresses. These are not always echoed in BAR0-3, and
2094	* BAR0-3 in a few cases contain junk!
2095	*/
2096	if (class == PCI_CLASS_STORAGE_IDE) {
2097	u8 progif;
2098	pci_read_config_byte(dev, PCI_CLASS_PROG, val: &progif);
2099	if ((progif & `1`) == `0`) {
2100	region.start = `0x1F0`;
2101	region.end = `0x1F7`;
2102	res = &dev->resource[`0`];
2103	res->flags = LEGACY_IO_RESOURCE;
2104	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
2105	pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n",
2106	res);
2107	region.start = `0x3F6`;
2108	region.end = `0x3F6`;
2109	res = &dev->resource[`1`];
2110	res->flags = LEGACY_IO_RESOURCE;
2111	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
2112	pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n",
2113	res);
2114	}
2115	if ((progif & `4`) == `0`) {
2116	region.start = `0x170`;
2117	region.end = `0x177`;
2118	res = &dev->resource[`2`];
2119	res->flags = LEGACY_IO_RESOURCE;
2120	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
2121	pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n",
2122	res);
2123	region.start = `0x376`;
2124	region.end = `0x376`;
2125	res = &dev->resource[`3`];
2126	res->flags = LEGACY_IO_RESOURCE;
2127	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
2128	pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n",
2129	res);
2130	}
2131	}
2132	break;
2133
2134	case PCI_HEADER_TYPE_BRIDGE: / bridge header /
2135	/*
2136	* The PCI-to-PCI bridge spec requires that subtractive
2137	* decoding (i.e. transparent) bridge must have programming
2138	* interface code of 0x01.
2139	*/
2140	pci_read_irq(dev);
2141	dev->transparent = ((dev->class & `0xff`) == `1`);
2142	pci_read_bases(dev, howmany: `2`, PCI_ROM_ADDRESS1);
2143	pci_read_bridge_windows(bridge: dev);
2144	set_pcie_hotplug_bridge(dev);
2145	pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
2146	if (pos) {
2147	pci_read_config_word(dev, where: pos + PCI_SSVID_VENDOR_ID, val: &dev->subsystem_vendor);
2148	pci_read_config_word(dev, where: pos + PCI_SSVID_DEVICE_ID, val: &dev->subsystem_device);
2149	}
2150	break;
2151
2152	case PCI_HEADER_TYPE_CARDBUS: / CardBus bridge header /
2153	if (class != PCI_CLASS_BRIDGE_CARDBUS)
2154	goto bad;
2155	pci_read_irq(dev);
2156	pci_read_bases(dev, howmany: `1`, rom: `0`);
2157	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, val: &dev->subsystem_vendor);
2158	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, val: &dev->subsystem_device);
2159	break;
2160
2161	default: / unknown header /
2162	pci_err(dev, "unknown header type %02x, ignoring device\n",
2163	dev->hdr_type);
2164	pci_release_of_node(dev);
2165	return -EIO;
2166
2167	bad:
2168	pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
2169	dev->class, dev->hdr_type);
2170	dev->class = PCI_CLASS_NOT_DEFINED << `8`;
2171	}
2172
2173	/ We found a fine healthy device, go go go... /
2174	return `0`;
2175	}
2176
2177	static void pci_configure_mps(struct pci_dev *dev)
2178	{
2179	struct pci_dev *bridge = pci_upstream_bridge(dev);
2180	int mps, mpss, p_mps, rc;
2181
2182	if (!pci_is_pcie(dev))
2183	return;
2184
2185	/ MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out /
2186	if (dev->is_virtfn)
2187	return;
2188
2189	/*
2190	* For Root Complex Integrated Endpoints, program the maximum
2191	* supported value unless limited by the PCIE_BUS_PEER2PEER case.
2192	*/
2193	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2194	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2195	mps = `128`;
2196	else
2197	mps = `128` << dev->pcie_mpss;
2198	rc = pcie_set_mps(dev, mps);
2199	if (rc) {
2200	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2201	mps);
2202	}
2203	return;
2204	}
2205
2206	if (!bridge \|\| !pci_is_pcie(dev: bridge))
2207	return;
2208
2209	mps = pcie_get_mps(dev);
2210	p_mps = pcie_get_mps(dev: bridge);
2211
2212	if (mps == p_mps)
2213	return;
2214
2215	if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2216	pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2217	mps, pci_name(bridge), p_mps);
2218	return;
2219	}
2220
2221	/*
2222	* Fancier MPS configuration is done later by
2223	* pcie_bus_configure_settings()
2224	*/
2225	if (pcie_bus_config != PCIE_BUS_DEFAULT)
2226	return;
2227
2228	mpss = `128` << dev->pcie_mpss;
2229	if (mpss < p_mps && pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2230	pcie_set_mps(dev: bridge, mps: mpss);
2231	pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2232	mpss, p_mps, `128` << bridge->pcie_mpss);
2233	p_mps = pcie_get_mps(dev: bridge);
2234	}
2235
2236	rc = pcie_set_mps(dev, mps: p_mps);
2237	if (rc) {
2238	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2239	p_mps);
2240	return;
2241	}
2242
2243	pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2244	p_mps, mps, mpss);
2245	}
2246
2247	int pci_configure_extended_tags(struct pci_dev dev, void* *ign)
2248	{
2249	struct pci_host_bridge *host;
2250	u32 cap;
2251	u16 ctl;
2252	int ret;
2253
2254	if (!pci_is_pcie(dev))
2255	return `0`;
2256
2257	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, val: &cap);
2258	if (ret)
2259	return `0`;
2260
2261	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2262	return `0`;
2263
2264	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
2265	if (ret)
2266	return `0`;
2267
2268	host = pci_find_host_bridge(bus: dev->bus);
2269	if (!host)
2270	return `0`;
2271
2272	/*
2273	* If some device in the hierarchy doesn't handle Extended Tags
2274	* correctly, make sure they're disabled.
2275	*/
2276	if (host->no_ext_tags) {
2277	if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2278	pci_info(dev, "disabling Extended Tags\n");
2279	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2280	PCI_EXP_DEVCTL_EXT_TAG);
2281	}
2282	return `0`;
2283	}
2284
2285	if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2286	pci_info(dev, "enabling Extended Tags\n");
2287	pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2288	PCI_EXP_DEVCTL_EXT_TAG);
2289	}
2290	return `0`;
2291	}
2292
2293	/**
2294	* pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2295	* @dev: PCI device to query
2296	*
2297	* Returns true if the device has enabled relaxed ordering attribute.
2298	*/
2299	bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2300	{
2301	u16 v;
2302
2303	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &v);
2304
2305	return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2306	}
2307	EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2308
2309	static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2310	{
2311	struct pci_dev *root;
2312
2313	/ PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs /
2314	if (dev->is_virtfn)
2315	return;
2316
2317	if (!pcie_relaxed_ordering_enabled(dev))
2318	return;
2319
2320	/*
2321	* For now, we only deal with Relaxed Ordering issues with Root
2322	* Ports. Peer-to-Peer DMA is another can of worms.
2323	*/
2324	root = pcie_find_root_port(dev);
2325	if (!root)
2326	return;
2327
2328	if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2329	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2330	PCI_EXP_DEVCTL_RELAX_EN);
2331	pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2332	}
2333	}
2334
2335	static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2336	{
2337	struct pci_dev *bridge;
2338	unsigned int eetlp_max;
2339	int pcie_type;
2340	u32 cap;
2341
2342	if (!pci_is_pcie(dev))
2343	return;
2344
2345	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, val: &cap);
2346	if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2347	return;
2348
2349	pcie_type = pci_pcie_type(dev);
2350
2351	eetlp_max = FIELD_GET(PCI_EXP_DEVCAP2_EE_PREFIX_MAX, cap);
2352	/ 00b means 4 /
2353	eetlp_max = eetlp_max ?: `4`;
2354
2355	if (pcie_type == PCI_EXP_TYPE_ROOT_PORT \|\|
2356	pcie_type == PCI_EXP_TYPE_RC_END)
2357	dev->eetlp_prefix_max = eetlp_max;
2358	else {
2359	bridge = pci_upstream_bridge(dev);
2360	if (bridge && bridge->eetlp_prefix_max)
2361	dev->eetlp_prefix_max = eetlp_max;
2362	}
2363	}
2364
2365	static void pci_configure_serr(struct pci_dev *dev)
2366	{
2367	u16 control;
2368
2369	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2370
2371	/*
2372	* A bridge will not forward ERR_ messages coming from an
2373	* endpoint unless SERR# forwarding is enabled.
2374	*/
2375	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &control);
2376	if (!(control & PCI_BRIDGE_CTL_SERR)) {
2377	control \|= PCI_BRIDGE_CTL_SERR;
2378	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: control);
2379	}
2380	}
2381	}
2382
2383	static void pci_configure_device(struct pci_dev *dev)
2384	{
2385	pci_configure_mps(dev);
2386	pci_configure_extended_tags(dev, NULL);
2387	pci_configure_relaxed_ordering(dev);
2388	pci_configure_ltr(pdev: dev);
2389	pci_configure_aspm_l1ss(dev);
2390	pci_configure_eetlp_prefix(dev);
2391	pci_configure_serr(dev);
2392
2393	pci_acpi_program_hp_params(dev);
2394	}
2395
2396	static void pci_release_capabilities(struct pci_dev *dev)
2397	{
2398	pci_aer_exit(d: dev);
2399	pci_rcec_exit(dev);
2400	pci_iov_release(dev);
2401	pci_free_cap_save_buffers(dev);
2402	}
2403
2404	/**
2405	* pci_release_dev - Free a PCI device structure when all users of it are
2406	* finished
2407	* @dev: device that's been disconnected
2408	*
2409	* Will be called only by the device core when all users of this PCI device are
2410	* done.
2411	*/
2412	static void pci_release_dev(struct device *dev)
2413	{
2414	struct pci_dev *pci_dev;
2415
2416	pci_dev = to_pci_dev(dev);
2417	pci_release_capabilities(dev: pci_dev);
2418	pci_release_of_node(dev: pci_dev);
2419	pcibios_release_device(dev: pci_dev);
2420	pci_bus_put(bus: pci_dev->bus);
2421	kfree(objp: pci_dev->driver_override);
2422	bitmap_free(bitmap: pci_dev->dma_alias_mask);
2423	dev_dbg(dev, "device released\n");
2424	kfree(objp: pci_dev);
2425	}
2426
2427	static const struct device_type pci_dev_type = {
2428	.groups = pci_dev_attr_groups,
2429	};
2430
2431	struct pci_dev pci_alloc_dev(struct* pci_bus *bus)
2432	{
2433	struct pci_dev *dev;
2434
2435	dev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
2436	if (!dev)
2437	return NULL;
2438
2439	INIT_LIST_HEAD(list: &dev->bus_list);
2440	dev->dev.type = &pci_dev_type;
2441	dev->bus = pci_bus_get(bus);
2442	dev->driver_exclusive_resource = (struct resource) {
2443	.name = "PCI Exclusive",
2444	.start = `0`,
2445	.end = -`1`,
2446	};
2447
2448	spin_lock_init(&dev->pcie_cap_lock);
2449	#ifdef CONFIG_PCI_MSI
2450	raw_spin_lock_init(&dev->msi_lock);
2451	#endif
2452	return dev;
2453	}
2454	EXPORT_SYMBOL(pci_alloc_dev);
2455
2456	static bool pci_bus_wait_rrs(struct pci_bus bus, int* devfn, u32 *l,
2457	int timeout)
2458	{
2459	int delay = `1`;
2460
2461	if (!pci_bus_rrs_vendor_id(l: *l))
2462	return true; / not a Configuration RRS completion /
2463
2464	if (!timeout)
2465	return false; / RRS, but caller doesn't want to wait /
2466
2467	/*
2468	* We got the reserved Vendor ID that indicates a completion with
2469	* Configuration Request Retry Status (RRS). Retry until we get a
2470	* valid Vendor ID or we time out.
2471	*/
2472	while (pci_bus_rrs_vendor_id(l: *l)) {
2473	if (delay > timeout) {
2474	pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2475	pci_domain_nr(bus), bus->number,
2476	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - `1`);
2477
2478	return false;
2479	}
2480	if (delay >= `1000`)
2481	pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2482	pci_domain_nr(bus), bus->number,
2483	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - `1`);
2484
2485	msleep(msecs: delay);
2486	delay *= `2`;
2487
2488	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2489	return false;
2490	}
2491
2492	if (delay >= `1000`)
2493	pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2494	pci_domain_nr(bus), bus->number,
2495	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - `1`);
2496
2497	return true;
2498	}
2499
2500	bool pci_bus_generic_read_dev_vendor_id(struct pci_bus bus, int* devfn, u32 *l,
2501	int timeout)
2502	{
2503	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2504	return false;
2505
2506	/ Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: /
2507	if (PCI_POSSIBLE_ERROR(l) \|\| l == `0x00000000` \|\|
2508	l == `0x0000ffff` \|\| l == `0xffff0000`)
2509	return false;
2510
2511	if (pci_bus_rrs_vendor_id(l: *l))
2512	return pci_bus_wait_rrs(bus, devfn, l, timeout);
2513
2514	return true;
2515	}
2516
2517	bool pci_bus_read_dev_vendor_id(struct pci_bus bus, int* devfn, u32 *l,
2518	int timeout)
2519	{
2520	#ifdef CONFIG_PCI_QUIRKS
2521	struct pci_dev *bridge = bus->self;
2522
2523	/*
2524	* Certain IDT switches have an issue where they improperly trigger
2525	* ACS Source Validation errors on completions for config reads.
2526	*/
2527	if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2528	bridge->device == `0x80b5`)
2529	return pci_idt_bus_quirk(bus, devfn, pl: l, rrs_timeout: timeout);
2530	#endif
2531
2532	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2533	}
2534	EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2535
2536	#if IS_ENABLED(CONFIG_PCI_PWRCTRL)
2537	static struct platform_device pci_pwrctrl_create_device(struct* pci_bus bus, int* devfn)
2538	{
2539	struct pci_host_bridge *host = pci_find_host_bridge(bus);
2540	struct platform_device *pdev;
2541	struct device_node *np;
2542
2543	np = of_pci_find_child_device(dev_of_node(&bus->dev), devfn);
2544	if (!np)
2545	return NULL;
2546
2547	pdev = of_find_device_by_node(np);
2548	if (pdev) {
2549	put_device(&pdev->dev);
2550	goto err_put_of_node;
2551	}
2552
2553	/*
2554	* First check whether the pwrctrl device really needs to be created or
2555	* not. This is decided based on at least one of the power supplies
2556	* being defined in the devicetree node of the device.
2557	*/
2558	if (!of_pci_supply_present(np)) {
2559	pr_debug("PCI/pwrctrl: Skipping OF node: %s\n", np->name);
2560	goto err_put_of_node;
2561	}
2562
2563	/ Now create the pwrctrl device /
2564	pdev = of_platform_device_create(np, NULL, &host->dev);
2565	if (!pdev) {
2566	pr_err("PCI/pwrctrl: Failed to create pwrctrl device for node: %s\n", np->name);
2567	goto err_put_of_node;
2568	}
2569
2570	of_node_put(np);
2571
2572	return pdev;
2573
2574	err_put_of_node:
2575	of_node_put(np);
2576
2577	return NULL;
2578	}
2579	#else
2580	static struct platform_device pci_pwrctrl_create_device(struct* pci_bus bus, int* devfn)
2581	{
2582	return NULL;
2583	}
2584	#endif
2585
2586	/*
2587	* Read the config data for a PCI device, sanity-check it,
2588	* and fill in the dev structure.
2589	*/
2590	static struct pci_dev pci_scan_device(struct* pci_bus bus, int* devfn)
2591	{
2592	struct pci_dev *dev;
2593	u32 l;
2594
2595	/*
2596	* Create pwrctrl device (if required) for the PCI device to handle the
2597	* power state. If the pwrctrl device is created, then skip scanning
2598	* further as the pwrctrl core will rescan the bus after powering on
2599	* the device.
2600	*/
2601	if (pci_pwrctrl_create_device(bus, devfn))
2602	return NULL;
2603
2604	if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, `60`*`1000`))
2605	return NULL;
2606
2607	dev = pci_alloc_dev(bus);
2608	if (!dev)
2609	return NULL;
2610
2611	dev->devfn = devfn;
2612	dev->vendor = l & `0xffff`;
2613	dev->device = (l >> `16`) & `0xffff`;
2614
2615	if (pci_setup_device(dev)) {
2616	pci_bus_put(bus: dev->bus);
2617	kfree(objp: dev);
2618	return NULL;
2619	}
2620
2621	return dev;
2622	}
2623
2624	void pcie_report_downtraining(struct pci_dev *dev)
2625	{
2626	if (!pci_is_pcie(dev))
2627	return;
2628
2629	/ Look from the device up to avoid downstream ports with no devices /
2630	if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2631	(pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2632	(pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2633	return;
2634
2635	/ Multi-function PCIe devices share the same link/status /
2636	if (PCI_FUNC(dev->devfn) != `0` \|\| dev->is_virtfn)
2637	return;
2638
2639	/ Print link status only if the device is constrained by the fabric /
2640	__pcie_print_link_status(dev, verbose: false);
2641	}
2642
2643	static void pci_imm_ready_init(struct pci_dev *dev)
2644	{
2645	u16 status;
2646
2647	pci_read_config_word(dev, PCI_STATUS, val: &status);
2648	if (status & PCI_STATUS_IMM_READY)
2649	dev->imm_ready = `1`;
2650	}
2651
2652	static void pci_init_capabilities(struct pci_dev *dev)
2653	{
2654	pci_ea_init(dev); / Enhanced Allocation /
2655	pci_msi_init(dev); / Disable MSI /
2656	pci_msix_init(dev); / Disable MSI-X /
2657
2658	/ Buffers for saving PCIe and PCI-X capabilities /
2659	pci_allocate_cap_save_buffers(dev);
2660
2661	pci_imm_ready_init(dev); / Immediate Readiness /
2662	pci_pm_init(dev); / Power Management /
2663	pci_vpd_init(dev); / Vital Product Data /
2664	pci_configure_ari(dev); / Alternative Routing-ID Forwarding /
2665	pci_iov_init(dev); / Single Root I/O Virtualization /
2666	pci_ats_init(dev); / Address Translation Services /
2667	pci_pri_init(dev); / Page Request Interface /
2668	pci_pasid_init(dev); / Process Address Space ID /
2669	pci_acs_init(dev); / Access Control Services /
2670	pci_ptm_init(dev); / Precision Time Measurement /
2671	pci_aer_init(d: dev); / Advanced Error Reporting /
2672	pci_dpc_init(pdev: dev); / Downstream Port Containment /
2673	pci_rcec_init(dev); / Root Complex Event Collector /
2674	pci_doe_init(pdev: dev); / Data Object Exchange /
2675	pci_tph_init(dev); / TLP Processing Hints /
2676	pci_rebar_init(pdev: dev); / Resizable BAR /
2677
2678	pcie_report_downtraining(dev);
2679	pci_init_reset_methods(dev);
2680	}
2681
2682	/*
2683	* This is the equivalent of pci_host_bridge_msi_domain() that acts on
2684	* devices. Firmware interfaces that can select the MSI domain on a
2685	* per-device basis should be called from here.
2686	*/
2687	static struct irq_domain pci_dev_msi_domain(struct* pci_dev *dev)
2688	{
2689	struct irq_domain *d;
2690
2691	/*
2692	* If a domain has been set through the pcibios_device_add()
2693	* callback, then this is the one (platform code knows best).
2694	*/
2695	d = dev_get_msi_domain(dev: &dev->dev);
2696	if (d)
2697	return d;
2698
2699	/*
2700	* Let's see if we have a firmware interface able to provide
2701	* the domain.
2702	*/
2703	d = pci_msi_get_device_domain(pdev: dev);
2704	if (d)
2705	return d;
2706
2707	return NULL;
2708	}
2709
2710	static void pci_set_msi_domain(struct pci_dev *dev)
2711	{
2712	struct irq_domain *d;
2713
2714	/*
2715	* If the platform or firmware interfaces cannot supply a
2716	* device-specific MSI domain, then inherit the default domain
2717	* from the host bridge itself.
2718	*/
2719	d = pci_dev_msi_domain(dev);
2720	if (!d)
2721	d = dev_get_msi_domain(dev: &dev->bus->dev);
2722
2723	dev_set_msi_domain(dev: &dev->dev, d);
2724	}
2725
2726	void pci_device_add(struct pci_dev dev, struct* pci_bus *bus)
2727	{
2728	int ret;
2729
2730	pci_configure_device(dev);
2731
2732	device_initialize(dev: &dev->dev);
2733	dev->dev.release = pci_release_dev;
2734
2735	set_dev_node(dev: &dev->dev, pcibus_to_node(bus));
2736	dev->dev.dma_mask = &dev->dma_mask;
2737	dev->dev.dma_parms = &dev->dma_parms;
2738	dev->dev.coherent_dma_mask = `0xffffffffull`;
2739
2740	dma_set_max_seg_size(dev: &dev->dev, size: `65536`);
2741	dma_set_seg_boundary(dev: &dev->dev, mask: `0xffffffff`);
2742
2743	pcie_failed_link_retrain(dev);
2744
2745	/ Fix up broken headers /
2746	pci_fixup_device(pass: pci_fixup_header, dev);
2747
2748	pci_reassigndev_resource_alignment(dev);
2749
2750	dev->state_saved = false;
2751
2752	pci_init_capabilities(dev);
2753
2754	/*
2755	* Add the device to our list of discovered devices
2756	* and the bus list for fixup functions, etc.
2757	*/
2758	down_write(sem: &pci_bus_sem);
2759	list_add_tail(new: &dev->bus_list, head: &bus->devices);
2760	up_write(sem: &pci_bus_sem);
2761
2762	ret = pcibios_device_add(dev);
2763	WARN_ON(ret < `0`);
2764
2765	/ Set up MSI IRQ domain /
2766	pci_set_msi_domain(dev);
2767
2768	/ Notifier could use PCI capabilities /
2769	ret = device_add(dev: &dev->dev);
2770	WARN_ON(ret < `0`);
2771
2772	pci_npem_create(dev);
2773
2774	pci_doe_sysfs_init(pdev: dev);
2775	}
2776
2777	struct pci_dev pci_scan_single_device(struct* pci_bus bus, int* devfn)
2778	{
2779	struct pci_dev *dev;
2780
2781	dev = pci_get_slot(bus, devfn);
2782	if (dev) {
2783	pci_dev_put(dev);
2784	return dev;
2785	}
2786
2787	dev = pci_scan_device(bus, devfn);
2788	if (!dev)
2789	return NULL;
2790
2791	pci_device_add(dev, bus);
2792
2793	return dev;
2794	}
2795	EXPORT_SYMBOL(pci_scan_single_device);
2796
2797	static int next_ari_fn(struct pci_bus bus, struct* pci_dev dev, int* fn)
2798	{
2799	int pos;
2800	u16 cap = `0`;
2801	unsigned int next_fn;
2802
2803	if (!dev)
2804	return -ENODEV;
2805
2806	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2807	if (!pos)
2808	return -ENODEV;
2809
2810	pci_read_config_word(dev, where: pos + PCI_ARI_CAP, val: &cap);
2811	next_fn = PCI_ARI_CAP_NFN(cap);
2812	if (next_fn <= fn)
2813	return -ENODEV; / protect against malformed list /
2814
2815	return next_fn;
2816	}
2817
2818	static int next_fn(struct pci_bus bus, struct* pci_dev dev, int* fn)
2819	{
2820	if (pci_ari_enabled(bus))
2821	return next_ari_fn(bus, dev, fn);
2822
2823	if (fn >= `7`)
2824	return -ENODEV;
2825	/ only multifunction devices may have more functions /
2826	if (dev && !dev->multifunction)
2827	return -ENODEV;
2828
2829	return fn + `1`;
2830	}
2831
2832	static int only_one_child(struct pci_bus *bus)
2833	{
2834	struct pci_dev *bridge = bus->self;
2835
2836	/*
2837	* Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2838	* we scan for all possible devices, not just Device 0.
2839	*/
2840	if (pci_has_flag(flag: PCI_SCAN_ALL_PCIE_DEVS))
2841	return `0`;
2842
2843	/*
2844	* A PCIe Downstream Port normally leads to a Link with only Device
2845	* 0 on it (PCIe spec r3.1, sec 7.3.1). As an optimization, scan
2846	* only for Device 0 in that situation.
2847	*/
2848	if (bridge && pci_is_pcie(dev: bridge) && pcie_downstream_port(dev: bridge))
2849	return `1`;
2850
2851	return `0`;
2852	}
2853
2854	/**
2855	* pci_scan_slot - Scan a PCI slot on a bus for devices
2856	* @bus: PCI bus to scan
2857	* @devfn: slot number to scan (must have zero function)
2858	*
2859	* Scan a PCI slot on the specified PCI bus for devices, adding
2860	* discovered devices to the @bus->devices list. New devices
2861	* will not have is_added set.
2862	*
2863	* Returns the number of new devices found.
2864	*/
2865	int pci_scan_slot(struct pci_bus bus, int* devfn)
2866	{
2867	struct pci_dev *dev;
2868	int fn = `0`, nr = `0`;
2869
2870	if (only_one_child(bus) && (devfn > `0`))
2871	return `0`; / Already scanned the entire slot /
2872
2873	do {
2874	dev = pci_scan_single_device(bus, devfn + fn);
2875	if (dev) {
2876	if (!pci_dev_is_added(dev))
2877	nr++;
2878	if (fn > `0`)
2879	dev->multifunction = `1`;
2880	} else if (fn == `0`) {
2881	/*
2882	* Function 0 is required unless we are running on
2883	* a hypervisor that passes through individual PCI
2884	* functions.
2885	*/
2886	if (!hypervisor_isolated_pci_functions())
2887	break;
2888	}
2889	fn = next_fn(bus, dev, fn);
2890	} while (fn >= `0`);
2891
2892	/ Only one slot has PCIe device /
2893	if (bus->self && nr)
2894	pcie_aspm_init_link_state(pdev: bus->self);
2895
2896	return nr;
2897	}
2898	EXPORT_SYMBOL(pci_scan_slot);
2899
2900	static int pcie_find_smpss(struct pci_dev dev, void* *data)
2901	{
2902	u8 *smpss = data;
2903
2904	if (!pci_is_pcie(dev))
2905	return `0`;
2906
2907	/*
2908	* We don't have a way to change MPS settings on devices that have
2909	* drivers attached. A hot-added device might support only the minimum
2910	* MPS setting (MPS=128). Therefore, if the fabric contains a bridge
2911	* where devices may be hot-added, we limit the fabric MPS to 128 so
2912	* hot-added devices will work correctly.
2913	*
2914	* However, if we hot-add a device to a slot directly below a Root
2915	* Port, it's impossible for there to be other existing devices below
2916	* the port. We don't limit the MPS in this case because we can
2917	* reconfigure MPS on both the Root Port and the hot-added device,
2918	* and there are no other devices involved.
2919	*
2920	* Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2921	*/
2922	if (dev->is_hotplug_bridge &&
2923	pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2924	*smpss = `0`;
2925
2926	if (*smpss > dev->pcie_mpss)
2927	*smpss = dev->pcie_mpss;
2928
2929	return `0`;
2930	}
2931
2932	static void pcie_write_mps(struct pci_dev dev, int* mps)
2933	{
2934	int rc;
2935
2936	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2937	mps = `128` << dev->pcie_mpss;
2938
2939	if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2940	dev->bus->self)
2941
2942	/*
2943	* For "Performance", the assumption is made that
2944	* downstream communication will never be larger than
2945	* the MRRS. So, the MPS only needs to be configured
2946	* for the upstream communication. This being the case,
2947	* walk from the top down and set the MPS of the child
2948	* to that of the parent bus.
2949	*
2950	* Configure the device MPS with the smaller of the
2951	* device MPSS or the bridge MPS (which is assumed to be
2952	* properly configured at this point to the largest
2953	* allowable MPS based on its parent bus).
2954	*/
2955	mps = min(mps, pcie_get_mps(dev->bus->self));
2956	}
2957
2958	rc = pcie_set_mps(dev, mps);
2959	if (rc)
2960	pci_err(dev, "Failed attempting to set the MPS\n");
2961	}
2962
2963	static void pcie_write_mrrs(struct pci_dev *dev)
2964	{
2965	int rc, mrrs;
2966
2967	/*
2968	* In the "safe" case, do not configure the MRRS. There appear to be
2969	* issues with setting MRRS to 0 on a number of devices.
2970	*/
2971	if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2972	return;
2973
2974	/*
2975	* For max performance, the MRRS must be set to the largest supported
2976	* value. However, it cannot be configured larger than the MPS the
2977	* device or the bus can support. This should already be properly
2978	* configured by a prior call to pcie_write_mps().
2979	*/
2980	mrrs = pcie_get_mps(dev);
2981
2982	/*
2983	* MRRS is a R/W register. Invalid values can be written, but a
2984	* subsequent read will verify if the value is acceptable or not.
2985	* If the MRRS value provided is not acceptable (e.g., too large),
2986	* shrink the value until it is acceptable to the HW.
2987	*/
2988	while (mrrs != pcie_get_readrq(dev) && mrrs >= `128`) {
2989	rc = pcie_set_readrq(dev, rq: mrrs);
2990	if (!rc)
2991	break;
2992
2993	pci_warn(dev, "Failed attempting to set the MRRS\n");
2994	mrrs /= `2`;
2995	}
2996
2997	if (mrrs < `128`)
2998	pci_err(dev, "MRRS was unable to be configured with a safe value. If problems are experienced, try running with pci=pcie_bus_safe\n");
2999	}
3000
3001	static int pcie_bus_configure_set(struct pci_dev dev, void* *data)
3002	{
3003	int mps, orig_mps;
3004
3005	if (!pci_is_pcie(dev))
3006	return `0`;
3007
3008	if (pcie_bus_config == PCIE_BUS_TUNE_OFF \|\|
3009	pcie_bus_config == PCIE_BUS_DEFAULT)
3010	return `0`;
3011
3012	mps = `128` << (u8 )data;
3013	orig_mps = pcie_get_mps(dev);
3014
3015	pcie_write_mps(dev, mps);
3016	pcie_write_mrrs(dev);
3017
3018	pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
3019	pcie_get_mps(dev), `128` << dev->pcie_mpss,
3020	orig_mps, pcie_get_readrq(dev));
3021
3022	return `0`;
3023	}
3024
3025	/*
3026	* pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
3027	* parents then children fashion. If this changes, then this code will not
3028	* work as designed.
3029	*/
3030	void pcie_bus_configure_settings(struct pci_bus *bus)
3031	{
3032	u8 smpss = `0`;
3033
3034	if (!bus->self)
3035	return;
3036
3037	if (!pci_is_pcie(dev: bus->self))
3038	return;
3039
3040	/*
3041	* FIXME - Peer to peer DMA is possible, though the endpoint would need
3042	* to be aware of the MPS of the destination. To work around this,
3043	* simply force the MPS of the entire system to the smallest possible.
3044	*/
3045	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
3046	smpss = `0`;
3047
3048	if (pcie_bus_config == PCIE_BUS_SAFE) {
3049	smpss = bus->self->pcie_mpss;
3050
3051	pcie_find_smpss(dev: bus->self, data: &smpss);
3052	pci_walk_bus(top: bus, cb: pcie_find_smpss, userdata: &smpss);
3053	}
3054
3055	pcie_bus_configure_set(dev: bus->self, data: &smpss);
3056	pci_walk_bus(top: bus, cb: pcie_bus_configure_set, userdata: &smpss);
3057	}
3058	EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
3059
3060	/*
3061	* Called after each bus is probed, but before its children are examined. This
3062	* is marked as __weak because multiple architectures define it.
3063	*/
3064	void __weak pcibios_fixup_bus(struct pci_bus *bus)
3065	{
3066	/ nothing to do, expected to be removed in the future /
3067	}
3068
3069	/**
3070	* pci_scan_child_bus_extend() - Scan devices below a bus
3071	* @bus: Bus to scan for devices
3072	* @available_buses: Total number of buses available (%0 does not try to
3073	* extend beyond the minimal)
3074	*
3075	* Scans devices below @bus including subordinate buses. Returns new
3076	* subordinate number including all the found devices. Passing
3077	* @available_buses causes the remaining bus space to be distributed
3078	* equally between hotplug-capable bridges to allow future extension of the
3079	* hierarchy.
3080	*/
3081	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
3082	unsigned int available_buses)
3083	{
3084	unsigned int used_buses, normal_bridges = `0`, hotplug_bridges = `0`;
3085	unsigned int start = bus->busn_res.start;
3086	unsigned int devnr, cmax, max = start;
3087	struct pci_dev *dev;
3088
3089	dev_dbg(&bus->dev, "scanning bus\n");
3090
3091	/ Go find them, Rover! /
3092	for (devnr = `0`; devnr < PCI_MAX_NR_DEVS; devnr++)
3093	pci_scan_slot(bus, PCI_DEVFN(devnr, `0`));
3094
3095	/ Reserve buses for SR-IOV capability /
3096	used_buses = pci_iov_bus_range(bus);
3097	max += used_buses;
3098
3099	/*
3100	* After performing arch-dependent fixup of the bus, look behind
3101	* all PCI-to-PCI bridges on this bus.
3102	*/
3103	if (!bus->is_added) {
3104	dev_dbg(&bus->dev, "fixups for bus\n");
3105	pcibios_fixup_bus(bus);
3106	bus->is_added = `1`;
3107	}
3108
3109	/*
3110	* Calculate how many hotplug bridges and normal bridges there
3111	* are on this bus. We will distribute the additional available
3112	* buses between hotplug bridges.
3113	*/
3114	for_each_pci_bridge(dev, bus) {
3115	if (dev->is_hotplug_bridge)
3116	hotplug_bridges++;
3117	else
3118	normal_bridges++;
3119	}
3120
3121	/*
3122	* Scan bridges that are already configured. We don't touch them
3123	* unless they are misconfigured (which will be done in the second
3124	* scan below).
3125	*/
3126	for_each_pci_bridge(dev, bus) {
3127	cmax = max;
3128	max = pci_scan_bridge_extend(bus, dev, max, available_buses: `0`, pass: `0`);
3129
3130	/*
3131	* Reserve one bus for each bridge now to avoid extending
3132	* hotplug bridges too much during the second scan below.
3133	*/
3134	used_buses++;
3135	if (max - cmax > `1`)
3136	used_buses += max - cmax - `1`;
3137	}
3138
3139	/ Scan bridges that need to be reconfigured /
3140	for_each_pci_bridge(dev, bus) {
3141	unsigned int buses = `0`;
3142
3143	if (!hotplug_bridges && normal_bridges == `1`) {
3144	/*
3145	* There is only one bridge on the bus (upstream
3146	* port) so it gets all available buses which it
3147	* can then distribute to the possible hotplug
3148	* bridges below.
3149	*/
3150	buses = available_buses;
3151	} else if (dev->is_hotplug_bridge) {
3152	/*
3153	* Distribute the extra buses between hotplug
3154	* bridges if any.
3155	*/
3156	buses = available_buses / hotplug_bridges;
3157	buses = min(buses, available_buses - used_buses + `1`);
3158	}
3159
3160	cmax = max;
3161	max = pci_scan_bridge_extend(bus, dev, max: cmax, available_buses: buses, pass: `1`);
3162	/ One bus is already accounted so don't add it again /
3163	if (max - cmax > `1`)
3164	used_buses += max - cmax - `1`;
3165	}
3166
3167	/*
3168	* Make sure a hotplug bridge has at least the minimum requested
3169	* number of buses but allow it to grow up to the maximum available
3170	* bus number if there is room.
3171	*/
3172	if (bus->self && bus->self->is_hotplug_bridge) {
3173	used_buses = max_t(unsigned int, available_buses,
3174	pci_hotplug_bus_size - `1`);
3175	if (max - start < used_buses) {
3176	max = start + used_buses;
3177
3178	/ Do not allocate more buses than we have room left /
3179	if (max > bus->busn_res.end)
3180	max = bus->busn_res.end;
3181
3182	dev_dbg(&bus->dev, "%pR extended by %#02x\n",
3183	&bus->busn_res, max - start);
3184	}
3185	}
3186
3187	/*
3188	* We've scanned the bus and so we know all about what's on
3189	* the other side of any bridges that may be on this bus plus
3190	* any devices.
3191	*
3192	* Return how far we've got finding sub-buses.
3193	*/
3194	dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
3195	return max;
3196	}
3197
3198	/**
3199	* pci_scan_child_bus() - Scan devices below a bus
3200	* @bus: Bus to scan for devices
3201	*
3202	* Scans devices below @bus including subordinate buses. Returns new
3203	* subordinate number including all the found devices.
3204	*/
3205	unsigned int pci_scan_child_bus(struct pci_bus *bus)
3206	{
3207	return pci_scan_child_bus_extend(bus, available_buses: `0`);
3208	}
3209	EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3210
3211	/**
3212	* pcibios_root_bridge_prepare - Platform-specific host bridge setup
3213	* @bridge: Host bridge to set up
3214	*
3215	* Default empty implementation. Replace with an architecture-specific setup
3216	* routine, if necessary.
3217	*/
3218	int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3219	{
3220	return `0`;
3221	}
3222
3223	void __weak pcibios_add_bus(struct pci_bus *bus)
3224	{
3225	}
3226
3227	void __weak pcibios_remove_bus(struct pci_bus *bus)
3228	{
3229	}
3230
3231	struct pci_bus pci_create_root_bus(struct* device parent, int* bus,
3232	struct pci_ops ops, void* sysdata, struct* list_head *resources)
3233	{
3234	int error;
3235	struct pci_host_bridge *bridge;
3236
3237	bridge = pci_alloc_host_bridge(`0`);
3238	if (!bridge)
3239	return NULL;
3240
3241	bridge->dev.parent = parent;
3242
3243	list_splice_init(list: resources, head: &bridge->windows);
3244	bridge->sysdata = sysdata;
3245	bridge->busnr = bus;
3246	bridge->ops = ops;
3247
3248	error = pci_register_host_bridge(bridge);
3249	if (error < `0`)
3250	goto err_out;
3251
3252	return bridge->bus;
3253
3254	err_out:
3255	put_device(dev: &bridge->dev);
3256	return NULL;
3257	}
3258	EXPORT_SYMBOL_GPL(pci_create_root_bus);
3259
3260	int pci_host_probe(struct pci_host_bridge *bridge)
3261	{
3262	struct pci_bus bus, child;
3263	int ret;
3264
3265	pci_lock_rescan_remove();
3266	ret = pci_scan_root_bus_bridge(bridge);
3267	pci_unlock_rescan_remove();
3268	if (ret < `0`) {
3269	dev_err(bridge->dev.parent, "Scanning root bridge failed");
3270	return ret;
3271	}
3272
3273	bus = bridge->bus;
3274
3275	/ If we must preserve the resource configuration, claim now /
3276	if (bridge->preserve_config)
3277	pci_bus_claim_resources(bus);
3278
3279	/*
3280	* Assign whatever was left unassigned. If we didn't claim above,
3281	* this will reassign everything.
3282	*/
3283	pci_assign_unassigned_root_bus_resources(bus);
3284
3285	list_for_each_entry(child, &bus->children, node)
3286	pcie_bus_configure_settings(child);
3287
3288	pci_lock_rescan_remove();
3289	pci_bus_add_devices(bus);
3290	pci_unlock_rescan_remove();
3291
3292	/*
3293	* Ensure pm_runtime_enable() is called for the controller drivers
3294	* before calling pci_host_probe(). The PM framework expects that
3295	* if the parent device supports runtime PM, it will be enabled
3296	* before child runtime PM is enabled.
3297	*/
3298	pm_runtime_set_active(dev: &bridge->dev);
3299	pm_runtime_no_callbacks(dev: &bridge->dev);
3300	devm_pm_runtime_enable(dev: &bridge->dev);
3301
3302	return `0`;
3303	}
3304	EXPORT_SYMBOL_GPL(pci_host_probe);
3305
3306	int pci_bus_insert_busn_res(struct pci_bus b, int* bus, int bus_max)
3307	{
3308	struct resource *res = &b->busn_res;
3309	struct resource parent_res, conflict;
3310
3311	res->start = bus;
3312	res->end = bus_max;
3313	res->flags = IORESOURCE_BUS;
3314
3315	if (!pci_is_root_bus(pbus: b))
3316	parent_res = &b->parent->busn_res;
3317	else {
3318	parent_res = get_pci_domain_busn_res(domain_nr: pci_domain_nr(bus: b));
3319	res->flags \|= IORESOURCE_PCI_FIXED;
3320	}
3321
3322	conflict = request_resource_conflict(root: parent_res, new: res);
3323
3324	if (conflict)
3325	dev_info(&b->dev,
3326	"busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3327	res, pci_is_root_bus(b) ? "domain " : "",
3328	parent_res, conflict->name, conflict);
3329
3330	return conflict == NULL;
3331	}
3332
3333	int pci_bus_update_busn_res_end(struct pci_bus b, int* bus_max)
3334	{
3335	struct resource *res = &b->busn_res;
3336	struct resource old_res = *res;
3337	resource_size_t size;
3338	int ret;
3339
3340	if (res->start > bus_max)
3341	return -EINVAL;
3342
3343	size = bus_max - res->start + `1`;
3344	ret = adjust_resource(res, start: res->start, size);
3345	dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3346	&old_res, ret ? "can not be" : "is", bus_max);
3347
3348	if (!ret && !res->parent)
3349	pci_bus_insert_busn_res(b, bus: res->start, bus_max: res->end);
3350
3351	return ret;
3352	}
3353
3354	void pci_bus_release_busn_res(struct pci_bus *b)
3355	{
3356	struct resource *res = &b->busn_res;
3357	int ret;
3358
3359	if (!res->flags \|\| !res->parent)
3360	return;
3361
3362	ret = release_resource(new: res);
3363	dev_info(&b->dev, "busn_res: %pR %s released\n",
3364	res, ret ? "can not be" : "is");
3365	}
3366
3367	int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3368	{
3369	struct resource_entry *window;
3370	bool found = false;
3371	struct pci_bus *b;
3372	int max, bus, ret;
3373
3374	if (!bridge)
3375	return -EINVAL;
3376
3377	resource_list_for_each_entry(window, &bridge->windows)
3378	if (window->res->flags & IORESOURCE_BUS) {
3379	bridge->busnr = window->res->start;
3380	found = true;
3381	break;
3382	}
3383
3384	ret = pci_register_host_bridge(bridge);
3385	if (ret < `0`)
3386	return ret;
3387
3388	b = bridge->bus;
3389	bus = bridge->busnr;
3390
3391	if (!found) {
3392	dev_info(&b->dev,
3393	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3394	bus);
3395	pci_bus_insert_busn_res(b, bus, bus_max: `255`);
3396	}
3397
3398	max = pci_scan_child_bus(b);
3399
3400	if (!found)
3401	pci_bus_update_busn_res_end(b, bus_max: max);
3402
3403	return `0`;
3404	}
3405	EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3406
3407	struct pci_bus pci_scan_root_bus(struct* device parent, int* bus,
3408	struct pci_ops ops, void* sysdata, struct* list_head *resources)
3409	{
3410	struct resource_entry *window;
3411	bool found = false;
3412	struct pci_bus *b;
3413	int max;
3414
3415	resource_list_for_each_entry(window, resources)
3416	if (window->res->flags & IORESOURCE_BUS) {
3417	found = true;
3418	break;
3419	}
3420
3421	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3422	if (!b)
3423	return NULL;
3424
3425	if (!found) {
3426	dev_info(&b->dev,
3427	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3428	bus);
3429	pci_bus_insert_busn_res(b, bus, bus_max: `255`);
3430	}
3431
3432	max = pci_scan_child_bus(b);
3433
3434	if (!found)
3435	pci_bus_update_busn_res_end(b, bus_max: max);
3436
3437	return b;
3438	}
3439	EXPORT_SYMBOL(pci_scan_root_bus);
3440
3441	struct pci_bus pci_scan_bus(int* bus, struct pci_ops *ops,
3442	void *sysdata)
3443	{
3444	LIST_HEAD(resources);
3445	struct pci_bus *b;
3446
3447	pci_add_resource(resources: &resources, res: &ioport_resource);
3448	pci_add_resource(resources: &resources, res: &iomem_resource);
3449	pci_add_resource(resources: &resources, res: &busn_resource);
3450	b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3451	if (b) {
3452	pci_scan_child_bus(b);
3453	} else {
3454	pci_free_resource_list(resources: &resources);
3455	}
3456	return b;
3457	}
3458	EXPORT_SYMBOL(pci_scan_bus);
3459
3460	/**
3461	* pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3462	* @bridge: PCI bridge for the bus to scan
3463	*
3464	* Scan a PCI bus and child buses for new devices, add them,
3465	* and enable them, resizing bridge mmio/io resource if necessary
3466	* and possible. The caller must ensure the child devices are already
3467	* removed for resizing to occur.
3468	*
3469	* Returns the max number of subordinate bus discovered.
3470	*/
3471	unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3472	{
3473	unsigned int max;
3474	struct pci_bus *bus = bridge->subordinate;
3475
3476	max = pci_scan_child_bus(bus);
3477
3478	pci_assign_unassigned_bridge_resources(bridge);
3479
3480	pci_bus_add_devices(bus);
3481
3482	return max;
3483	}
3484
3485	/**
3486	* pci_rescan_bus - Scan a PCI bus for devices
3487	* @bus: PCI bus to scan
3488	*
3489	* Scan a PCI bus and child buses for new devices, add them,
3490	* and enable them.
3491	*
3492	* Returns the max number of subordinate bus discovered.
3493	*/
3494	unsigned int pci_rescan_bus(struct pci_bus *bus)
3495	{
3496	unsigned int max;
3497
3498	max = pci_scan_child_bus(bus);
3499	pci_assign_unassigned_bus_resources(bus);
3500	pci_bus_add_devices(bus);
3501
3502	return max;
3503	}
3504	EXPORT_SYMBOL_GPL(pci_rescan_bus);
3505
3506	/*
3507	* pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3508	* routines should always be executed under this mutex.
3509	*/
3510	DEFINE_MUTEX(pci_rescan_remove_lock);
3511
3512	void pci_lock_rescan_remove(void)
3513	{
3514	mutex_lock(lock: &pci_rescan_remove_lock);
3515	}
3516	EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3517
3518	void pci_unlock_rescan_remove(void)
3519	{
3520	mutex_unlock(lock: &pci_rescan_remove_lock);
3521	}
3522	EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3523
3524	static int __init pci_sort_bf_cmp(const struct device *d_a,
3525	const struct device *d_b)
3526	{
3527	const struct pci_dev *a = to_pci_dev(d_a);
3528	const struct pci_dev *b = to_pci_dev(d_b);
3529
3530	if (pci_domain_nr(bus: a->bus) < pci_domain_nr(bus: b->bus)) return -`1`;
3531	else if (pci_domain_nr(bus: a->bus) > pci_domain_nr(bus: b->bus)) return `1`;
3532
3533	if (a->bus->number < b->bus->number) return -`1`;
3534	else if (a->bus->number > b->bus->number) return `1`;
3535
3536	if (a->devfn < b->devfn) return -`1`;
3537	else if (a->devfn > b->devfn) return `1`;
3538
3539	return `0`;
3540	}
3541
3542	void __init pci_sort_breadthfirst(void)
3543	{
3544	bus_sort_breadthfirst(bus: &pci_bus_type, compare: &pci_sort_bf_cmp);
3545	}
3546
3547	int pci_hp_add_bridge(struct pci_dev *dev)
3548	{
3549	struct pci_bus *parent = dev->bus;
3550	int busnr, start = parent->busn_res.start;
3551	unsigned int available_buses = `0`;
3552	int end = parent->busn_res.end;
3553
3554	for (busnr = start; busnr <= end; busnr++) {
3555	if (!pci_find_bus(domain: pci_domain_nr(bus: parent), busnr))
3556	break;
3557	}
3558	if (busnr-- > end) {
3559	pci_err(dev, "No bus number available for hot-added bridge\n");
3560	return -`1`;
3561	}
3562
3563	/ Scan bridges that are already configured /
3564	busnr = pci_scan_bridge(parent, dev, busnr, `0`);
3565
3566	/*
3567	* Distribute the available bus numbers between hotplug-capable
3568	* bridges to make extending the chain later possible.
3569	*/
3570	available_buses = end - busnr;
3571
3572	/ Scan bridges that need to be reconfigured /
3573	pci_scan_bridge_extend(bus: parent, dev, max: busnr, available_buses, pass: `1`);
3574
3575	if (!dev->subordinate)
3576	return -`1`;
3577
3578	return `0`;
3579	}
3580	EXPORT_SYMBOL_GPL(pci_hp_add_bridge);
3581

Browse the source code of Linux/drivers/pci/probe.c