1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Purpose: PCI Express Port Bus Driver
4 *
5 * Copyright (C) 2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
8
9#include <linux/bitfield.h>
10#include <linux/dmi.h>
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/pci.h>
14#include <linux/kernel.h>
15#include <linux/errno.h>
16#include <linux/pm.h>
17#include <linux/pm_runtime.h>
18#include <linux/string.h>
19#include <linux/slab.h>
20#include <linux/aer.h>
21
22#include "../pci.h"
23#include "portdrv.h"
24
25/*
26 * The PCIe Capability Interrupt Message Number (PCIe r3.1, sec 7.8.2) must
27 * be one of the first 32 MSI-X entries. Per PCI r3.0, sec 6.8.3.1, MSI
28 * supports a maximum of 32 vectors per function.
29 */
30#define PCIE_PORT_MAX_MSI_ENTRIES 32
31
32#define get_descriptor_id(type, service) (((type - 4) << 8) | service)
33
34struct portdrv_service_data {
35 struct pcie_port_service_driver *drv;
36 struct device *dev;
37 u32 service;
38};
39
40/**
41 * release_pcie_device - free PCI Express port service device structure
42 * @dev: Port service device to release
43 *
44 * Invoked automatically when device is being removed in response to
45 * device_unregister(dev). Release all resources being claimed.
46 */
47static void release_pcie_device(struct device *dev)
48{
49 kfree(to_pcie_device(dev));
50}
51
52/*
53 * Fill in *pme, *aer, *dpc with the relevant Interrupt Message Numbers if
54 * services are enabled in "mask". Return the number of MSI/MSI-X vectors
55 * required to accommodate the largest Message Number.
56 */
57static int pcie_message_numbers(struct pci_dev *dev, int mask,
58 u32 *pme, u32 *aer, u32 *dpc)
59{
60 u32 nvec = 0, pos;
61 u16 reg16;
62
63 /*
64 * The Interrupt Message Number indicates which vector is used, i.e.,
65 * the MSI-X table entry or the MSI offset between the base Message
66 * Data and the generated interrupt message. See PCIe r3.1, sec
67 * 7.8.2, 7.10.10, 7.31.2.
68 */
69
70 if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
71 PCIE_PORT_SERVICE_BWCTRL)) {
72 pcie_capability_read_word(dev, PCI_EXP_FLAGS, val: &reg16);
73 *pme = FIELD_GET(PCI_EXP_FLAGS_IRQ, reg16);
74 nvec = *pme + 1;
75 }
76
77#ifdef CONFIG_PCIEAER
78 if (mask & PCIE_PORT_SERVICE_AER) {
79 u32 reg32;
80
81 pos = dev->aer_cap;
82 if (pos) {
83 pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS,
84 &reg32);
85 *aer = FIELD_GET(PCI_ERR_ROOT_AER_IRQ, reg32);
86 nvec = max(nvec, *aer + 1);
87 }
88 }
89#endif
90
91 if (mask & PCIE_PORT_SERVICE_DPC) {
92 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC);
93 if (pos) {
94 pci_read_config_word(dev, where: pos + PCI_EXP_DPC_CAP,
95 val: &reg16);
96 *dpc = FIELD_GET(PCI_EXP_DPC_IRQ, reg16);
97 nvec = max(nvec, *dpc + 1);
98 }
99 }
100
101 return nvec;
102}
103
104/**
105 * pcie_port_enable_irq_vec - try to set up MSI-X or MSI as interrupt mode
106 * for given port
107 * @dev: PCI Express port to handle
108 * @irqs: Array of interrupt vectors to populate
109 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
110 *
111 * Return value: 0 on success, error code on failure
112 */
113static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
114{
115 int nr_entries, nvec, pcie_irq;
116 u32 pme = 0, aer = 0, dpc = 0;
117
118 /* Allocate the maximum possible number of MSI/MSI-X vectors */
119 nr_entries = pci_alloc_irq_vectors(dev, min_vecs: 1, PCIE_PORT_MAX_MSI_ENTRIES,
120 PCI_IRQ_MSIX | PCI_IRQ_MSI);
121 if (nr_entries < 0)
122 return nr_entries;
123
124 /* See how many and which Interrupt Message Numbers we actually use */
125 nvec = pcie_message_numbers(dev, mask, pme: &pme, aer: &aer, dpc: &dpc);
126 if (nvec > nr_entries) {
127 pci_free_irq_vectors(dev);
128 return -EIO;
129 }
130
131 /*
132 * If we allocated more than we need, free them and reallocate fewer.
133 *
134 * Reallocating may change the specific vectors we get, so
135 * pci_irq_vector() must be done *after* the reallocation.
136 *
137 * If we're using MSI, hardware is *allowed* to change the Interrupt
138 * Message Numbers when we free and reallocate the vectors, but we
139 * assume it won't because we allocate enough vectors for the
140 * biggest Message Number we found.
141 */
142 if (nvec != nr_entries) {
143 pci_free_irq_vectors(dev);
144
145 nr_entries = pci_alloc_irq_vectors(dev, min_vecs: nvec, max_vecs: nvec,
146 PCI_IRQ_MSIX | PCI_IRQ_MSI);
147 if (nr_entries < 0)
148 return nr_entries;
149 }
150
151 /* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */
152 if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
153 PCIE_PORT_SERVICE_BWCTRL)) {
154 pcie_irq = pci_irq_vector(dev, nr: pme);
155 irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq;
156 irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq;
157 irqs[PCIE_PORT_SERVICE_BWCTRL_SHIFT] = pcie_irq;
158 }
159
160 if (mask & PCIE_PORT_SERVICE_AER)
161 irqs[PCIE_PORT_SERVICE_AER_SHIFT] = pci_irq_vector(dev, nr: aer);
162
163 if (mask & PCIE_PORT_SERVICE_DPC)
164 irqs[PCIE_PORT_SERVICE_DPC_SHIFT] = pci_irq_vector(dev, nr: dpc);
165
166 return 0;
167}
168
169/**
170 * pcie_init_service_irqs - initialize irqs for PCI Express port services
171 * @dev: PCI Express port to handle
172 * @irqs: Array of irqs to populate
173 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
174 *
175 * Return value: Interrupt mode associated with the port
176 */
177static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask)
178{
179 int ret, i;
180
181 for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
182 irqs[i] = -1;
183
184 /*
185 * If we support PME but can't use MSI/MSI-X for it, we have to
186 * fall back to INTx or other interrupts, e.g., a system shared
187 * interrupt.
188 */
189 if ((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi())
190 goto intx_irq;
191
192 /* Try to use MSI-X or MSI if supported */
193 if (pcie_port_enable_irq_vec(dev, irqs, mask) == 0)
194 return 0;
195
196intx_irq:
197 /* fall back to INTX IRQ */
198 ret = pci_alloc_irq_vectors(dev, min_vecs: 1, max_vecs: 1, PCI_IRQ_INTX);
199 if (ret < 0)
200 return -ENODEV;
201
202 for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
203 irqs[i] = pci_irq_vector(dev, nr: 0);
204
205 return 0;
206}
207
208/**
209 * get_port_device_capability - discover capabilities of a PCI Express port
210 * @dev: PCI Express port to examine
211 *
212 * The capabilities are read from the port's PCI Express configuration registers
213 * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and
214 * 7.9 - 7.11.
215 *
216 * Return value: Bitmask of discovered port capabilities
217 */
218static int get_port_device_capability(struct pci_dev *dev)
219{
220 struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus);
221 int services = 0;
222
223 if (dev->is_pciehp &&
224 (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
225 pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) &&
226 (pcie_ports_native || host->native_pcie_hotplug)) {
227 services |= PCIE_PORT_SERVICE_HP;
228
229 /*
230 * Disable hot-plug interrupts in case they have been enabled
231 * by the BIOS and the hot-plug service driver won't be loaded
232 * to handle them.
233 */
234 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
235 pcie_capability_clear_word(dev, PCI_EXP_SLTCTL,
236 PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE);
237 }
238
239#ifdef CONFIG_PCIEAER
240 if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
241 pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) &&
242 dev->aer_cap && pci_aer_available() &&
243 (pcie_ports_native || host->native_aer))
244 services |= PCIE_PORT_SERVICE_AER;
245#endif
246
247 /* Root Ports and Root Complex Event Collectors may generate PMEs */
248 if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
249 pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) &&
250 (pcie_ports_native || host->native_pme)) {
251 services |= PCIE_PORT_SERVICE_PME;
252
253 /*
254 * Disable PME interrupt on this port in case it's been enabled
255 * by the BIOS (the PME service driver will enable it when
256 * necessary).
257 */
258 pcie_pme_interrupt_enable(dev, enable: false);
259 }
260
261 /*
262 * With dpc-native, allow Linux to use DPC even if it doesn't have
263 * permission to use AER.
264 */
265 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) &&
266 pci_aer_available() &&
267 (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER)))
268 services |= PCIE_PORT_SERVICE_DPC;
269
270 /* Enable bandwidth control if more than one speed is supported. */
271 if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
272 pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
273 u32 linkcap;
274
275 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, val: &linkcap);
276 if (linkcap & PCI_EXP_LNKCAP_LBNC &&
277 hweight8(dev->supported_speeds) > 1)
278 services |= PCIE_PORT_SERVICE_BWCTRL;
279 }
280
281 return services;
282}
283
284/**
285 * pcie_device_init - allocate and initialize PCI Express port service device
286 * @pdev: PCI Express port to associate the service device with
287 * @service: Type of service to associate with the service device
288 * @irq: Interrupt vector to associate with the service device
289 */
290static int pcie_device_init(struct pci_dev *pdev, int service, int irq)
291{
292 int retval;
293 struct pcie_device *pcie;
294 struct device *device;
295
296 pcie = kzalloc(sizeof(*pcie), GFP_KERNEL);
297 if (!pcie)
298 return -ENOMEM;
299 pcie->port = pdev;
300 pcie->irq = irq;
301 pcie->service = service;
302
303 /* Initialize generic device interface */
304 device = &pcie->device;
305 device->bus = &pcie_port_bus_type;
306 device->release = release_pcie_device; /* callback to free pcie dev */
307 dev_set_name(dev: device, name: "%s:pcie%03x",
308 pci_name(pdev),
309 get_descriptor_id(pci_pcie_type(pdev), service));
310 device->parent = &pdev->dev;
311 device_enable_async_suspend(dev: device);
312
313 retval = device_register(dev: device);
314 if (retval) {
315 put_device(dev: device);
316 return retval;
317 }
318
319 pm_runtime_no_callbacks(dev: device);
320
321 return 0;
322}
323
324/**
325 * pcie_port_device_register - register PCI Express port
326 * @dev: PCI Express port to register
327 *
328 * Allocate the port extension structure and register services associated with
329 * the port.
330 */
331static int pcie_port_device_register(struct pci_dev *dev)
332{
333 int status, capabilities, i, nr_service;
334 int irqs[PCIE_PORT_DEVICE_MAXSERVICES];
335
336 /* Enable PCI Express port device */
337 status = pci_enable_device(dev);
338 if (status)
339 return status;
340
341 /* Get and check PCI Express port services */
342 capabilities = get_port_device_capability(dev);
343 if (!capabilities)
344 return 0;
345
346 pci_set_master(dev);
347 /*
348 * Initialize service irqs. Don't use service devices that
349 * require interrupts if there is no way to generate them.
350 * However, some drivers may have a polling mode (e.g. pciehp_poll_mode)
351 * that can be used in the absence of irqs. Allow them to determine
352 * if that is to be used.
353 */
354 status = pcie_init_service_irqs(dev, irqs, mask: capabilities);
355 if (status) {
356 capabilities &= PCIE_PORT_SERVICE_HP;
357 if (!capabilities)
358 goto error_disable;
359 }
360
361 /* Allocate child services if any */
362 status = -ENODEV;
363 nr_service = 0;
364 for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) {
365 int service = 1 << i;
366 if (!(capabilities & service))
367 continue;
368 if (!pcie_device_init(pdev: dev, service, irq: irqs[i]))
369 nr_service++;
370 }
371 if (!nr_service)
372 goto error_cleanup_irqs;
373
374 return 0;
375
376error_cleanup_irqs:
377 pci_free_irq_vectors(dev);
378error_disable:
379 pci_disable_device(dev);
380 return status;
381}
382
383typedef int (*pcie_callback_t)(struct pcie_device *);
384
385static int pcie_port_device_iter(struct device *dev, void *data)
386{
387 struct pcie_port_service_driver *service_driver;
388 size_t offset = *(size_t *)data;
389 pcie_callback_t cb;
390
391 if ((dev->bus == &pcie_port_bus_type) && dev->driver) {
392 service_driver = to_service_driver(dev->driver);
393 cb = *(pcie_callback_t *)((void *)service_driver + offset);
394 if (cb)
395 return cb(to_pcie_device(dev));
396 }
397 return 0;
398}
399
400#ifdef CONFIG_PM
401/**
402 * pcie_port_device_suspend - suspend port services associated with a PCIe port
403 * @dev: PCI Express port to handle
404 */
405static int pcie_port_device_suspend(struct device *dev)
406{
407 size_t off = offsetof(struct pcie_port_service_driver, suspend);
408 return device_for_each_child(parent: dev, data: &off, fn: pcie_port_device_iter);
409}
410
411static int pcie_port_device_resume_noirq(struct device *dev)
412{
413 size_t off = offsetof(struct pcie_port_service_driver, resume_noirq);
414 return device_for_each_child(parent: dev, data: &off, fn: pcie_port_device_iter);
415}
416
417/**
418 * pcie_port_device_resume - resume port services associated with a PCIe port
419 * @dev: PCI Express port to handle
420 */
421static int pcie_port_device_resume(struct device *dev)
422{
423 size_t off = offsetof(struct pcie_port_service_driver, resume);
424 return device_for_each_child(parent: dev, data: &off, fn: pcie_port_device_iter);
425}
426
427/**
428 * pcie_port_device_runtime_suspend - runtime suspend port services
429 * @dev: PCI Express port to handle
430 */
431static int pcie_port_device_runtime_suspend(struct device *dev)
432{
433 size_t off = offsetof(struct pcie_port_service_driver, runtime_suspend);
434 return device_for_each_child(parent: dev, data: &off, fn: pcie_port_device_iter);
435}
436
437/**
438 * pcie_port_device_runtime_resume - runtime resume port services
439 * @dev: PCI Express port to handle
440 */
441static int pcie_port_device_runtime_resume(struct device *dev)
442{
443 size_t off = offsetof(struct pcie_port_service_driver, runtime_resume);
444 return device_for_each_child(parent: dev, data: &off, fn: pcie_port_device_iter);
445}
446#endif /* PM */
447
448static int remove_iter(struct device *dev, void *data)
449{
450 if (dev->bus == &pcie_port_bus_type)
451 device_unregister(dev);
452 return 0;
453}
454
455static int find_service_iter(struct device *device, void *data)
456{
457 struct pcie_port_service_driver *service_driver;
458 struct portdrv_service_data *pdrvs;
459 u32 service;
460
461 pdrvs = (struct portdrv_service_data *) data;
462 service = pdrvs->service;
463
464 if (device->bus == &pcie_port_bus_type && device->driver) {
465 service_driver = to_service_driver(device->driver);
466 if (service_driver->service == service) {
467 pdrvs->drv = service_driver;
468 pdrvs->dev = device;
469 return 1;
470 }
471 }
472
473 return 0;
474}
475
476/**
477 * pcie_port_find_device - find the struct device
478 * @dev: PCI Express port the service is associated with
479 * @service: For the service to find
480 *
481 * Find the struct device associated with given service on a pci_dev
482 */
483struct device *pcie_port_find_device(struct pci_dev *dev,
484 u32 service)
485{
486 struct device *device;
487 struct portdrv_service_data pdrvs;
488
489 pdrvs.dev = NULL;
490 pdrvs.service = service;
491 device_for_each_child(parent: &dev->dev, data: &pdrvs, fn: find_service_iter);
492
493 device = pdrvs.dev;
494 return device;
495}
496EXPORT_SYMBOL_GPL(pcie_port_find_device);
497
498/**
499 * pcie_port_device_remove - unregister PCI Express port service devices
500 * @dev: PCI Express port the service devices to unregister are associated with
501 *
502 * Remove PCI Express port service devices associated with given port and
503 * disable MSI-X or MSI for the port.
504 */
505static void pcie_port_device_remove(struct pci_dev *dev)
506{
507 device_for_each_child(parent: &dev->dev, NULL, fn: remove_iter);
508 pci_free_irq_vectors(dev);
509}
510
511/**
512 * pcie_port_probe_service - probe driver for given PCI Express port service
513 * @dev: PCI Express port service device to probe against
514 *
515 * If PCI Express port service driver is registered with
516 * pcie_port_service_register(), this function will be called by the driver core
517 * whenever match is found between the driver and a port service device.
518 */
519static int pcie_port_probe_service(struct device *dev)
520{
521 struct pcie_device *pciedev;
522 struct pcie_port_service_driver *driver;
523 int status;
524
525 if (!dev || !dev->driver)
526 return -ENODEV;
527
528 driver = to_service_driver(dev->driver);
529 if (!driver || !driver->probe)
530 return -ENODEV;
531
532 pciedev = to_pcie_device(dev);
533 status = driver->probe(pciedev);
534 if (status)
535 return status;
536
537 get_device(dev);
538 return 0;
539}
540
541/**
542 * pcie_port_remove_service - detach driver from given PCI Express port service
543 * @dev: PCI Express port service device to handle
544 *
545 * If PCI Express port service driver is registered with
546 * pcie_port_service_register(), this function will be called by the driver core
547 * when device_unregister() is called for the port service device associated
548 * with the driver.
549 */
550static int pcie_port_remove_service(struct device *dev)
551{
552 struct pcie_device *pciedev;
553 struct pcie_port_service_driver *driver;
554
555 if (!dev || !dev->driver)
556 return 0;
557
558 pciedev = to_pcie_device(dev);
559 driver = to_service_driver(dev->driver);
560 if (driver && driver->remove) {
561 driver->remove(pciedev);
562 put_device(dev);
563 }
564 return 0;
565}
566
567/**
568 * pcie_port_shutdown_service - shut down given PCI Express port service
569 * @dev: PCI Express port service device to handle
570 *
571 * If PCI Express port service driver is registered with
572 * pcie_port_service_register(), this function will be called by the driver core
573 * when device_shutdown() is called for the port service device associated
574 * with the driver.
575 */
576static void pcie_port_shutdown_service(struct device *dev) {}
577
578/**
579 * pcie_port_service_register - register PCI Express port service driver
580 * @new: PCI Express port service driver to register
581 */
582int pcie_port_service_register(struct pcie_port_service_driver *new)
583{
584 if (pcie_ports_disabled)
585 return -ENODEV;
586
587 new->driver.name = new->name;
588 new->driver.bus = &pcie_port_bus_type;
589 new->driver.probe = pcie_port_probe_service;
590 new->driver.remove = pcie_port_remove_service;
591 new->driver.shutdown = pcie_port_shutdown_service;
592
593 return driver_register(drv: &new->driver);
594}
595
596/**
597 * pcie_port_service_unregister - unregister PCI Express port service driver
598 * @drv: PCI Express port service driver to unregister
599 */
600void pcie_port_service_unregister(struct pcie_port_service_driver *drv)
601{
602 driver_unregister(drv: &drv->driver);
603}
604
605/* If this switch is set, PCIe port native services should not be enabled. */
606bool pcie_ports_disabled;
607
608/*
609 * If the user specified "pcie_ports=native", use the PCIe services regardless
610 * of whether the platform has given us permission. On ACPI systems, this
611 * means we ignore _OSC.
612 */
613bool pcie_ports_native;
614
615/*
616 * If the user specified "pcie_ports=dpc-native", use the Linux DPC PCIe
617 * service even if the platform hasn't given us permission.
618 */
619bool pcie_ports_dpc_native;
620
621static int __init pcie_port_setup(char *str)
622{
623 if (!strncmp(str, "compat", 6))
624 pcie_ports_disabled = true;
625 else if (!strncmp(str, "native", 6))
626 pcie_ports_native = true;
627 else if (!strncmp(str, "dpc-native", 10))
628 pcie_ports_dpc_native = true;
629
630 return 1;
631}
632__setup("pcie_ports=", pcie_port_setup);
633
634/* global data */
635
636#ifdef CONFIG_PM
637static int pcie_port_runtime_suspend(struct device *dev)
638{
639 if (!to_pci_dev(dev)->bridge_d3)
640 return -EBUSY;
641
642 return pcie_port_device_runtime_suspend(dev);
643}
644
645static int pcie_port_runtime_idle(struct device *dev)
646{
647 /*
648 * Assume the PCI core has set bridge_d3 whenever it thinks the port
649 * should be good to go to D3. Everything else, including moving
650 * the port to D3, is handled by the PCI core.
651 */
652 return to_pci_dev(dev)->bridge_d3 ? 0 : -EBUSY;
653}
654
655static const struct dev_pm_ops pcie_portdrv_pm_ops = {
656 .suspend = pcie_port_device_suspend,
657 .resume_noirq = pcie_port_device_resume_noirq,
658 .resume = pcie_port_device_resume,
659 .freeze = pcie_port_device_suspend,
660 .thaw = pcie_port_device_resume,
661 .poweroff = pcie_port_device_suspend,
662 .restore_noirq = pcie_port_device_resume_noirq,
663 .restore = pcie_port_device_resume,
664 .runtime_suspend = pcie_port_runtime_suspend,
665 .runtime_resume = pcie_port_device_runtime_resume,
666 .runtime_idle = pcie_port_runtime_idle,
667};
668
669#define PCIE_PORTDRV_PM_OPS (&pcie_portdrv_pm_ops)
670
671#else /* !PM */
672
673#define PCIE_PORTDRV_PM_OPS NULL
674#endif /* !PM */
675
676/*
677 * pcie_portdrv_probe - Probe PCI-Express port devices
678 * @dev: PCI-Express port device being probed
679 *
680 * If detected invokes the pcie_port_device_register() method for
681 * this port device.
682 *
683 */
684static int pcie_portdrv_probe(struct pci_dev *dev,
685 const struct pci_device_id *id)
686{
687 int type = pci_pcie_type(dev);
688 int status;
689
690 if (!pci_is_pcie(dev) ||
691 ((type != PCI_EXP_TYPE_ROOT_PORT) &&
692 (type != PCI_EXP_TYPE_UPSTREAM) &&
693 (type != PCI_EXP_TYPE_DOWNSTREAM) &&
694 (type != PCI_EXP_TYPE_RC_EC)))
695 return -ENODEV;
696
697 if (type == PCI_EXP_TYPE_RC_EC)
698 pcie_link_rcec(rcec: dev);
699
700 status = pcie_port_device_register(dev);
701 if (status)
702 return status;
703
704 pci_save_state(dev);
705
706 dev_pm_set_driver_flags(dev: &dev->dev, DPM_FLAG_NO_DIRECT_COMPLETE |
707 DPM_FLAG_SMART_SUSPEND);
708
709 if (pci_bridge_d3_possible(dev)) {
710 /*
711 * Keep the port resumed 100ms to make sure things like
712 * config space accesses from userspace (lspci) will not
713 * cause the port to repeatedly suspend and resume.
714 */
715 pm_runtime_set_autosuspend_delay(dev: &dev->dev, delay: 100);
716 pm_runtime_use_autosuspend(dev: &dev->dev);
717 pm_runtime_mark_last_busy(dev: &dev->dev);
718 pm_runtime_put_autosuspend(dev: &dev->dev);
719 pm_runtime_allow(dev: &dev->dev);
720 }
721
722 return 0;
723}
724
725static void pcie_portdrv_remove(struct pci_dev *dev)
726{
727 if (pci_bridge_d3_possible(dev)) {
728 pm_runtime_forbid(dev: &dev->dev);
729 pm_runtime_get_noresume(dev: &dev->dev);
730 pm_runtime_dont_use_autosuspend(dev: &dev->dev);
731 }
732
733 pcie_port_device_remove(dev);
734
735 pci_disable_device(dev);
736}
737
738static void pcie_portdrv_shutdown(struct pci_dev *dev)
739{
740 if (pci_bridge_d3_possible(dev)) {
741 pm_runtime_forbid(dev: &dev->dev);
742 pm_runtime_get_noresume(dev: &dev->dev);
743 pm_runtime_dont_use_autosuspend(dev: &dev->dev);
744 }
745
746 pcie_port_device_remove(dev);
747}
748
749static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev,
750 pci_channel_state_t error)
751{
752 if (error == pci_channel_io_frozen)
753 return PCI_ERS_RESULT_NEED_RESET;
754 return PCI_ERS_RESULT_CAN_RECOVER;
755}
756
757static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev)
758{
759 size_t off = offsetof(struct pcie_port_service_driver, slot_reset);
760 device_for_each_child(parent: &dev->dev, data: &off, fn: pcie_port_device_iter);
761
762 pci_restore_state(dev);
763 pci_save_state(dev);
764 return PCI_ERS_RESULT_RECOVERED;
765}
766
767static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev)
768{
769 return PCI_ERS_RESULT_RECOVERED;
770}
771
772/*
773 * LINUX Device Driver Model
774 */
775static const struct pci_device_id port_pci_ids[] = {
776 /* handle any PCI-Express port */
777 { PCI_DEVICE_CLASS(PCI_CLASS_BRIDGE_PCI_NORMAL, ~0) },
778 /* subtractive decode PCI-to-PCI bridge, class type is 060401h */
779 { PCI_DEVICE_CLASS(PCI_CLASS_BRIDGE_PCI_SUBTRACTIVE, ~0) },
780 /* handle any Root Complex Event Collector */
781 { PCI_DEVICE_CLASS(((PCI_CLASS_SYSTEM_RCEC << 8) | 0x00), ~0) },
782 { },
783};
784
785static const struct pci_error_handlers pcie_portdrv_err_handler = {
786 .error_detected = pcie_portdrv_error_detected,
787 .slot_reset = pcie_portdrv_slot_reset,
788 .mmio_enabled = pcie_portdrv_mmio_enabled,
789};
790
791static struct pci_driver pcie_portdriver = {
792 .name = "pcieport",
793 .id_table = port_pci_ids,
794
795 .probe = pcie_portdrv_probe,
796 .remove = pcie_portdrv_remove,
797 .shutdown = pcie_portdrv_shutdown,
798
799 .err_handler = &pcie_portdrv_err_handler,
800
801 .driver_managed_dma = true,
802
803 .driver.pm = PCIE_PORTDRV_PM_OPS,
804};
805
806static int __init dmi_pcie_pme_disable_msi(const struct dmi_system_id *d)
807{
808 pr_notice("%s detected: will not use MSI for PCIe PME signaling\n",
809 d->ident);
810 pcie_pme_disable_msi();
811 return 0;
812}
813
814static const struct dmi_system_id pcie_portdrv_dmi_table[] __initconst = {
815 /*
816 * Boxes that should not use MSI for PCIe PME signaling.
817 */
818 {
819 .callback = dmi_pcie_pme_disable_msi,
820 .ident = "MSI Wind U-100",
821 .matches = {
822 DMI_MATCH(DMI_SYS_VENDOR,
823 "MICRO-STAR INTERNATIONAL CO., LTD"),
824 DMI_MATCH(DMI_PRODUCT_NAME, "U-100"),
825 },
826 },
827 {}
828};
829
830static void __init pcie_init_services(void)
831{
832 pcie_aer_init();
833 pcie_pme_init();
834 pcie_dpc_init();
835 pcie_bwctrl_init();
836 pcie_hp_init();
837}
838
839static int __init pcie_portdrv_init(void)
840{
841 if (pcie_ports_disabled)
842 return -EACCES;
843
844 pcie_init_services();
845 dmi_check_system(list: pcie_portdrv_dmi_table);
846
847 return pci_register_driver(&pcie_portdriver);
848}
849device_initcall(pcie_portdrv_init);
850