1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * libahci.c - Common AHCI SATA low-level routines
4 *
5 * Maintained by: Tejun Heo <tj@kernel.org>
6 * Please ALWAYS copy linux-ide@vger.kernel.org
7 * on emails.
8 *
9 * Copyright 2004-2005 Red Hat, Inc.
10 *
11 * libata documentation is available via 'make {ps|pdf}docs',
12 * as Documentation/driver-api/libata.rst
13 *
14 * AHCI hardware documentation:
15 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
16 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
17 */
18
19#include <linux/bitops.h>
20#include <linux/kernel.h>
21#include <linux/gfp.h>
22#include <linux/module.h>
23#include <linux/nospec.h>
24#include <linux/blkdev.h>
25#include <linux/delay.h>
26#include <linux/interrupt.h>
27#include <linux/dma-mapping.h>
28#include <linux/device.h>
29#include <scsi/scsi_host.h>
30#include <scsi/scsi_cmnd.h>
31#include <linux/libata.h>
32#include <linux/pci.h>
33#include "ahci.h"
34#include "libata.h"
35
36static int ahci_skip_host_reset;
37int ahci_ignore_sss;
38EXPORT_SYMBOL_GPL(ahci_ignore_sss);
39
40module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
41MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
42
43module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
44MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
45
46static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
47 unsigned hints);
48static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
49static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
50 size_t size);
51static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
52 ssize_t size);
53
54
55
56static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
57static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
58static void ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
59static void ahci_qc_ncq_fill_rtf(struct ata_port *ap, u64 done_mask);
60static int ahci_port_start(struct ata_port *ap);
61static void ahci_port_stop(struct ata_port *ap);
62static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc);
63static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
64static void ahci_freeze(struct ata_port *ap);
65static void ahci_thaw(struct ata_port *ap);
66static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
67static void ahci_enable_fbs(struct ata_port *ap);
68static void ahci_disable_fbs(struct ata_port *ap);
69static void ahci_pmp_attach(struct ata_port *ap);
70static void ahci_pmp_detach(struct ata_port *ap);
71static int ahci_softreset(struct ata_link *link, unsigned int *class,
72 unsigned long deadline);
73static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
74 unsigned long deadline);
75static int ahci_hardreset(struct ata_link *link, unsigned int *class,
76 unsigned long deadline);
77static void ahci_postreset(struct ata_link *link, unsigned int *class);
78static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
79static void ahci_dev_config(struct ata_device *dev);
80#ifdef CONFIG_PM
81static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
82#endif
83static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
84static ssize_t ahci_activity_store(struct ata_device *dev,
85 enum sw_activity val);
86static void ahci_init_sw_activity(struct ata_link *link);
87
88static ssize_t ahci_show_host_caps(struct device *dev,
89 struct device_attribute *attr, char *buf);
90static ssize_t ahci_show_host_cap2(struct device *dev,
91 struct device_attribute *attr, char *buf);
92static ssize_t ahci_show_host_version(struct device *dev,
93 struct device_attribute *attr, char *buf);
94static ssize_t ahci_show_port_cmd(struct device *dev,
95 struct device_attribute *attr, char *buf);
96static ssize_t ahci_read_em_buffer(struct device *dev,
97 struct device_attribute *attr, char *buf);
98static ssize_t ahci_store_em_buffer(struct device *dev,
99 struct device_attribute *attr,
100 const char *buf, size_t size);
101static ssize_t ahci_show_em_supported(struct device *dev,
102 struct device_attribute *attr, char *buf);
103static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance);
104
105static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
106static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
107static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
108static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
109static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
110 ahci_read_em_buffer, ahci_store_em_buffer);
111static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
112
113static struct attribute *ahci_shost_attrs[] = {
114 &dev_attr_link_power_management_supported.attr,
115 &dev_attr_link_power_management_policy.attr,
116 &dev_attr_em_message_type.attr,
117 &dev_attr_em_message.attr,
118 &dev_attr_ahci_host_caps.attr,
119 &dev_attr_ahci_host_cap2.attr,
120 &dev_attr_ahci_host_version.attr,
121 &dev_attr_ahci_port_cmd.attr,
122 &dev_attr_em_buffer.attr,
123 &dev_attr_em_message_supported.attr,
124 NULL
125};
126
127static const struct attribute_group ahci_shost_attr_group = {
128 .attrs = ahci_shost_attrs
129};
130
131const struct attribute_group *ahci_shost_groups[] = {
132 &ahci_shost_attr_group,
133 NULL
134};
135EXPORT_SYMBOL_GPL(ahci_shost_groups);
136
137static struct attribute *ahci_sdev_attrs[] = {
138 &dev_attr_sw_activity.attr,
139 &dev_attr_unload_heads.attr,
140 &dev_attr_ncq_prio_supported.attr,
141 &dev_attr_ncq_prio_enable.attr,
142 NULL
143};
144
145static const struct attribute_group ahci_sdev_attr_group = {
146 .attrs = ahci_sdev_attrs
147};
148
149const struct attribute_group *ahci_sdev_groups[] = {
150 &ahci_sdev_attr_group,
151 NULL
152};
153EXPORT_SYMBOL_GPL(ahci_sdev_groups);
154
155struct ata_port_operations ahci_ops = {
156 .inherits = &sata_pmp_port_ops,
157
158 .qc_defer = ahci_pmp_qc_defer,
159 .qc_prep = ahci_qc_prep,
160 .qc_issue = ahci_qc_issue,
161 .qc_fill_rtf = ahci_qc_fill_rtf,
162 .qc_ncq_fill_rtf = ahci_qc_ncq_fill_rtf,
163
164 .freeze = ahci_freeze,
165 .thaw = ahci_thaw,
166 .reset.softreset = ahci_softreset,
167 .reset.hardreset = ahci_hardreset,
168 .reset.postreset = ahci_postreset,
169 .pmp_reset.softreset = ahci_softreset,
170 .error_handler = ahci_error_handler,
171 .post_internal_cmd = ahci_post_internal_cmd,
172 .dev_config = ahci_dev_config,
173
174 .scr_read = ahci_scr_read,
175 .scr_write = ahci_scr_write,
176 .pmp_attach = ahci_pmp_attach,
177 .pmp_detach = ahci_pmp_detach,
178
179 .set_lpm = ahci_set_lpm,
180 .em_show = ahci_led_show,
181 .em_store = ahci_led_store,
182 .sw_activity_show = ahci_activity_show,
183 .sw_activity_store = ahci_activity_store,
184 .transmit_led_message = ahci_transmit_led_message,
185#ifdef CONFIG_PM
186 .port_suspend = ahci_port_suspend,
187 .port_resume = ahci_port_resume,
188#endif
189 .port_start = ahci_port_start,
190 .port_stop = ahci_port_stop,
191};
192EXPORT_SYMBOL_GPL(ahci_ops);
193
194struct ata_port_operations ahci_pmp_retry_srst_ops = {
195 .inherits = &ahci_ops,
196 .reset.softreset = ahci_pmp_retry_softreset,
197};
198EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
199
200static bool ahci_em_messages __read_mostly = true;
201module_param(ahci_em_messages, bool, 0444);
202/* add other LED protocol types when they become supported */
203MODULE_PARM_DESC(ahci_em_messages,
204 "AHCI Enclosure Management Message control (0 = off, 1 = on)");
205
206/* device sleep idle timeout in ms */
207static int devslp_idle_timeout __read_mostly = 1000;
208module_param(devslp_idle_timeout, int, 0644);
209MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
210
211static void ahci_enable_ahci(void __iomem *mmio)
212{
213 int i;
214 u32 tmp;
215
216 /* turn on AHCI_EN */
217 tmp = readl(addr: mmio + HOST_CTL);
218 if (tmp & HOST_AHCI_EN)
219 return;
220
221 /* Some controllers need AHCI_EN to be written multiple times.
222 * Try a few times before giving up.
223 */
224 for (i = 0; i < 5; i++) {
225 tmp |= HOST_AHCI_EN;
226 writel(val: tmp, addr: mmio + HOST_CTL);
227 tmp = readl(addr: mmio + HOST_CTL); /* flush && sanity check */
228 if (tmp & HOST_AHCI_EN)
229 return;
230 msleep(msecs: 10);
231 }
232
233 WARN_ON(1);
234}
235
236/**
237 * ahci_rpm_get_port - Make sure the port is powered on
238 * @ap: Port to power on
239 *
240 * Whenever there is need to access the AHCI host registers outside of
241 * normal execution paths, call this function to make sure the host is
242 * actually powered on.
243 */
244static int ahci_rpm_get_port(struct ata_port *ap)
245{
246 return pm_runtime_get_sync(dev: ap->dev);
247}
248
249/**
250 * ahci_rpm_put_port - Undoes ahci_rpm_get_port()
251 * @ap: Port to power down
252 *
253 * Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
254 * if it has no more active users.
255 */
256static void ahci_rpm_put_port(struct ata_port *ap)
257{
258 pm_runtime_put(dev: ap->dev);
259}
260
261static ssize_t ahci_show_host_caps(struct device *dev,
262 struct device_attribute *attr, char *buf)
263{
264 struct Scsi_Host *shost = class_to_shost(dev);
265 struct ata_port *ap = ata_shost_to_port(host: shost);
266 struct ahci_host_priv *hpriv = ap->host->private_data;
267
268 return sprintf(buf, fmt: "%x\n", hpriv->cap);
269}
270
271static ssize_t ahci_show_host_cap2(struct device *dev,
272 struct device_attribute *attr, char *buf)
273{
274 struct Scsi_Host *shost = class_to_shost(dev);
275 struct ata_port *ap = ata_shost_to_port(host: shost);
276 struct ahci_host_priv *hpriv = ap->host->private_data;
277
278 return sprintf(buf, fmt: "%x\n", hpriv->cap2);
279}
280
281static ssize_t ahci_show_host_version(struct device *dev,
282 struct device_attribute *attr, char *buf)
283{
284 struct Scsi_Host *shost = class_to_shost(dev);
285 struct ata_port *ap = ata_shost_to_port(host: shost);
286 struct ahci_host_priv *hpriv = ap->host->private_data;
287
288 return sprintf(buf, fmt: "%x\n", hpriv->version);
289}
290
291static ssize_t ahci_show_port_cmd(struct device *dev,
292 struct device_attribute *attr, char *buf)
293{
294 struct Scsi_Host *shost = class_to_shost(dev);
295 struct ata_port *ap = ata_shost_to_port(host: shost);
296 void __iomem *port_mmio = ahci_port_base(ap);
297 ssize_t ret;
298
299 ahci_rpm_get_port(ap);
300 ret = sprintf(buf, fmt: "%x\n", readl(addr: port_mmio + PORT_CMD));
301 ahci_rpm_put_port(ap);
302
303 return ret;
304}
305
306static ssize_t ahci_read_em_buffer(struct device *dev,
307 struct device_attribute *attr, char *buf)
308{
309 struct Scsi_Host *shost = class_to_shost(dev);
310 struct ata_port *ap = ata_shost_to_port(host: shost);
311 struct ahci_host_priv *hpriv = ap->host->private_data;
312 void __iomem *mmio = hpriv->mmio;
313 void __iomem *em_mmio = mmio + hpriv->em_loc;
314 u32 em_ctl, msg;
315 unsigned long flags;
316 size_t count;
317 int i;
318
319 ahci_rpm_get_port(ap);
320 spin_lock_irqsave(ap->lock, flags);
321
322 em_ctl = readl(addr: mmio + HOST_EM_CTL);
323 if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
324 !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
325 spin_unlock_irqrestore(lock: ap->lock, flags);
326 ahci_rpm_put_port(ap);
327 return -EINVAL;
328 }
329
330 if (!(em_ctl & EM_CTL_MR)) {
331 spin_unlock_irqrestore(lock: ap->lock, flags);
332 ahci_rpm_put_port(ap);
333 return -EAGAIN;
334 }
335
336 if (!(em_ctl & EM_CTL_SMB))
337 em_mmio += hpriv->em_buf_sz;
338
339 count = hpriv->em_buf_sz;
340
341 /* the count should not be larger than PAGE_SIZE */
342 if (count > PAGE_SIZE) {
343 if (printk_ratelimit())
344 ata_port_warn(ap,
345 "EM read buffer size too large: "
346 "buffer size %u, page size %lu\n",
347 hpriv->em_buf_sz, PAGE_SIZE);
348 count = PAGE_SIZE;
349 }
350
351 for (i = 0; i < count; i += 4) {
352 msg = readl(addr: em_mmio + i);
353 buf[i] = msg & 0xff;
354 buf[i + 1] = (msg >> 8) & 0xff;
355 buf[i + 2] = (msg >> 16) & 0xff;
356 buf[i + 3] = (msg >> 24) & 0xff;
357 }
358
359 spin_unlock_irqrestore(lock: ap->lock, flags);
360 ahci_rpm_put_port(ap);
361
362 return i;
363}
364
365static ssize_t ahci_store_em_buffer(struct device *dev,
366 struct device_attribute *attr,
367 const char *buf, size_t size)
368{
369 struct Scsi_Host *shost = class_to_shost(dev);
370 struct ata_port *ap = ata_shost_to_port(host: shost);
371 struct ahci_host_priv *hpriv = ap->host->private_data;
372 void __iomem *mmio = hpriv->mmio;
373 void __iomem *em_mmio = mmio + hpriv->em_loc;
374 const unsigned char *msg_buf = buf;
375 u32 em_ctl, msg;
376 unsigned long flags;
377 int i;
378
379 /* check size validity */
380 if (!(ap->flags & ATA_FLAG_EM) ||
381 !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
382 size % 4 || size > hpriv->em_buf_sz)
383 return -EINVAL;
384
385 ahci_rpm_get_port(ap);
386 spin_lock_irqsave(ap->lock, flags);
387
388 em_ctl = readl(addr: mmio + HOST_EM_CTL);
389 if (em_ctl & EM_CTL_TM) {
390 spin_unlock_irqrestore(lock: ap->lock, flags);
391 ahci_rpm_put_port(ap);
392 return -EBUSY;
393 }
394
395 for (i = 0; i < size; i += 4) {
396 msg = msg_buf[i] | msg_buf[i + 1] << 8 |
397 msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
398 writel(val: msg, addr: em_mmio + i);
399 }
400
401 writel(val: em_ctl | EM_CTL_TM, addr: mmio + HOST_EM_CTL);
402
403 spin_unlock_irqrestore(lock: ap->lock, flags);
404 ahci_rpm_put_port(ap);
405
406 return size;
407}
408
409static ssize_t ahci_show_em_supported(struct device *dev,
410 struct device_attribute *attr, char *buf)
411{
412 struct Scsi_Host *shost = class_to_shost(dev);
413 struct ata_port *ap = ata_shost_to_port(host: shost);
414 struct ahci_host_priv *hpriv = ap->host->private_data;
415 void __iomem *mmio = hpriv->mmio;
416 u32 em_ctl;
417
418 ahci_rpm_get_port(ap);
419 em_ctl = readl(addr: mmio + HOST_EM_CTL);
420 ahci_rpm_put_port(ap);
421
422 return sprintf(buf, fmt: "%s%s%s%s\n",
423 em_ctl & EM_CTL_LED ? "led " : "",
424 em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
425 em_ctl & EM_CTL_SES ? "ses-2 " : "",
426 em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
427}
428
429/**
430 * ahci_save_initial_config - Save and fixup initial config values
431 * @dev: target AHCI device
432 * @hpriv: host private area to store config values
433 *
434 * Some registers containing configuration info might be setup by
435 * BIOS and might be cleared on reset. This function saves the
436 * initial values of those registers into @hpriv such that they
437 * can be restored after controller reset.
438 *
439 * If inconsistent, config values are fixed up by this function.
440 *
441 * If it is not set already this function sets hpriv->start_engine to
442 * ahci_start_engine.
443 *
444 * LOCKING:
445 * None.
446 */
447void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv)
448{
449 void __iomem *mmio = hpriv->mmio;
450 void __iomem *port_mmio;
451 unsigned long port_map;
452 u32 cap, cap2, vers;
453 int i;
454
455 /* make sure AHCI mode is enabled before accessing CAP */
456 ahci_enable_ahci(mmio);
457
458 /*
459 * Values prefixed with saved_ are written back to the HBA and ports
460 * registers after reset. Values without are used for driver operation.
461 */
462
463 /*
464 * Override HW-init HBA capability fields with the platform-specific
465 * values. The rest of the HBA capabilities are defined as Read-only
466 * and can't be modified in CSR anyway.
467 */
468 cap = readl(addr: mmio + HOST_CAP);
469 if (hpriv->saved_cap)
470 cap = (cap & ~(HOST_CAP_SSS | HOST_CAP_MPS)) | hpriv->saved_cap;
471 hpriv->saved_cap = cap;
472
473 /* CAP2 register is only defined for AHCI 1.2 and later */
474 vers = readl(addr: mmio + HOST_VERSION);
475 if ((vers >> 16) > 1 ||
476 ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
477 hpriv->saved_cap2 = cap2 = readl(addr: mmio + HOST_CAP2);
478 else
479 hpriv->saved_cap2 = cap2 = 0;
480
481 /* some chips have errata preventing 64bit use */
482 if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
483 dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
484 cap &= ~HOST_CAP_64;
485 }
486
487 if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
488 dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
489 cap &= ~HOST_CAP_NCQ;
490 }
491
492 if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
493 dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
494 cap |= HOST_CAP_NCQ;
495 }
496
497 if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
498 dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
499 cap &= ~HOST_CAP_PMP;
500 }
501
502 if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
503 dev_info(dev,
504 "controller can't do SNTF, turning off CAP_SNTF\n");
505 cap &= ~HOST_CAP_SNTF;
506 }
507
508 if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
509 dev_info(dev,
510 "controller can't do DEVSLP, turning off\n");
511 cap2 &= ~HOST_CAP2_SDS;
512 cap2 &= ~HOST_CAP2_SADM;
513 }
514
515 if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
516 dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
517 cap |= HOST_CAP_FBS;
518 }
519
520 if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {
521 dev_info(dev, "controller can't do FBS, turning off CAP_FBS\n");
522 cap &= ~HOST_CAP_FBS;
523 }
524
525 if (!(cap & HOST_CAP_ALPM) && (hpriv->flags & AHCI_HFLAG_YES_ALPM)) {
526 dev_info(dev, "controller can do ALPM, turning on CAP_ALPM\n");
527 cap |= HOST_CAP_ALPM;
528 }
529
530 if ((cap & HOST_CAP_SXS) && (hpriv->flags & AHCI_HFLAG_NO_SXS)) {
531 dev_info(dev, "controller does not support SXS, disabling CAP_SXS\n");
532 cap &= ~HOST_CAP_SXS;
533 }
534
535 /* Override the HBA ports mapping if the platform needs it */
536 port_map = readl(addr: mmio + HOST_PORTS_IMPL);
537 if (hpriv->saved_port_map && port_map != hpriv->saved_port_map) {
538 dev_info(dev, "forcing port_map 0x%lx -> 0x%x\n",
539 port_map, hpriv->saved_port_map);
540 port_map = hpriv->saved_port_map;
541 } else {
542 hpriv->saved_port_map = port_map;
543 }
544
545 /* mask_port_map not set means that all ports are available */
546 if (hpriv->mask_port_map) {
547 dev_warn(dev, "masking port_map 0x%lx -> 0x%lx\n",
548 port_map,
549 port_map & hpriv->mask_port_map);
550 port_map &= hpriv->mask_port_map;
551 }
552
553 /* cross check port_map and cap.n_ports */
554 if (port_map) {
555 int map_ports = 0;
556
557 for (i = 0; i < AHCI_MAX_PORTS; i++)
558 if (port_map & (1 << i))
559 map_ports++;
560
561 /* If PI has more ports than n_ports, whine, clear
562 * port_map and let it be generated from n_ports.
563 */
564 if (map_ports > ahci_nr_ports(cap)) {
565 dev_warn(dev,
566 "implemented port map (0x%lx) contains more ports than nr_ports (%u), using nr_ports\n",
567 port_map, ahci_nr_ports(cap));
568 port_map = 0;
569 }
570 }
571
572 /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
573 if (!port_map && vers < 0x10300) {
574 port_map = (1 << ahci_nr_ports(cap)) - 1;
575 dev_warn(dev, "forcing PORTS_IMPL to 0x%lx\n", port_map);
576
577 /* write the fixed up value to the PI register */
578 hpriv->saved_port_map = port_map;
579 }
580
581 /*
582 * Preserve the ports capabilities defined by the platform. Note there
583 * is no need in storing the rest of the P#.CMD fields since they are
584 * volatile.
585 */
586 for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) {
587 if (hpriv->saved_port_cap[i])
588 continue;
589
590 port_mmio = __ahci_port_base(hpriv, port_no: i);
591 hpriv->saved_port_cap[i] =
592 readl(addr: port_mmio + PORT_CMD) & PORT_CMD_CAP;
593 }
594
595 /* record values to use during operation */
596 hpriv->cap = cap;
597 hpriv->cap2 = cap2;
598 hpriv->version = vers;
599 hpriv->port_map = port_map;
600
601 if (!hpriv->start_engine)
602 hpriv->start_engine = ahci_start_engine;
603
604 if (!hpriv->stop_engine)
605 hpriv->stop_engine = ahci_stop_engine;
606
607 if (!hpriv->irq_handler)
608 hpriv->irq_handler = ahci_single_level_irq_intr;
609}
610EXPORT_SYMBOL_GPL(ahci_save_initial_config);
611
612/**
613 * ahci_restore_initial_config - Restore initial config
614 * @host: target ATA host
615 *
616 * Restore initial config stored by ahci_save_initial_config().
617 *
618 * LOCKING:
619 * None.
620 */
621static void ahci_restore_initial_config(struct ata_host *host)
622{
623 struct ahci_host_priv *hpriv = host->private_data;
624 unsigned long port_map = hpriv->port_map;
625 void __iomem *mmio = hpriv->mmio;
626 void __iomem *port_mmio;
627 int i;
628
629 writel(val: hpriv->saved_cap, addr: mmio + HOST_CAP);
630 if (hpriv->saved_cap2)
631 writel(val: hpriv->saved_cap2, addr: mmio + HOST_CAP2);
632 writel(val: hpriv->saved_port_map, addr: mmio + HOST_PORTS_IMPL);
633 (void) readl(addr: mmio + HOST_PORTS_IMPL); /* flush */
634
635 for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) {
636 port_mmio = __ahci_port_base(hpriv, port_no: i);
637 writel(val: hpriv->saved_port_cap[i], addr: port_mmio + PORT_CMD);
638 }
639}
640
641static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
642{
643 static const int offset[] = {
644 [SCR_STATUS] = PORT_SCR_STAT,
645 [SCR_CONTROL] = PORT_SCR_CTL,
646 [SCR_ERROR] = PORT_SCR_ERR,
647 [SCR_ACTIVE] = PORT_SCR_ACT,
648 [SCR_NOTIFICATION] = PORT_SCR_NTF,
649 };
650 struct ahci_host_priv *hpriv = ap->host->private_data;
651
652 if (sc_reg < ARRAY_SIZE(offset) &&
653 (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
654 return offset[sc_reg];
655 return 0;
656}
657
658static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
659{
660 void __iomem *port_mmio = ahci_port_base(ap: link->ap);
661 int offset = ahci_scr_offset(ap: link->ap, sc_reg);
662
663 if (offset) {
664 *val = readl(addr: port_mmio + offset);
665 return 0;
666 }
667 return -EINVAL;
668}
669
670static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
671{
672 void __iomem *port_mmio = ahci_port_base(ap: link->ap);
673 int offset = ahci_scr_offset(ap: link->ap, sc_reg);
674
675 if (offset) {
676 writel(val, addr: port_mmio + offset);
677 return 0;
678 }
679 return -EINVAL;
680}
681
682void ahci_start_engine(struct ata_port *ap)
683{
684 void __iomem *port_mmio = ahci_port_base(ap);
685 u32 tmp;
686
687 /* start DMA */
688 tmp = readl(addr: port_mmio + PORT_CMD);
689 tmp |= PORT_CMD_START;
690 writel(val: tmp, addr: port_mmio + PORT_CMD);
691 readl(addr: port_mmio + PORT_CMD); /* flush */
692}
693EXPORT_SYMBOL_GPL(ahci_start_engine);
694
695int ahci_stop_engine(struct ata_port *ap)
696{
697 void __iomem *port_mmio = ahci_port_base(ap);
698 struct ahci_host_priv *hpriv = ap->host->private_data;
699 u32 tmp;
700
701 /*
702 * On some controllers, stopping a port's DMA engine while the port
703 * is in ALPM state (partial or slumber) results in failures on
704 * subsequent DMA engine starts. For those controllers, put the
705 * port back in active state before stopping its DMA engine.
706 */
707 if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&
708 (ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&
709 ahci_set_lpm(link: &ap->link, policy: ATA_LPM_MAX_POWER, hints: ATA_LPM_WAKE_ONLY)) {
710 dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");
711 return -EIO;
712 }
713
714 tmp = readl(addr: port_mmio + PORT_CMD);
715
716 /* check if the HBA is idle */
717 if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
718 return 0;
719
720 /*
721 * Don't try to issue commands but return with ENODEV if the
722 * AHCI controller not available anymore (e.g. due to PCIe hot
723 * unplugging). Otherwise a 500ms delay for each port is added.
724 */
725 if (tmp == 0xffffffff) {
726 dev_err(ap->host->dev, "AHCI controller unavailable!\n");
727 return -ENODEV;
728 }
729
730 /* setting HBA to idle */
731 tmp &= ~PORT_CMD_START;
732 writel(val: tmp, addr: port_mmio + PORT_CMD);
733
734 /* wait for engine to stop. This could be as long as 500 msec */
735 tmp = ata_wait_register(ap, reg: port_mmio + PORT_CMD,
736 mask: PORT_CMD_LIST_ON, val: PORT_CMD_LIST_ON, interval: 1, timeout: 500);
737 if (tmp & PORT_CMD_LIST_ON)
738 return -EIO;
739
740 return 0;
741}
742EXPORT_SYMBOL_GPL(ahci_stop_engine);
743
744void ahci_start_fis_rx(struct ata_port *ap)
745{
746 void __iomem *port_mmio = ahci_port_base(ap);
747 struct ahci_host_priv *hpriv = ap->host->private_data;
748 struct ahci_port_priv *pp = ap->private_data;
749 u32 tmp;
750
751 /* set FIS registers */
752 if (hpriv->cap & HOST_CAP_64)
753 writel(val: (pp->cmd_slot_dma >> 16) >> 16,
754 addr: port_mmio + PORT_LST_ADDR_HI);
755 writel(val: pp->cmd_slot_dma & 0xffffffff, addr: port_mmio + PORT_LST_ADDR);
756
757 if (hpriv->cap & HOST_CAP_64)
758 writel(val: (pp->rx_fis_dma >> 16) >> 16,
759 addr: port_mmio + PORT_FIS_ADDR_HI);
760 writel(val: pp->rx_fis_dma & 0xffffffff, addr: port_mmio + PORT_FIS_ADDR);
761
762 /* enable FIS reception */
763 tmp = readl(addr: port_mmio + PORT_CMD);
764 tmp |= PORT_CMD_FIS_RX;
765 writel(val: tmp, addr: port_mmio + PORT_CMD);
766
767 /* flush */
768 readl(addr: port_mmio + PORT_CMD);
769}
770EXPORT_SYMBOL_GPL(ahci_start_fis_rx);
771
772static int ahci_stop_fis_rx(struct ata_port *ap)
773{
774 void __iomem *port_mmio = ahci_port_base(ap);
775 u32 tmp;
776
777 /* disable FIS reception */
778 tmp = readl(addr: port_mmio + PORT_CMD);
779 tmp &= ~PORT_CMD_FIS_RX;
780 writel(val: tmp, addr: port_mmio + PORT_CMD);
781
782 /* wait for completion, spec says 500ms, give it 1000 */
783 tmp = ata_wait_register(ap, reg: port_mmio + PORT_CMD, mask: PORT_CMD_FIS_ON,
784 val: PORT_CMD_FIS_ON, interval: 10, timeout: 1000);
785 if (tmp & PORT_CMD_FIS_ON)
786 return -EBUSY;
787
788 return 0;
789}
790
791static void ahci_power_up(struct ata_port *ap)
792{
793 struct ahci_host_priv *hpriv = ap->host->private_data;
794 void __iomem *port_mmio = ahci_port_base(ap);
795 u32 cmd;
796
797 cmd = readl(addr: port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
798
799 /* spin up device */
800 if (hpriv->cap & HOST_CAP_SSS) {
801 cmd |= PORT_CMD_SPIN_UP;
802 writel(val: cmd, addr: port_mmio + PORT_CMD);
803 }
804
805 /* wake up link */
806 writel(val: cmd | PORT_CMD_ICC_ACTIVE, addr: port_mmio + PORT_CMD);
807}
808
809static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
810 unsigned int hints)
811{
812 struct ata_port *ap = link->ap;
813 struct ahci_host_priv *hpriv = ap->host->private_data;
814 struct ahci_port_priv *pp = ap->private_data;
815 void __iomem *port_mmio = ahci_port_base(ap);
816
817 if (policy != ATA_LPM_MAX_POWER) {
818 /* wakeup flag only applies to the max power policy */
819 hints &= ~ATA_LPM_WAKE_ONLY;
820
821 /*
822 * Disable interrupts on Phy Ready. This keeps us from
823 * getting woken up due to spurious phy ready
824 * interrupts.
825 */
826 pp->intr_mask &= ~PORT_IRQ_PHYRDY;
827 writel(val: pp->intr_mask, addr: port_mmio + PORT_IRQ_MASK);
828
829 sata_link_scr_lpm(link, policy, spm_wakeup: false);
830 }
831
832 if (hpriv->cap & HOST_CAP_ALPM) {
833 u32 cmd = readl(addr: port_mmio + PORT_CMD);
834
835 if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
836 if (!(hints & ATA_LPM_WAKE_ONLY))
837 cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
838 cmd |= PORT_CMD_ICC_ACTIVE;
839
840 writel(val: cmd, addr: port_mmio + PORT_CMD);
841 readl(addr: port_mmio + PORT_CMD);
842
843 /* wait 10ms to be sure we've come out of LPM state */
844 ata_msleep(ap, msecs: 10);
845
846 if (hints & ATA_LPM_WAKE_ONLY)
847 return 0;
848 } else {
849 cmd |= PORT_CMD_ALPE;
850 if (policy == ATA_LPM_MIN_POWER)
851 cmd |= PORT_CMD_ASP;
852 else if (policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
853 cmd &= ~PORT_CMD_ASP;
854
855 /* write out new cmd value */
856 writel(val: cmd, addr: port_mmio + PORT_CMD);
857 }
858 }
859
860 /* set aggressive device sleep */
861 if ((hpriv->cap2 & HOST_CAP2_SDS) &&
862 (hpriv->cap2 & HOST_CAP2_SADM) &&
863 (link->device->flags & ATA_DFLAG_DEVSLP)) {
864 if (policy == ATA_LPM_MIN_POWER ||
865 policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
866 ahci_set_aggressive_devslp(ap, sleep: true);
867 else
868 ahci_set_aggressive_devslp(ap, sleep: false);
869 }
870
871 if (policy == ATA_LPM_MAX_POWER) {
872 sata_link_scr_lpm(link, policy, spm_wakeup: false);
873
874 /* turn PHYRDY IRQ back on */
875 pp->intr_mask |= PORT_IRQ_PHYRDY;
876 writel(val: pp->intr_mask, addr: port_mmio + PORT_IRQ_MASK);
877 }
878
879 return 0;
880}
881
882#ifdef CONFIG_PM
883static void ahci_power_down(struct ata_port *ap)
884{
885 struct ahci_host_priv *hpriv = ap->host->private_data;
886 void __iomem *port_mmio = ahci_port_base(ap);
887 u32 cmd, scontrol;
888
889 if (!(hpriv->cap & HOST_CAP_SSS))
890 return;
891
892 /* put device into listen mode, first set PxSCTL.DET to 0 */
893 scontrol = readl(addr: port_mmio + PORT_SCR_CTL);
894 scontrol &= ~0xf;
895 writel(val: scontrol, addr: port_mmio + PORT_SCR_CTL);
896
897 /* then set PxCMD.SUD to 0 */
898 cmd = readl(addr: port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
899 cmd &= ~PORT_CMD_SPIN_UP;
900 writel(val: cmd, addr: port_mmio + PORT_CMD);
901}
902#endif
903
904static void ahci_start_port(struct ata_port *ap)
905{
906 struct ahci_host_priv *hpriv = ap->host->private_data;
907 struct ahci_port_priv *pp = ap->private_data;
908 struct ata_link *link;
909 struct ahci_em_priv *emp;
910 ssize_t rc;
911 int i;
912
913 /* enable FIS reception */
914 ahci_start_fis_rx(ap);
915
916 /* enable DMA */
917 if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
918 hpriv->start_engine(ap);
919
920 /* turn on LEDs */
921 if (ap->flags & ATA_FLAG_EM) {
922 ata_for_each_link(link, ap, EDGE) {
923 emp = &pp->em_priv[link->pmp];
924
925 /* EM Transmit bit maybe busy during init */
926 for (i = 0; i < EM_MAX_RETRY; i++) {
927 rc = ap->ops->transmit_led_message(ap,
928 emp->led_state,
929 4);
930 /*
931 * If busy, give a breather but do not
932 * release EH ownership by using msleep()
933 * instead of ata_msleep(). EM Transmit
934 * bit is busy for the whole host and
935 * releasing ownership will cause other
936 * ports to fail the same way.
937 */
938 if (rc == -EBUSY)
939 msleep(msecs: 1);
940 else
941 break;
942 }
943 }
944 }
945
946 if (ap->flags & ATA_FLAG_SW_ACTIVITY)
947 ata_for_each_link(link, ap, EDGE)
948 ahci_init_sw_activity(link);
949
950}
951
952static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
953{
954 int rc;
955 struct ahci_host_priv *hpriv = ap->host->private_data;
956
957 /* disable DMA */
958 rc = hpriv->stop_engine(ap);
959 if (rc) {
960 *emsg = "failed to stop engine";
961 return rc;
962 }
963
964 /* disable FIS reception */
965 rc = ahci_stop_fis_rx(ap);
966 if (rc) {
967 *emsg = "failed stop FIS RX";
968 return rc;
969 }
970
971 return 0;
972}
973
974int ahci_reset_controller(struct ata_host *host)
975{
976 struct ahci_host_priv *hpriv = host->private_data;
977 void __iomem *mmio = hpriv->mmio;
978 u32 tmp;
979
980 /*
981 * We must be in AHCI mode, before using anything AHCI-specific, such
982 * as HOST_RESET.
983 */
984 ahci_enable_ahci(mmio);
985
986 /* Global controller reset */
987 if (ahci_skip_host_reset) {
988 dev_info(host->dev, "Skipping global host reset\n");
989 return 0;
990 }
991
992 tmp = readl(addr: mmio + HOST_CTL);
993 if (!(tmp & HOST_RESET)) {
994 writel(val: tmp | HOST_RESET, addr: mmio + HOST_CTL);
995 readl(addr: mmio + HOST_CTL); /* flush */
996 }
997
998 /*
999 * To perform host reset, OS should set HOST_RESET and poll until this
1000 * bit is read to be "0". Reset must complete within 1 second, or the
1001 * hardware should be considered fried.
1002 */
1003 tmp = ata_wait_register(NULL, reg: mmio + HOST_CTL, mask: HOST_RESET,
1004 val: HOST_RESET, interval: 10, timeout: 1000);
1005 if (tmp & HOST_RESET) {
1006 dev_err(host->dev, "Controller reset failed (0x%x)\n",
1007 tmp);
1008 return -EIO;
1009 }
1010
1011 /* Turn on AHCI mode */
1012 ahci_enable_ahci(mmio);
1013
1014 /* Some registers might be cleared on reset. Restore initial values. */
1015 if (!(hpriv->flags & AHCI_HFLAG_NO_WRITE_TO_RO))
1016 ahci_restore_initial_config(host);
1017
1018 return 0;
1019}
1020EXPORT_SYMBOL_GPL(ahci_reset_controller);
1021
1022static void ahci_sw_activity(struct ata_link *link)
1023{
1024 struct ata_port *ap = link->ap;
1025 struct ahci_port_priv *pp = ap->private_data;
1026 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1027
1028 if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
1029 return;
1030
1031 emp->activity++;
1032 if (!timer_pending(timer: &emp->timer))
1033 mod_timer(timer: &emp->timer, expires: jiffies + msecs_to_jiffies(m: 10));
1034}
1035
1036static void ahci_sw_activity_blink(struct timer_list *t)
1037{
1038 struct ahci_em_priv *emp = timer_container_of(emp, t, timer);
1039 struct ata_link *link = emp->link;
1040 struct ata_port *ap = link->ap;
1041
1042 unsigned long led_message = emp->led_state;
1043 u32 activity_led_state;
1044 unsigned long flags;
1045
1046 led_message &= EM_MSG_LED_VALUE;
1047 led_message |= ap->port_no | (link->pmp << 8);
1048
1049 /* check to see if we've had activity. If so,
1050 * toggle state of LED and reset timer. If not,
1051 * turn LED to desired idle state.
1052 */
1053 spin_lock_irqsave(ap->lock, flags);
1054 if (emp->saved_activity != emp->activity) {
1055 emp->saved_activity = emp->activity;
1056 /* get the current LED state */
1057 activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
1058
1059 if (activity_led_state)
1060 activity_led_state = 0;
1061 else
1062 activity_led_state = 1;
1063
1064 /* clear old state */
1065 led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
1066
1067 /* toggle state */
1068 led_message |= (activity_led_state << 16);
1069 mod_timer(timer: &emp->timer, expires: jiffies + msecs_to_jiffies(m: 100));
1070 } else {
1071 /* switch to idle */
1072 led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
1073 if (emp->blink_policy == BLINK_OFF)
1074 led_message |= (1 << 16);
1075 }
1076 spin_unlock_irqrestore(lock: ap->lock, flags);
1077 ap->ops->transmit_led_message(ap, led_message, 4);
1078}
1079
1080static void ahci_init_sw_activity(struct ata_link *link)
1081{
1082 struct ata_port *ap = link->ap;
1083 struct ahci_port_priv *pp = ap->private_data;
1084 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1085
1086 /* init activity stats, setup timer */
1087 emp->saved_activity = emp->activity = 0;
1088 emp->link = link;
1089 timer_setup(&emp->timer, ahci_sw_activity_blink, 0);
1090
1091 /* check our blink policy and set flag for link if it's enabled */
1092 if (emp->blink_policy)
1093 link->flags |= ATA_LFLAG_SW_ACTIVITY;
1094}
1095
1096int ahci_reset_em(struct ata_host *host)
1097{
1098 struct ahci_host_priv *hpriv = host->private_data;
1099 void __iomem *mmio = hpriv->mmio;
1100 u32 em_ctl;
1101
1102 em_ctl = readl(addr: mmio + HOST_EM_CTL);
1103 if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
1104 return -EINVAL;
1105
1106 writel(val: em_ctl | EM_CTL_RST, addr: mmio + HOST_EM_CTL);
1107 return 0;
1108}
1109EXPORT_SYMBOL_GPL(ahci_reset_em);
1110
1111static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
1112 ssize_t size)
1113{
1114 struct ahci_host_priv *hpriv = ap->host->private_data;
1115 struct ahci_port_priv *pp = ap->private_data;
1116 void __iomem *mmio = hpriv->mmio;
1117 u32 em_ctl;
1118 u32 message[] = {0, 0};
1119 unsigned long flags;
1120 int pmp;
1121 struct ahci_em_priv *emp;
1122
1123 /* get the slot number from the message */
1124 pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1125 if (pmp < EM_MAX_SLOTS)
1126 emp = &pp->em_priv[pmp];
1127 else
1128 return -EINVAL;
1129
1130 ahci_rpm_get_port(ap);
1131 spin_lock_irqsave(ap->lock, flags);
1132
1133 /*
1134 * if we are still busy transmitting a previous message,
1135 * do not allow
1136 */
1137 em_ctl = readl(addr: mmio + HOST_EM_CTL);
1138 if (em_ctl & EM_CTL_TM) {
1139 spin_unlock_irqrestore(lock: ap->lock, flags);
1140 ahci_rpm_put_port(ap);
1141 return -EBUSY;
1142 }
1143
1144 if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
1145 /*
1146 * create message header - this is all zero except for
1147 * the message size, which is 4 bytes.
1148 */
1149 message[0] |= (4 << 8);
1150
1151 /* ignore 0:4 of byte zero, fill in port info yourself */
1152 message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
1153
1154 /* write message to EM_LOC */
1155 writel(val: message[0], addr: mmio + hpriv->em_loc);
1156 writel(val: message[1], addr: mmio + hpriv->em_loc+4);
1157
1158 /*
1159 * tell hardware to transmit the message
1160 */
1161 writel(val: em_ctl | EM_CTL_TM, addr: mmio + HOST_EM_CTL);
1162 }
1163
1164 /* save off new led state for port/slot */
1165 emp->led_state = state;
1166
1167 spin_unlock_irqrestore(lock: ap->lock, flags);
1168 ahci_rpm_put_port(ap);
1169
1170 return size;
1171}
1172
1173static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
1174{
1175 struct ahci_port_priv *pp = ap->private_data;
1176 struct ata_link *link;
1177 struct ahci_em_priv *emp;
1178 int rc = 0;
1179
1180 ata_for_each_link(link, ap, EDGE) {
1181 emp = &pp->em_priv[link->pmp];
1182 rc += sprintf(buf, fmt: "%lx\n", emp->led_state);
1183 }
1184 return rc;
1185}
1186
1187static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1188 size_t size)
1189{
1190 unsigned int state;
1191 int pmp;
1192 struct ahci_port_priv *pp = ap->private_data;
1193 struct ahci_em_priv *emp;
1194
1195 if (kstrtouint(s: buf, base: 0, res: &state) < 0)
1196 return -EINVAL;
1197
1198 /* get the slot number from the message */
1199 pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1200 if (pmp < EM_MAX_SLOTS) {
1201 pmp = array_index_nospec(pmp, EM_MAX_SLOTS);
1202 emp = &pp->em_priv[pmp];
1203 } else {
1204 return -EINVAL;
1205 }
1206
1207 /* mask off the activity bits if we are in sw_activity
1208 * mode, user should turn off sw_activity before setting
1209 * activity led through em_message
1210 */
1211 if (emp->blink_policy)
1212 state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1213
1214 return ap->ops->transmit_led_message(ap, state, size);
1215}
1216
1217static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1218{
1219 struct ata_link *link = dev->link;
1220 struct ata_port *ap = link->ap;
1221 struct ahci_port_priv *pp = ap->private_data;
1222 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1223 u32 port_led_state = emp->led_state;
1224
1225 /* save the desired Activity LED behavior */
1226 if (val == OFF) {
1227 /* clear LFLAG */
1228 link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1229
1230 /* set the LED to OFF */
1231 port_led_state &= EM_MSG_LED_VALUE_OFF;
1232 port_led_state |= (ap->port_no | (link->pmp << 8));
1233 ap->ops->transmit_led_message(ap, port_led_state, 4);
1234 } else {
1235 link->flags |= ATA_LFLAG_SW_ACTIVITY;
1236 if (val == BLINK_OFF) {
1237 /* set LED to ON for idle */
1238 port_led_state &= EM_MSG_LED_VALUE_OFF;
1239 port_led_state |= (ap->port_no | (link->pmp << 8));
1240 port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
1241 ap->ops->transmit_led_message(ap, port_led_state, 4);
1242 }
1243 }
1244 emp->blink_policy = val;
1245 return 0;
1246}
1247
1248static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1249{
1250 struct ata_link *link = dev->link;
1251 struct ata_port *ap = link->ap;
1252 struct ahci_port_priv *pp = ap->private_data;
1253 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1254
1255 /* display the saved value of activity behavior for this
1256 * disk.
1257 */
1258 return sprintf(buf, fmt: "%d\n", emp->blink_policy);
1259}
1260
1261static void ahci_port_clear_pending_irq(struct ata_port *ap)
1262{
1263 struct ahci_host_priv *hpriv = ap->host->private_data;
1264 void __iomem *port_mmio = ahci_port_base(ap);
1265 u32 tmp;
1266
1267 /* clear SError */
1268 tmp = readl(addr: port_mmio + PORT_SCR_ERR);
1269 dev_dbg(ap->host->dev, "PORT_SCR_ERR 0x%x\n", tmp);
1270 writel(val: tmp, addr: port_mmio + PORT_SCR_ERR);
1271
1272 /* clear port IRQ */
1273 tmp = readl(addr: port_mmio + PORT_IRQ_STAT);
1274 dev_dbg(ap->host->dev, "PORT_IRQ_STAT 0x%x\n", tmp);
1275 if (tmp)
1276 writel(val: tmp, addr: port_mmio + PORT_IRQ_STAT);
1277
1278 writel(val: 1 << ap->port_no, addr: hpriv->mmio + HOST_IRQ_STAT);
1279}
1280
1281static void ahci_port_init(struct device *dev, struct ata_port *ap,
1282 int port_no, void __iomem *mmio,
1283 void __iomem *port_mmio)
1284{
1285 const char *emsg = NULL;
1286 int rc;
1287
1288 /* make sure port is not active */
1289 rc = ahci_deinit_port(ap, emsg: &emsg);
1290 if (rc)
1291 dev_warn(dev, "%s (%d)\n", emsg, rc);
1292
1293 ahci_port_clear_pending_irq(ap);
1294}
1295
1296void ahci_init_controller(struct ata_host *host)
1297{
1298 struct ahci_host_priv *hpriv = host->private_data;
1299 void __iomem *mmio = hpriv->mmio;
1300 int i;
1301 void __iomem *port_mmio;
1302 u32 tmp;
1303
1304 for (i = 0; i < host->n_ports; i++) {
1305 struct ata_port *ap = host->ports[i];
1306
1307 port_mmio = ahci_port_base(ap);
1308 if (ata_port_is_dummy(ap))
1309 continue;
1310
1311 ahci_port_init(dev: host->dev, ap, port_no: i, mmio, port_mmio);
1312 }
1313
1314 tmp = readl(addr: mmio + HOST_CTL);
1315 dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);
1316 writel(val: tmp | HOST_IRQ_EN, addr: mmio + HOST_CTL);
1317 tmp = readl(addr: mmio + HOST_CTL);
1318 dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);
1319}
1320EXPORT_SYMBOL_GPL(ahci_init_controller);
1321
1322static void ahci_dev_config(struct ata_device *dev)
1323{
1324 struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1325
1326 if ((dev->class == ATA_DEV_ATAPI) &&
1327 (hpriv->flags & AHCI_HFLAG_ATAPI_DMA_QUIRK))
1328 dev->quirks |= ATA_QUIRK_ATAPI_MOD16_DMA;
1329
1330 if (hpriv->flags & AHCI_HFLAG_SECT255) {
1331 dev->max_sectors = 255;
1332 ata_dev_info(dev,
1333 "SB600 AHCI: limiting to 255 sectors per cmd\n");
1334 }
1335}
1336
1337unsigned int ahci_dev_classify(struct ata_port *ap)
1338{
1339 void __iomem *port_mmio = ahci_port_base(ap);
1340 struct ata_taskfile tf;
1341 u32 tmp;
1342
1343 tmp = readl(addr: port_mmio + PORT_SIG);
1344 tf.lbah = (tmp >> 24) & 0xff;
1345 tf.lbam = (tmp >> 16) & 0xff;
1346 tf.lbal = (tmp >> 8) & 0xff;
1347 tf.nsect = (tmp) & 0xff;
1348
1349 return ata_port_classify(ap, tf: &tf);
1350}
1351EXPORT_SYMBOL_GPL(ahci_dev_classify);
1352
1353void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1354 u32 opts)
1355{
1356 dma_addr_t cmd_tbl_dma;
1357
1358 cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1359
1360 pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1361 pp->cmd_slot[tag].status = 0;
1362 pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1363 pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1364}
1365EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1366
1367int ahci_kick_engine(struct ata_port *ap)
1368{
1369 void __iomem *port_mmio = ahci_port_base(ap);
1370 struct ahci_host_priv *hpriv = ap->host->private_data;
1371 u8 status = readl(addr: port_mmio + PORT_TFDATA) & 0xFF;
1372 u32 tmp;
1373 int busy, rc;
1374
1375 /* stop engine */
1376 rc = hpriv->stop_engine(ap);
1377 if (rc)
1378 goto out_restart;
1379
1380 /* need to do CLO?
1381 * always do CLO if PMP is attached (AHCI-1.3 9.2)
1382 */
1383 busy = status & (ATA_BUSY | ATA_DRQ);
1384 if (!busy && !sata_pmp_attached(ap)) {
1385 rc = 0;
1386 goto out_restart;
1387 }
1388
1389 if (!(hpriv->cap & HOST_CAP_CLO)) {
1390 rc = -EOPNOTSUPP;
1391 goto out_restart;
1392 }
1393
1394 /* perform CLO */
1395 tmp = readl(addr: port_mmio + PORT_CMD);
1396 tmp |= PORT_CMD_CLO;
1397 writel(val: tmp, addr: port_mmio + PORT_CMD);
1398
1399 rc = 0;
1400 tmp = ata_wait_register(ap, reg: port_mmio + PORT_CMD,
1401 mask: PORT_CMD_CLO, val: PORT_CMD_CLO, interval: 1, timeout: 500);
1402 if (tmp & PORT_CMD_CLO)
1403 rc = -EIO;
1404
1405 /* restart engine */
1406 out_restart:
1407 hpriv->start_engine(ap);
1408 return rc;
1409}
1410EXPORT_SYMBOL_GPL(ahci_kick_engine);
1411
1412static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1413 struct ata_taskfile *tf, int is_cmd, u16 flags,
1414 unsigned int timeout_msec)
1415{
1416 const u32 cmd_fis_len = 5; /* five dwords */
1417 struct ahci_port_priv *pp = ap->private_data;
1418 void __iomem *port_mmio = ahci_port_base(ap);
1419 u8 *fis = pp->cmd_tbl;
1420 u32 tmp;
1421
1422 /* prep the command */
1423 ata_tf_to_fis(tf, pmp, is_cmd, fis);
1424 ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1425
1426 /* set port value for softreset of Port Multiplier */
1427 if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {
1428 tmp = readl(addr: port_mmio + PORT_FBS);
1429 tmp &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
1430 tmp |= pmp << PORT_FBS_DEV_OFFSET;
1431 writel(val: tmp, addr: port_mmio + PORT_FBS);
1432 pp->fbs_last_dev = pmp;
1433 }
1434
1435 /* issue & wait */
1436 writel(val: 1, addr: port_mmio + PORT_CMD_ISSUE);
1437
1438 if (timeout_msec) {
1439 tmp = ata_wait_register(ap, reg: port_mmio + PORT_CMD_ISSUE,
1440 mask: 0x1, val: 0x1, interval: 1, timeout: timeout_msec);
1441 if (tmp & 0x1) {
1442 ahci_kick_engine(ap);
1443 return -EBUSY;
1444 }
1445 } else
1446 readl(addr: port_mmio + PORT_CMD_ISSUE); /* flush */
1447
1448 return 0;
1449}
1450
1451int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1452 int pmp, unsigned long deadline,
1453 int (*check_ready)(struct ata_link *link))
1454{
1455 struct ata_port *ap = link->ap;
1456 struct ahci_host_priv *hpriv = ap->host->private_data;
1457 struct ahci_port_priv *pp = ap->private_data;
1458 const char *reason = NULL;
1459 unsigned long now;
1460 unsigned int msecs;
1461 struct ata_taskfile tf;
1462 bool fbs_disabled = false;
1463 int rc;
1464
1465 /* prepare for SRST (AHCI-1.1 10.4.1) */
1466 rc = ahci_kick_engine(ap);
1467 if (rc && rc != -EOPNOTSUPP)
1468 ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1469
1470 /*
1471 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1472 * clear PxFBS.EN to '0' prior to issuing software reset to devices
1473 * that is attached to port multiplier.
1474 */
1475 if (!ata_is_host_link(link) && pp->fbs_enabled) {
1476 ahci_disable_fbs(ap);
1477 fbs_disabled = true;
1478 }
1479
1480 ata_tf_init(dev: link->device, tf: &tf);
1481
1482 /* issue the first H2D Register FIS */
1483 msecs = 0;
1484 now = jiffies;
1485 if (time_after(deadline, now))
1486 msecs = jiffies_to_msecs(j: deadline - now);
1487
1488 tf.ctl |= ATA_SRST;
1489 if (ahci_exec_polled_cmd(ap, pmp, tf: &tf, is_cmd: 0,
1490 flags: AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, timeout_msec: msecs)) {
1491 rc = -EIO;
1492 reason = "1st FIS failed";
1493 goto fail;
1494 }
1495
1496 /* spec says at least 5us, but be generous and sleep for 1ms */
1497 ata_msleep(ap, msecs: 1);
1498
1499 /* issue the second H2D Register FIS */
1500 tf.ctl &= ~ATA_SRST;
1501 ahci_exec_polled_cmd(ap, pmp, tf: &tf, is_cmd: 0, flags: 0, timeout_msec: 0);
1502
1503 /* wait for link to become ready */
1504 rc = ata_wait_after_reset(link, deadline, check_ready);
1505 if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1506 /*
1507 * Workaround for cases where link online status can't
1508 * be trusted. Treat device readiness timeout as link
1509 * offline.
1510 */
1511 ata_link_info(link, "device not ready, treating as offline\n");
1512 *class = ATA_DEV_NONE;
1513 } else if (rc) {
1514 /* link occupied, -ENODEV too is an error */
1515 reason = "device not ready";
1516 goto fail;
1517 } else
1518 *class = ahci_dev_classify(ap);
1519
1520 /* re-enable FBS if disabled before */
1521 if (fbs_disabled)
1522 ahci_enable_fbs(ap);
1523
1524 return 0;
1525
1526 fail:
1527 ata_link_err(link, "softreset failed (%s)\n", reason);
1528 return rc;
1529}
1530
1531int ahci_check_ready(struct ata_link *link)
1532{
1533 void __iomem *port_mmio = ahci_port_base(ap: link->ap);
1534 u8 status = readl(addr: port_mmio + PORT_TFDATA) & 0xFF;
1535
1536 return ata_check_ready(status);
1537}
1538EXPORT_SYMBOL_GPL(ahci_check_ready);
1539
1540static int ahci_softreset(struct ata_link *link, unsigned int *class,
1541 unsigned long deadline)
1542{
1543 int pmp = sata_srst_pmp(link);
1544
1545 return ahci_do_softreset(link, class, pmp, deadline, check_ready: ahci_check_ready);
1546}
1547EXPORT_SYMBOL_GPL(ahci_do_softreset);
1548
1549static int ahci_bad_pmp_check_ready(struct ata_link *link)
1550{
1551 void __iomem *port_mmio = ahci_port_base(ap: link->ap);
1552 u8 status = readl(addr: port_mmio + PORT_TFDATA) & 0xFF;
1553 u32 irq_status = readl(addr: port_mmio + PORT_IRQ_STAT);
1554
1555 /*
1556 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1557 * which can save timeout delay.
1558 */
1559 if (irq_status & PORT_IRQ_BAD_PMP)
1560 return -EIO;
1561
1562 return ata_check_ready(status);
1563}
1564
1565static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
1566 unsigned long deadline)
1567{
1568 struct ata_port *ap = link->ap;
1569 void __iomem *port_mmio = ahci_port_base(ap);
1570 int pmp = sata_srst_pmp(link);
1571 int rc;
1572 u32 irq_sts;
1573
1574 rc = ahci_do_softreset(link, class, pmp, deadline,
1575 ahci_bad_pmp_check_ready);
1576
1577 /*
1578 * Soft reset fails with IPMS set when PMP is enabled but
1579 * SATA HDD/ODD is connected to SATA port, do soft reset
1580 * again to port 0.
1581 */
1582 if (rc == -EIO) {
1583 irq_sts = readl(addr: port_mmio + PORT_IRQ_STAT);
1584 if (irq_sts & PORT_IRQ_BAD_PMP) {
1585 ata_link_warn(link,
1586 "applying PMP SRST workaround "
1587 "and retrying\n");
1588 rc = ahci_do_softreset(link, class, 0, deadline,
1589 ahci_check_ready);
1590 }
1591 }
1592
1593 return rc;
1594}
1595
1596int ahci_do_hardreset(struct ata_link *link, unsigned int *class,
1597 unsigned long deadline, bool *online)
1598{
1599 const unsigned int *timing = sata_ehc_deb_timing(ehc: &link->eh_context);
1600 struct ata_port *ap = link->ap;
1601 struct ahci_port_priv *pp = ap->private_data;
1602 struct ahci_host_priv *hpriv = ap->host->private_data;
1603 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1604 struct ata_taskfile tf;
1605 int rc;
1606
1607 hpriv->stop_engine(ap);
1608
1609 /* clear D2H reception area to properly wait for D2H FIS */
1610 ata_tf_init(dev: link->device, tf: &tf);
1611 tf.status = ATA_BUSY;
1612 ata_tf_to_fis(tf: &tf, pmp: 0, is_cmd: 0, fis: d2h_fis);
1613
1614 ahci_port_clear_pending_irq(ap);
1615
1616 rc = sata_link_hardreset(link, timing, deadline, online,
1617 check_ready: ahci_check_ready);
1618
1619 hpriv->start_engine(ap);
1620
1621 if (*online)
1622 *class = ahci_dev_classify(ap);
1623
1624 return rc;
1625}
1626EXPORT_SYMBOL_GPL(ahci_do_hardreset);
1627
1628static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1629 unsigned long deadline)
1630{
1631 bool online;
1632
1633 return ahci_do_hardreset(link, class, deadline, &online);
1634}
1635
1636static void ahci_postreset(struct ata_link *link, unsigned int *class)
1637{
1638 struct ata_port *ap = link->ap;
1639 void __iomem *port_mmio = ahci_port_base(ap);
1640 u32 new_tmp, tmp;
1641
1642 ata_std_postreset(link, classes: class);
1643
1644 /* Make sure port's ATAPI bit is set appropriately */
1645 new_tmp = tmp = readl(addr: port_mmio + PORT_CMD);
1646 if (*class == ATA_DEV_ATAPI)
1647 new_tmp |= PORT_CMD_ATAPI;
1648 else
1649 new_tmp &= ~PORT_CMD_ATAPI;
1650 if (new_tmp != tmp) {
1651 writel(val: new_tmp, addr: port_mmio + PORT_CMD);
1652 readl(addr: port_mmio + PORT_CMD); /* flush */
1653 }
1654}
1655
1656static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1657{
1658 struct scatterlist *sg;
1659 struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1660 unsigned int si;
1661
1662 /*
1663 * Next, the S/G list.
1664 */
1665 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1666 dma_addr_t addr = sg_dma_address(sg);
1667 u32 sg_len = sg_dma_len(sg);
1668
1669 ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1670 ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1671 ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1672 }
1673
1674 return si;
1675}
1676
1677static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1678{
1679 struct ata_port *ap = qc->ap;
1680 struct ahci_port_priv *pp = ap->private_data;
1681
1682 if (!sata_pmp_attached(ap) || pp->fbs_enabled)
1683 return ata_std_qc_defer(qc);
1684 else
1685 return sata_pmp_qc_defer_cmd_switch(qc);
1686}
1687
1688static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc)
1689{
1690 struct ata_port *ap = qc->ap;
1691 struct ahci_port_priv *pp = ap->private_data;
1692 int is_atapi = ata_is_atapi(prot: qc->tf.protocol);
1693 void *cmd_tbl;
1694 u32 opts;
1695 const u32 cmd_fis_len = 5; /* five dwords */
1696 unsigned int n_elem;
1697
1698 /*
1699 * Fill in command table information. First, the header,
1700 * a SATA Register - Host to Device command FIS.
1701 */
1702 cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
1703
1704 ata_tf_to_fis(tf: &qc->tf, pmp: qc->dev->link->pmp, is_cmd: 1, fis: cmd_tbl);
1705 if (is_atapi) {
1706 memset(s: cmd_tbl + AHCI_CMD_TBL_CDB, c: 0, n: 32);
1707 memcpy(to: cmd_tbl + AHCI_CMD_TBL_CDB, from: qc->cdb, len: qc->dev->cdb_len);
1708 }
1709
1710 n_elem = 0;
1711 if (qc->flags & ATA_QCFLAG_DMAMAP)
1712 n_elem = ahci_fill_sg(qc, cmd_tbl);
1713
1714 /*
1715 * Fill in command slot information.
1716 */
1717 opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1718 if (qc->tf.flags & ATA_TFLAG_WRITE)
1719 opts |= AHCI_CMD_WRITE;
1720 if (is_atapi)
1721 opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1722
1723 ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
1724
1725 return AC_ERR_OK;
1726}
1727
1728static void ahci_fbs_dec_intr(struct ata_port *ap)
1729{
1730 struct ahci_port_priv *pp = ap->private_data;
1731 void __iomem *port_mmio = ahci_port_base(ap);
1732 u32 fbs = readl(addr: port_mmio + PORT_FBS);
1733 int retries = 3;
1734
1735 BUG_ON(!pp->fbs_enabled);
1736
1737 /* time to wait for DEC is not specified by AHCI spec,
1738 * add a retry loop for safety.
1739 */
1740 writel(val: fbs | PORT_FBS_DEC, addr: port_mmio + PORT_FBS);
1741 fbs = readl(addr: port_mmio + PORT_FBS);
1742 while ((fbs & PORT_FBS_DEC) && retries--) {
1743 udelay(usec: 1);
1744 fbs = readl(addr: port_mmio + PORT_FBS);
1745 }
1746
1747 if (fbs & PORT_FBS_DEC)
1748 dev_err(ap->host->dev, "failed to clear device error\n");
1749}
1750
1751static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1752{
1753 struct ahci_host_priv *hpriv = ap->host->private_data;
1754 struct ahci_port_priv *pp = ap->private_data;
1755 struct ata_eh_info *host_ehi = &ap->link.eh_info;
1756 struct ata_link *link = NULL;
1757 struct ata_queued_cmd *active_qc;
1758 struct ata_eh_info *active_ehi;
1759 bool fbs_need_dec = false;
1760 u32 serror;
1761
1762 /* determine active link with error */
1763 if (pp->fbs_enabled) {
1764 void __iomem *port_mmio = ahci_port_base(ap);
1765 u32 fbs = readl(addr: port_mmio + PORT_FBS);
1766 int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1767
1768 if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
1769 link = &ap->pmp_link[pmp];
1770 fbs_need_dec = true;
1771 }
1772
1773 } else
1774 ata_for_each_link(link, ap, EDGE)
1775 if (ata_link_active(link))
1776 break;
1777
1778 if (!link)
1779 link = &ap->link;
1780
1781 active_qc = ata_qc_from_tag(ap, tag: link->active_tag);
1782 active_ehi = &link->eh_info;
1783
1784 /* record irq stat */
1785 ata_ehi_clear_desc(ehi: host_ehi);
1786 ata_ehi_push_desc(ehi: host_ehi, fmt: "irq_stat 0x%08x", irq_stat);
1787
1788 /* AHCI needs SError cleared; otherwise, it might lock up */
1789 ahci_scr_read(link: &ap->link, sc_reg: SCR_ERROR, val: &serror);
1790 ahci_scr_write(link: &ap->link, sc_reg: SCR_ERROR, val: serror);
1791 host_ehi->serror |= serror;
1792
1793 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
1794 if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1795 irq_stat &= ~PORT_IRQ_IF_ERR;
1796
1797 if (irq_stat & PORT_IRQ_TF_ERR) {
1798 /* If qc is active, charge it; otherwise, the active
1799 * link. There's no active qc on NCQ errors. It will
1800 * be determined by EH by reading log page 10h.
1801 */
1802 if (active_qc)
1803 active_qc->err_mask |= AC_ERR_DEV;
1804 else
1805 active_ehi->err_mask |= AC_ERR_DEV;
1806
1807 if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1808 host_ehi->serror &= ~SERR_INTERNAL;
1809 }
1810
1811 if (irq_stat & PORT_IRQ_UNK_FIS) {
1812 u32 *unk = pp->rx_fis + RX_FIS_UNK;
1813
1814 active_ehi->err_mask |= AC_ERR_HSM;
1815 active_ehi->action |= ATA_EH_RESET;
1816 ata_ehi_push_desc(ehi: active_ehi,
1817 fmt: "unknown FIS %08x %08x %08x %08x" ,
1818 unk[0], unk[1], unk[2], unk[3]);
1819 }
1820
1821 if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1822 active_ehi->err_mask |= AC_ERR_HSM;
1823 active_ehi->action |= ATA_EH_RESET;
1824 ata_ehi_push_desc(ehi: active_ehi, fmt: "incorrect PMP");
1825 }
1826
1827 if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1828 host_ehi->err_mask |= AC_ERR_HOST_BUS;
1829 host_ehi->action |= ATA_EH_RESET;
1830 ata_ehi_push_desc(ehi: host_ehi, fmt: "host bus error");
1831 }
1832
1833 if (irq_stat & PORT_IRQ_IF_ERR) {
1834 if (fbs_need_dec)
1835 active_ehi->err_mask |= AC_ERR_DEV;
1836 else {
1837 host_ehi->err_mask |= AC_ERR_ATA_BUS;
1838 host_ehi->action |= ATA_EH_RESET;
1839 }
1840
1841 ata_ehi_push_desc(ehi: host_ehi, fmt: "interface fatal error");
1842 }
1843
1844 if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1845 ata_ehi_hotplugged(ehi: host_ehi);
1846 ata_ehi_push_desc(ehi: host_ehi, fmt: "%s",
1847 irq_stat & PORT_IRQ_CONNECT ?
1848 "connection status changed" : "PHY RDY changed");
1849 }
1850
1851 /* okay, let's hand over to EH */
1852
1853 if (irq_stat & PORT_IRQ_FREEZE)
1854 ata_port_freeze(ap);
1855 else if (fbs_need_dec) {
1856 ata_link_abort(link);
1857 ahci_fbs_dec_intr(ap);
1858 } else
1859 ata_port_abort(ap);
1860}
1861
1862static void ahci_qc_complete(struct ata_port *ap, void __iomem *port_mmio)
1863{
1864 struct ata_eh_info *ehi = &ap->link.eh_info;
1865 struct ahci_port_priv *pp = ap->private_data;
1866 u32 qc_active = 0;
1867 int rc;
1868
1869 /*
1870 * pp->active_link is not reliable once FBS is enabled, both
1871 * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1872 * NCQ and non-NCQ commands may be in flight at the same time.
1873 */
1874 if (pp->fbs_enabled) {
1875 if (ap->qc_active) {
1876 qc_active = readl(addr: port_mmio + PORT_SCR_ACT);
1877 qc_active |= readl(addr: port_mmio + PORT_CMD_ISSUE);
1878 }
1879 } else {
1880 /* pp->active_link is valid iff any command is in flight */
1881 if (ap->qc_active && pp->active_link->sactive)
1882 qc_active = readl(addr: port_mmio + PORT_SCR_ACT);
1883 else
1884 qc_active = readl(addr: port_mmio + PORT_CMD_ISSUE);
1885 }
1886
1887 rc = ata_qc_complete_multiple(ap, qc_active);
1888 if (unlikely(rc < 0 && !(ap->pflags & ATA_PFLAG_RESETTING))) {
1889 ehi->err_mask |= AC_ERR_HSM;
1890 ehi->action |= ATA_EH_RESET;
1891 ata_port_freeze(ap);
1892 }
1893}
1894
1895static void ahci_handle_port_interrupt(struct ata_port *ap,
1896 void __iomem *port_mmio, u32 status)
1897{
1898 struct ahci_port_priv *pp = ap->private_data;
1899 struct ahci_host_priv *hpriv = ap->host->private_data;
1900
1901 /* ignore BAD_PMP while resetting */
1902 if (unlikely(ap->pflags & ATA_PFLAG_RESETTING))
1903 status &= ~PORT_IRQ_BAD_PMP;
1904
1905 if (sata_lpm_ignore_phy_events(link: &ap->link)) {
1906 status &= ~PORT_IRQ_PHYRDY;
1907 ahci_scr_write(link: &ap->link, sc_reg: SCR_ERROR, val: SERR_PHYRDY_CHG);
1908 }
1909
1910 if (unlikely(status & PORT_IRQ_ERROR)) {
1911 /*
1912 * Before getting the error notification, we may have
1913 * received SDB FISes notifying successful completions.
1914 * Handle these first and then handle the error.
1915 */
1916 ahci_qc_complete(ap, port_mmio);
1917 ahci_error_intr(ap, irq_stat: status);
1918 return;
1919 }
1920
1921 if (status & PORT_IRQ_SDB_FIS) {
1922 /* If SNotification is available, leave notification
1923 * handling to sata_async_notification(). If not,
1924 * emulate it by snooping SDB FIS RX area.
1925 *
1926 * Snooping FIS RX area is probably cheaper than
1927 * poking SNotification but some constrollers which
1928 * implement SNotification, ICH9 for example, don't
1929 * store AN SDB FIS into receive area.
1930 */
1931 if (hpriv->cap & HOST_CAP_SNTF)
1932 sata_async_notification(ap);
1933 else {
1934 /* If the 'N' bit in word 0 of the FIS is set,
1935 * we just received asynchronous notification.
1936 * Tell libata about it.
1937 *
1938 * Lack of SNotification should not appear in
1939 * ahci 1.2, so the workaround is unnecessary
1940 * when FBS is enabled.
1941 */
1942 if (pp->fbs_enabled)
1943 WARN_ON_ONCE(1);
1944 else {
1945 const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1946 u32 f0 = le32_to_cpu(f[0]);
1947 if (f0 & (1 << 15))
1948 sata_async_notification(ap);
1949 }
1950 }
1951 }
1952
1953 /* Handle completed commands */
1954 ahci_qc_complete(ap, port_mmio);
1955}
1956
1957static void ahci_port_intr(struct ata_port *ap)
1958{
1959 void __iomem *port_mmio = ahci_port_base(ap);
1960 u32 status;
1961
1962 status = readl(addr: port_mmio + PORT_IRQ_STAT);
1963 writel(val: status, addr: port_mmio + PORT_IRQ_STAT);
1964
1965 ahci_handle_port_interrupt(ap, port_mmio, status);
1966}
1967
1968static irqreturn_t ahci_multi_irqs_intr_hard(int irq, void *dev_instance)
1969{
1970 struct ata_port *ap = dev_instance;
1971 void __iomem *port_mmio = ahci_port_base(ap);
1972 u32 status;
1973
1974 status = readl(addr: port_mmio + PORT_IRQ_STAT);
1975 writel(val: status, addr: port_mmio + PORT_IRQ_STAT);
1976
1977 spin_lock(lock: ap->lock);
1978 ahci_handle_port_interrupt(ap, port_mmio, status);
1979 spin_unlock(lock: ap->lock);
1980
1981 return IRQ_HANDLED;
1982}
1983
1984u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked)
1985{
1986 unsigned int i, handled = 0;
1987
1988 for (i = 0; i < host->n_ports; i++) {
1989 struct ata_port *ap;
1990
1991 if (!(irq_masked & (1 << i)))
1992 continue;
1993
1994 ap = host->ports[i];
1995 if (ap) {
1996 ahci_port_intr(ap);
1997 } else {
1998 if (ata_ratelimit())
1999 dev_warn(host->dev,
2000 "interrupt on disabled port %u\n", i);
2001 }
2002
2003 handled = 1;
2004 }
2005
2006 return handled;
2007}
2008EXPORT_SYMBOL_GPL(ahci_handle_port_intr);
2009
2010static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance)
2011{
2012 struct ata_host *host = dev_instance;
2013 struct ahci_host_priv *hpriv;
2014 unsigned int rc = 0;
2015 void __iomem *mmio;
2016 u32 irq_stat, irq_masked;
2017
2018 hpriv = host->private_data;
2019 mmio = hpriv->mmio;
2020
2021 /* sigh. 0xffffffff is a valid return from h/w */
2022 irq_stat = readl(addr: mmio + HOST_IRQ_STAT);
2023 if (!irq_stat)
2024 return IRQ_NONE;
2025
2026 irq_masked = irq_stat & hpriv->port_map;
2027
2028 spin_lock(lock: &host->lock);
2029
2030 rc = ahci_handle_port_intr(host, irq_masked);
2031
2032 /* HOST_IRQ_STAT behaves as level triggered latch meaning that
2033 * it should be cleared after all the port events are cleared;
2034 * otherwise, it will raise a spurious interrupt after each
2035 * valid one. Please read section 10.6.2 of ahci 1.1 for more
2036 * information.
2037 *
2038 * Also, use the unmasked value to clear interrupt as spurious
2039 * pending event on a dummy port might cause screaming IRQ.
2040 */
2041 writel(val: irq_stat, addr: mmio + HOST_IRQ_STAT);
2042
2043 spin_unlock(lock: &host->lock);
2044
2045 return IRQ_RETVAL(rc);
2046}
2047
2048unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
2049{
2050 struct ata_port *ap = qc->ap;
2051 void __iomem *port_mmio = ahci_port_base(ap);
2052 struct ahci_port_priv *pp = ap->private_data;
2053
2054 /* Keep track of the currently active link. It will be used
2055 * in completion path to determine whether NCQ phase is in
2056 * progress.
2057 */
2058 pp->active_link = qc->dev->link;
2059
2060 if (ata_is_ncq(prot: qc->tf.protocol))
2061 writel(val: 1 << qc->hw_tag, addr: port_mmio + PORT_SCR_ACT);
2062
2063 if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
2064 u32 fbs = readl(addr: port_mmio + PORT_FBS);
2065 fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
2066 fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
2067 writel(val: fbs, addr: port_mmio + PORT_FBS);
2068 pp->fbs_last_dev = qc->dev->link->pmp;
2069 }
2070
2071 writel(val: 1 << qc->hw_tag, addr: port_mmio + PORT_CMD_ISSUE);
2072
2073 ahci_sw_activity(link: qc->dev->link);
2074
2075 return 0;
2076}
2077EXPORT_SYMBOL_GPL(ahci_qc_issue);
2078
2079static void ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
2080{
2081 struct ahci_port_priv *pp = qc->ap->private_data;
2082 u8 *rx_fis = pp->rx_fis;
2083
2084 if (pp->fbs_enabled)
2085 rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
2086
2087 /*
2088 * After a successful execution of an ATA PIO data-in command,
2089 * the device doesn't send D2H Reg FIS to update the TF and
2090 * the host should take TF and E_Status from the preceding PIO
2091 * Setup FIS.
2092 */
2093 if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
2094 !(qc->flags & ATA_QCFLAG_EH)) {
2095 ata_tf_from_fis(fis: rx_fis + RX_FIS_PIO_SETUP, tf: &qc->result_tf);
2096 qc->result_tf.status = (rx_fis + RX_FIS_PIO_SETUP)[15];
2097 return;
2098 }
2099
2100 /*
2101 * For NCQ commands, we never get a D2H FIS, so reading the D2H Register
2102 * FIS area of the Received FIS Structure (which contains a copy of the
2103 * last D2H FIS received) will contain an outdated status code.
2104 * For NCQ commands, we instead get a SDB FIS, so read the SDB FIS area
2105 * instead. However, the SDB FIS does not contain the LBA, so we can't
2106 * use the ata_tf_from_fis() helper.
2107 */
2108 if (ata_is_ncq(prot: qc->tf.protocol)) {
2109 const u8 *fis = rx_fis + RX_FIS_SDB;
2110
2111 /*
2112 * Successful NCQ commands have been filled already.
2113 * A failed NCQ command will read the status here.
2114 * (Note that a failed NCQ command will get a more specific
2115 * error when reading the NCQ Command Error log.)
2116 */
2117 qc->result_tf.status = fis[2];
2118 qc->result_tf.error = fis[3];
2119 return;
2120 }
2121
2122 ata_tf_from_fis(fis: rx_fis + RX_FIS_D2H_REG, tf: &qc->result_tf);
2123}
2124
2125static void ahci_qc_ncq_fill_rtf(struct ata_port *ap, u64 done_mask)
2126{
2127 struct ahci_port_priv *pp = ap->private_data;
2128 const u8 *fis;
2129
2130 /* No outstanding commands. */
2131 if (!ap->qc_active)
2132 return;
2133
2134 /*
2135 * FBS not enabled, so read status and error once, since they are shared
2136 * for all QCs.
2137 */
2138 if (!pp->fbs_enabled) {
2139 u8 status, error;
2140
2141 /* No outstanding NCQ commands. */
2142 if (!pp->active_link->sactive)
2143 return;
2144
2145 fis = pp->rx_fis + RX_FIS_SDB;
2146 status = fis[2];
2147 error = fis[3];
2148
2149 while (done_mask) {
2150 struct ata_queued_cmd *qc;
2151 unsigned int tag = __ffs64(word: done_mask);
2152
2153 qc = ata_qc_from_tag(ap, tag);
2154 if (qc && ata_is_ncq(prot: qc->tf.protocol)) {
2155 qc->result_tf.status = status;
2156 qc->result_tf.error = error;
2157 qc->result_tf.flags = qc->tf.flags;
2158 qc->flags |= ATA_QCFLAG_RTF_FILLED;
2159 }
2160 done_mask &= ~(1ULL << tag);
2161 }
2162
2163 return;
2164 }
2165
2166 /*
2167 * FBS enabled, so read the status and error for each QC, since the QCs
2168 * can belong to different PMP links. (Each PMP link has its own FIS
2169 * Receive Area.)
2170 */
2171 while (done_mask) {
2172 struct ata_queued_cmd *qc;
2173 unsigned int tag = __ffs64(word: done_mask);
2174
2175 qc = ata_qc_from_tag(ap, tag);
2176 if (qc && ata_is_ncq(prot: qc->tf.protocol)) {
2177 fis = pp->rx_fis;
2178 fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
2179 fis += RX_FIS_SDB;
2180 qc->result_tf.status = fis[2];
2181 qc->result_tf.error = fis[3];
2182 qc->result_tf.flags = qc->tf.flags;
2183 qc->flags |= ATA_QCFLAG_RTF_FILLED;
2184 }
2185 done_mask &= ~(1ULL << tag);
2186 }
2187}
2188
2189static void ahci_freeze(struct ata_port *ap)
2190{
2191 void __iomem *port_mmio = ahci_port_base(ap);
2192
2193 /* turn IRQ off */
2194 writel(val: 0, addr: port_mmio + PORT_IRQ_MASK);
2195}
2196
2197static void ahci_thaw(struct ata_port *ap)
2198{
2199 struct ahci_host_priv *hpriv = ap->host->private_data;
2200 void __iomem *mmio = hpriv->mmio;
2201 void __iomem *port_mmio = ahci_port_base(ap);
2202 u32 tmp;
2203 struct ahci_port_priv *pp = ap->private_data;
2204
2205 /* clear IRQ */
2206 tmp = readl(addr: port_mmio + PORT_IRQ_STAT);
2207 writel(val: tmp, addr: port_mmio + PORT_IRQ_STAT);
2208 writel(val: 1 << ap->port_no, addr: mmio + HOST_IRQ_STAT);
2209
2210 /* turn IRQ back on */
2211 writel(val: pp->intr_mask, addr: port_mmio + PORT_IRQ_MASK);
2212}
2213
2214void ahci_error_handler(struct ata_port *ap)
2215{
2216 struct ahci_host_priv *hpriv = ap->host->private_data;
2217
2218 if (!ata_port_is_frozen(ap)) {
2219 /* restart engine */
2220 hpriv->stop_engine(ap);
2221 hpriv->start_engine(ap);
2222 }
2223
2224 sata_pmp_error_handler(ap);
2225
2226 if (!ata_dev_enabled(dev: ap->link.device))
2227 hpriv->stop_engine(ap);
2228}
2229EXPORT_SYMBOL_GPL(ahci_error_handler);
2230
2231static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
2232{
2233 struct ata_port *ap = qc->ap;
2234
2235 /* make DMA engine forget about the failed command */
2236 if (qc->flags & ATA_QCFLAG_EH)
2237 ahci_kick_engine(ap);
2238}
2239
2240static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
2241{
2242 struct ahci_host_priv *hpriv = ap->host->private_data;
2243 void __iomem *port_mmio = ahci_port_base(ap);
2244 struct ata_device *dev = ap->link.device;
2245 u32 devslp, dm, dito, mdat, deto, dito_conf;
2246 int rc;
2247 unsigned int err_mask;
2248
2249 devslp = readl(addr: port_mmio + PORT_DEVSLP);
2250 if (!(devslp & PORT_DEVSLP_DSP)) {
2251 dev_info(ap->host->dev, "port does not support device sleep\n");
2252 return;
2253 }
2254
2255 /* disable device sleep */
2256 if (!sleep) {
2257 if (devslp & PORT_DEVSLP_ADSE) {
2258 writel(val: devslp & ~PORT_DEVSLP_ADSE,
2259 addr: port_mmio + PORT_DEVSLP);
2260 err_mask = ata_dev_set_feature(dev,
2261 subcmd: SETFEATURES_SATA_DISABLE,
2262 action: SATA_DEVSLP);
2263 if (err_mask && err_mask != AC_ERR_DEV)
2264 ata_dev_warn(dev, "failed to disable DEVSLP\n");
2265 }
2266 return;
2267 }
2268
2269 dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
2270 dito = devslp_idle_timeout / (dm + 1);
2271 if (dito > 0x3ff)
2272 dito = 0x3ff;
2273
2274 dito_conf = (devslp >> PORT_DEVSLP_DITO_OFFSET) & 0x3FF;
2275
2276 /* device sleep was already enabled and same dito */
2277 if ((devslp & PORT_DEVSLP_ADSE) && (dito_conf == dito))
2278 return;
2279
2280 /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2281 rc = hpriv->stop_engine(ap);
2282 if (rc)
2283 return;
2284
2285 /* Use the nominal value 10 ms if the read MDAT is zero,
2286 * the nominal value of DETO is 20 ms.
2287 */
2288 if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
2289 ATA_LOG_DEVSLP_VALID_MASK) {
2290 mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
2291 ATA_LOG_DEVSLP_MDAT_MASK;
2292 if (!mdat)
2293 mdat = 10;
2294 deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
2295 if (!deto)
2296 deto = 20;
2297 } else {
2298 mdat = 10;
2299 deto = 20;
2300 }
2301
2302 /* Make dito, mdat, deto bits to 0s */
2303 devslp &= ~GENMASK_ULL(24, 2);
2304 devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
2305 (mdat << PORT_DEVSLP_MDAT_OFFSET) |
2306 (deto << PORT_DEVSLP_DETO_OFFSET) |
2307 PORT_DEVSLP_ADSE);
2308 writel(val: devslp, addr: port_mmio + PORT_DEVSLP);
2309
2310 hpriv->start_engine(ap);
2311
2312 /* enable device sleep feature for the drive */
2313 err_mask = ata_dev_set_feature(dev,
2314 subcmd: SETFEATURES_SATA_ENABLE,
2315 action: SATA_DEVSLP);
2316 if (err_mask && err_mask != AC_ERR_DEV)
2317 ata_dev_warn(dev, "failed to enable DEVSLP\n");
2318}
2319
2320static void ahci_enable_fbs(struct ata_port *ap)
2321{
2322 struct ahci_host_priv *hpriv = ap->host->private_data;
2323 struct ahci_port_priv *pp = ap->private_data;
2324 void __iomem *port_mmio = ahci_port_base(ap);
2325 u32 fbs;
2326 int rc;
2327
2328 if (!pp->fbs_supported)
2329 return;
2330
2331 fbs = readl(addr: port_mmio + PORT_FBS);
2332 if (fbs & PORT_FBS_EN) {
2333 pp->fbs_enabled = true;
2334 pp->fbs_last_dev = -1; /* initialization */
2335 return;
2336 }
2337
2338 rc = hpriv->stop_engine(ap);
2339 if (rc)
2340 return;
2341
2342 writel(val: fbs | PORT_FBS_EN, addr: port_mmio + PORT_FBS);
2343 fbs = readl(addr: port_mmio + PORT_FBS);
2344 if (fbs & PORT_FBS_EN) {
2345 dev_info(ap->host->dev, "FBS is enabled\n");
2346 pp->fbs_enabled = true;
2347 pp->fbs_last_dev = -1; /* initialization */
2348 } else
2349 dev_err(ap->host->dev, "Failed to enable FBS\n");
2350
2351 hpriv->start_engine(ap);
2352}
2353
2354static void ahci_disable_fbs(struct ata_port *ap)
2355{
2356 struct ahci_host_priv *hpriv = ap->host->private_data;
2357 struct ahci_port_priv *pp = ap->private_data;
2358 void __iomem *port_mmio = ahci_port_base(ap);
2359 u32 fbs;
2360 int rc;
2361
2362 if (!pp->fbs_supported)
2363 return;
2364
2365 fbs = readl(addr: port_mmio + PORT_FBS);
2366 if ((fbs & PORT_FBS_EN) == 0) {
2367 pp->fbs_enabled = false;
2368 return;
2369 }
2370
2371 rc = hpriv->stop_engine(ap);
2372 if (rc)
2373 return;
2374
2375 writel(val: fbs & ~PORT_FBS_EN, addr: port_mmio + PORT_FBS);
2376 fbs = readl(addr: port_mmio + PORT_FBS);
2377 if (fbs & PORT_FBS_EN)
2378 dev_err(ap->host->dev, "Failed to disable FBS\n");
2379 else {
2380 dev_info(ap->host->dev, "FBS is disabled\n");
2381 pp->fbs_enabled = false;
2382 }
2383
2384 hpriv->start_engine(ap);
2385}
2386
2387static void ahci_pmp_attach(struct ata_port *ap)
2388{
2389 void __iomem *port_mmio = ahci_port_base(ap);
2390 struct ahci_port_priv *pp = ap->private_data;
2391 u32 cmd;
2392
2393 cmd = readl(addr: port_mmio + PORT_CMD);
2394 cmd |= PORT_CMD_PMP;
2395 writel(val: cmd, addr: port_mmio + PORT_CMD);
2396
2397 ahci_enable_fbs(ap);
2398
2399 pp->intr_mask |= PORT_IRQ_BAD_PMP;
2400
2401 /*
2402 * We must not change the port interrupt mask register if the
2403 * port is marked frozen, the value in pp->intr_mask will be
2404 * restored later when the port is thawed.
2405 *
2406 * Note that during initialization, the port is marked as
2407 * frozen since the irq handler is not yet registered.
2408 */
2409 if (!ata_port_is_frozen(ap))
2410 writel(val: pp->intr_mask, addr: port_mmio + PORT_IRQ_MASK);
2411}
2412
2413static void ahci_pmp_detach(struct ata_port *ap)
2414{
2415 void __iomem *port_mmio = ahci_port_base(ap);
2416 struct ahci_port_priv *pp = ap->private_data;
2417 u32 cmd;
2418
2419 ahci_disable_fbs(ap);
2420
2421 cmd = readl(addr: port_mmio + PORT_CMD);
2422 cmd &= ~PORT_CMD_PMP;
2423 writel(val: cmd, addr: port_mmio + PORT_CMD);
2424
2425 pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
2426
2427 /* see comment above in ahci_pmp_attach() */
2428 if (!ata_port_is_frozen(ap))
2429 writel(val: pp->intr_mask, addr: port_mmio + PORT_IRQ_MASK);
2430}
2431
2432int ahci_port_resume(struct ata_port *ap)
2433{
2434 ahci_rpm_get_port(ap);
2435
2436 ahci_power_up(ap);
2437 ahci_start_port(ap);
2438
2439 if (sata_pmp_attached(ap))
2440 ahci_pmp_attach(ap);
2441 else
2442 ahci_pmp_detach(ap);
2443
2444 return 0;
2445}
2446EXPORT_SYMBOL_GPL(ahci_port_resume);
2447
2448#ifdef CONFIG_PM
2449static void ahci_handle_s2idle(struct ata_port *ap)
2450{
2451 void __iomem *port_mmio = ahci_port_base(ap);
2452 u32 devslp;
2453
2454 if (pm_suspend_via_firmware())
2455 return;
2456 devslp = readl(addr: port_mmio + PORT_DEVSLP);
2457 if ((devslp & PORT_DEVSLP_ADSE))
2458 ata_msleep(ap, msecs: devslp_idle_timeout);
2459}
2460
2461static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2462{
2463 const char *emsg = NULL;
2464 int rc;
2465
2466 rc = ahci_deinit_port(ap, emsg: &emsg);
2467 if (rc == 0)
2468 ahci_power_down(ap);
2469 else {
2470 ata_port_err(ap, "%s (%d)\n", emsg, rc);
2471 ata_port_freeze(ap);
2472 }
2473
2474 if (acpi_storage_d3(dev: ap->host->dev))
2475 ahci_handle_s2idle(ap);
2476
2477 ahci_rpm_put_port(ap);
2478 return rc;
2479}
2480#endif
2481
2482static int ahci_port_start(struct ata_port *ap)
2483{
2484 struct ahci_host_priv *hpriv = ap->host->private_data;
2485 struct device *dev = ap->host->dev;
2486 struct ahci_port_priv *pp;
2487 void *mem;
2488 dma_addr_t mem_dma;
2489 size_t dma_sz, rx_fis_sz;
2490
2491 pp = devm_kzalloc(dev, size: sizeof(*pp), GFP_KERNEL);
2492 if (!pp)
2493 return -ENOMEM;
2494
2495 if (ap->host->n_ports > 1) {
2496 pp->irq_desc = devm_kzalloc(dev, size: 8, GFP_KERNEL);
2497 if (!pp->irq_desc) {
2498 devm_kfree(dev, p: pp);
2499 return -ENOMEM;
2500 }
2501 snprintf(buf: pp->irq_desc, size: 8,
2502 fmt: "%s%d", dev_driver_string(dev), ap->port_no);
2503 }
2504
2505 /* check FBS capability */
2506 if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2507 void __iomem *port_mmio = ahci_port_base(ap);
2508 u32 cmd = readl(addr: port_mmio + PORT_CMD);
2509 if (cmd & PORT_CMD_FBSCP)
2510 pp->fbs_supported = true;
2511 else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2512 dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2513 ap->port_no);
2514 pp->fbs_supported = true;
2515 } else
2516 dev_warn(dev, "port %d is not capable of FBS\n",
2517 ap->port_no);
2518 }
2519
2520 if (pp->fbs_supported) {
2521 dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2522 rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2523 } else {
2524 dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2525 rx_fis_sz = AHCI_RX_FIS_SZ;
2526 }
2527
2528 mem = dmam_alloc_coherent(dev, size: dma_sz, dma_handle: &mem_dma, GFP_KERNEL);
2529 if (!mem)
2530 return -ENOMEM;
2531
2532 /*
2533 * First item in chunk of DMA memory: 32-slot command table,
2534 * 32 bytes each in size
2535 */
2536 pp->cmd_slot = mem;
2537 pp->cmd_slot_dma = mem_dma;
2538
2539 mem += AHCI_CMD_SLOT_SZ;
2540 mem_dma += AHCI_CMD_SLOT_SZ;
2541
2542 /*
2543 * Second item: Received-FIS area
2544 */
2545 pp->rx_fis = mem;
2546 pp->rx_fis_dma = mem_dma;
2547
2548 mem += rx_fis_sz;
2549 mem_dma += rx_fis_sz;
2550
2551 /*
2552 * Third item: data area for storing a single command
2553 * and its scatter-gather table
2554 */
2555 pp->cmd_tbl = mem;
2556 pp->cmd_tbl_dma = mem_dma;
2557
2558 /*
2559 * Save off initial list of interrupts to be enabled.
2560 * This could be changed later
2561 */
2562 pp->intr_mask = DEF_PORT_IRQ;
2563
2564 /*
2565 * Switch to per-port locking in case each port has its own MSI vector.
2566 */
2567 if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
2568 spin_lock_init(&pp->lock);
2569 ap->lock = &pp->lock;
2570 }
2571
2572 ap->private_data = pp;
2573
2574 /* engage engines, captain */
2575 return ahci_port_resume(ap);
2576}
2577
2578static void ahci_port_stop(struct ata_port *ap)
2579{
2580 const char *emsg = NULL;
2581 struct ahci_host_priv *hpriv = ap->host->private_data;
2582 void __iomem *host_mmio = hpriv->mmio;
2583 int rc;
2584
2585 /* de-initialize port */
2586 rc = ahci_deinit_port(ap, emsg: &emsg);
2587 if (rc)
2588 ata_port_warn(ap, "%s (%d)\n", emsg, rc);
2589
2590 /*
2591 * Clear GHC.IS to prevent stuck INTx after disabling MSI and
2592 * re-enabling INTx.
2593 */
2594 writel(val: 1 << ap->port_no, addr: host_mmio + HOST_IRQ_STAT);
2595
2596 ahci_rpm_put_port(ap);
2597}
2598
2599void ahci_print_info(struct ata_host *host, const char *scc_s)
2600{
2601 struct ahci_host_priv *hpriv = host->private_data;
2602 u32 vers, cap, cap2, impl, speed;
2603 const char *speed_s;
2604
2605 vers = hpriv->version;
2606 cap = hpriv->cap;
2607 cap2 = hpriv->cap2;
2608 impl = hpriv->port_map;
2609
2610 speed = (cap >> 20) & 0xf;
2611 if (speed == 1)
2612 speed_s = "1.5";
2613 else if (speed == 2)
2614 speed_s = "3";
2615 else if (speed == 3)
2616 speed_s = "6";
2617 else
2618 speed_s = "?";
2619
2620 dev_info(host->dev,
2621 "AHCI vers %02x%02x.%02x%02x, "
2622 "%u command slots, %s Gbps, %s mode\n"
2623 ,
2624
2625 (vers >> 24) & 0xff,
2626 (vers >> 16) & 0xff,
2627 (vers >> 8) & 0xff,
2628 vers & 0xff,
2629
2630 ((cap >> 8) & 0x1f) + 1,
2631 speed_s,
2632 scc_s);
2633
2634 dev_info(host->dev,
2635 "%u/%u ports implemented (port mask 0x%x)\n"
2636 ,
2637
2638 hweight32(impl),
2639 (cap & 0x1f) + 1,
2640 impl);
2641
2642 dev_info(host->dev,
2643 "flags: "
2644 "%s%s%s%s%s%s%s"
2645 "%s%s%s%s%s%s%s"
2646 "%s%s%s%s%s%s%s"
2647 "%s%s\n"
2648 ,
2649
2650 cap & HOST_CAP_64 ? "64bit " : "",
2651 cap & HOST_CAP_NCQ ? "ncq " : "",
2652 cap & HOST_CAP_SNTF ? "sntf " : "",
2653 cap & HOST_CAP_MPS ? "ilck " : "",
2654 cap & HOST_CAP_SSS ? "stag " : "",
2655 cap & HOST_CAP_ALPM ? "pm " : "",
2656 cap & HOST_CAP_LED ? "led " : "",
2657 cap & HOST_CAP_CLO ? "clo " : "",
2658 cap & HOST_CAP_ONLY ? "only " : "",
2659 cap & HOST_CAP_PMP ? "pmp " : "",
2660 cap & HOST_CAP_FBS ? "fbs " : "",
2661 cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2662 cap & HOST_CAP_SSC ? "slum " : "",
2663 cap & HOST_CAP_PART ? "part " : "",
2664 cap & HOST_CAP_CCC ? "ccc " : "",
2665 cap & HOST_CAP_EMS ? "ems " : "",
2666 cap & HOST_CAP_SXS ? "sxs " : "",
2667 cap2 & HOST_CAP2_DESO ? "deso " : "",
2668 cap2 & HOST_CAP2_SADM ? "sadm " : "",
2669 cap2 & HOST_CAP2_SDS ? "sds " : "",
2670 cap2 & HOST_CAP2_APST ? "apst " : "",
2671 cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2672 cap2 & HOST_CAP2_BOH ? "boh " : ""
2673 );
2674}
2675EXPORT_SYMBOL_GPL(ahci_print_info);
2676
2677void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2678 struct ata_port_info *pi)
2679{
2680 u8 messages;
2681 void __iomem *mmio = hpriv->mmio;
2682 u32 em_loc = readl(addr: mmio + HOST_EM_LOC);
2683 u32 em_ctl = readl(addr: mmio + HOST_EM_CTL);
2684
2685 if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
2686 return;
2687
2688 messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2689
2690 if (messages) {
2691 /* store em_loc */
2692 hpriv->em_loc = ((em_loc >> 16) * 4);
2693 hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2694 hpriv->em_msg_type = messages;
2695 pi->flags |= ATA_FLAG_EM;
2696 if (!(em_ctl & EM_CTL_ALHD))
2697 pi->flags |= ATA_FLAG_SW_ACTIVITY;
2698 }
2699}
2700EXPORT_SYMBOL_GPL(ahci_set_em_messages);
2701
2702static int ahci_host_activate_multi_irqs(struct ata_host *host,
2703 const struct scsi_host_template *sht)
2704{
2705 struct ahci_host_priv *hpriv = host->private_data;
2706 int i, rc;
2707
2708 rc = ata_host_start(host);
2709 if (rc)
2710 return rc;
2711 /*
2712 * Requests IRQs according to AHCI-1.1 when multiple MSIs were
2713 * allocated. That is one MSI per port, starting from @irq.
2714 */
2715 for (i = 0; i < host->n_ports; i++) {
2716 struct ahci_port_priv *pp = host->ports[i]->private_data;
2717 int irq = hpriv->get_irq_vector(host, i);
2718
2719 /* Do not receive interrupts sent by dummy ports */
2720 if (!pp) {
2721 disable_irq(irq);
2722 continue;
2723 }
2724
2725 rc = devm_request_irq(dev: host->dev, irq, handler: ahci_multi_irqs_intr_hard,
2726 irqflags: 0, devname: pp->irq_desc, dev_id: host->ports[i]);
2727
2728 if (rc)
2729 return rc;
2730 ata_port_desc_misc(ap: host->ports[i], irq);
2731 }
2732
2733 return ata_host_register(host, sht);
2734}
2735
2736/**
2737 * ahci_host_activate - start AHCI host, request IRQs and register it
2738 * @host: target ATA host
2739 * @sht: scsi_host_template to use when registering the host
2740 *
2741 * LOCKING:
2742 * Inherited from calling layer (may sleep).
2743 *
2744 * RETURNS:
2745 * 0 on success, -errno otherwise.
2746 */
2747int ahci_host_activate(struct ata_host *host, const struct scsi_host_template *sht)
2748{
2749 struct ahci_host_priv *hpriv = host->private_data;
2750 int irq = hpriv->irq;
2751 int rc;
2752
2753 if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
2754 if (hpriv->irq_handler &&
2755 hpriv->irq_handler != ahci_single_level_irq_intr)
2756 dev_warn(host->dev,
2757 "both AHCI_HFLAG_MULTI_MSI flag set and custom irq handler implemented\n");
2758 if (!hpriv->get_irq_vector) {
2759 dev_err(host->dev,
2760 "AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");
2761 return -EIO;
2762 }
2763
2764 rc = ahci_host_activate_multi_irqs(host, sht);
2765 } else {
2766 rc = ata_host_activate(host, irq, irq_handler: hpriv->irq_handler,
2767 IRQF_SHARED, sht);
2768 }
2769
2770
2771 return rc;
2772}
2773EXPORT_SYMBOL_GPL(ahci_host_activate);
2774
2775MODULE_AUTHOR("Jeff Garzik");
2776MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2777MODULE_LICENSE("GPL");
2778