1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation
4 * Copyright (C) 2009, 2010, 2011 Red Hat, Inc.
5 * Copyright (C) 2009, 2010, 2011 Amit Shah <amit.shah@redhat.com>
6 */
7#include <linux/cdev.h>
8#include <linux/debugfs.h>
9#include <linux/completion.h>
10#include <linux/device.h>
11#include <linux/err.h>
12#include <linux/freezer.h>
13#include <linux/fs.h>
14#include <linux/splice.h>
15#include <linux/pagemap.h>
16#include <linux/idr.h>
17#include <linux/init.h>
18#include <linux/list.h>
19#include <linux/poll.h>
20#include <linux/sched.h>
21#include <linux/slab.h>
22#include <linux/spinlock.h>
23#include <linux/virtio.h>
24#include <linux/virtio_console.h>
25#include <linux/wait.h>
26#include <linux/workqueue.h>
27#include <linux/module.h>
28#include <linux/dma-mapping.h>
29#include <linux/string_choices.h>
30#include "../tty/hvc/hvc_console.h"
31
32#define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)
33#define VIRTCONS_MAX_PORTS 0x8000
34
35/*
36 * This is a global struct for storing common data for all the devices
37 * this driver handles.
38 *
39 * Mainly, it has a linked list for all the consoles in one place so
40 * that callbacks from hvc for get_chars(), put_chars() work properly
41 * across multiple devices and multiple ports per device.
42 */
43struct ports_driver_data {
44 /* Used for exporting per-port information to debugfs */
45 struct dentry *debugfs_dir;
46
47 /* List of all the devices we're handling */
48 struct list_head portdevs;
49
50 /* All the console devices handled by this driver */
51 struct list_head consoles;
52};
53
54static struct ports_driver_data pdrvdata;
55
56static const struct class port_class = {
57 .name = "virtio-ports",
58};
59
60static DEFINE_SPINLOCK(pdrvdata_lock);
61static DECLARE_COMPLETION(early_console_added);
62
63/* This struct holds information that's relevant only for console ports */
64struct console {
65 /* We'll place all consoles in a list in the pdrvdata struct */
66 struct list_head list;
67
68 /* The hvc device associated with this console port */
69 struct hvc_struct *hvc;
70
71 /* The size of the console */
72 struct winsize ws;
73
74 /*
75 * This number identifies the number that we used to register
76 * with hvc in hvc_instantiate() and hvc_alloc(); this is the
77 * number passed on by the hvc callbacks to us to
78 * differentiate between the other console ports handled by
79 * this driver
80 */
81 u32 vtermno;
82};
83
84static DEFINE_IDA(vtermno_ida);
85
86struct port_buffer {
87 char *buf;
88
89 /* size of the buffer in *buf above */
90 size_t size;
91
92 /* used length of the buffer */
93 size_t len;
94 /* offset in the buf from which to consume data */
95 size_t offset;
96
97 /* DMA address of buffer */
98 dma_addr_t dma;
99
100 /* Device we got DMA memory from */
101 struct device *dev;
102
103 /* List of pending dma buffers to free */
104 struct list_head list;
105
106 /* If sgpages == 0 then buf is used */
107 unsigned int sgpages;
108
109 /* sg is used if spages > 0. sg must be the last in is struct */
110 struct scatterlist sg[] __counted_by(sgpages);
111};
112
113/*
114 * This is a per-device struct that stores data common to all the
115 * ports for that device (vdev->priv).
116 */
117struct ports_device {
118 /* Next portdev in the list, head is in the pdrvdata struct */
119 struct list_head list;
120
121 /*
122 * Workqueue handlers where we process deferred work after
123 * notification
124 */
125 struct work_struct control_work;
126 struct work_struct config_work;
127
128 struct list_head ports;
129
130 /* To protect the list of ports */
131 spinlock_t ports_lock;
132
133 /* To protect the vq operations for the control channel */
134 spinlock_t c_ivq_lock;
135 spinlock_t c_ovq_lock;
136
137 /* max. number of ports this device can hold */
138 u32 max_nr_ports;
139
140 /* The virtio device we're associated with */
141 struct virtio_device *vdev;
142
143 /*
144 * A couple of virtqueues for the control channel: one for
145 * guest->host transfers, one for host->guest transfers
146 */
147 struct virtqueue *c_ivq, *c_ovq;
148
149 /*
150 * A control packet buffer for guest->host requests, protected
151 * by c_ovq_lock.
152 */
153 struct virtio_console_control cpkt;
154
155 /* Array of per-port IO virtqueues */
156 struct virtqueue **in_vqs, **out_vqs;
157
158 /* Major number for this device. Ports will be created as minors. */
159 int chr_major;
160};
161
162struct port_stats {
163 unsigned long bytes_sent, bytes_received, bytes_discarded;
164};
165
166/* This struct holds the per-port data */
167struct port {
168 /* Next port in the list, head is in the ports_device */
169 struct list_head list;
170
171 /* Pointer to the parent virtio_console device */
172 struct ports_device *portdev;
173
174 /* The current buffer from which data has to be fed to readers */
175 struct port_buffer *inbuf;
176
177 /*
178 * To protect the operations on the in_vq associated with this
179 * port. Has to be a spinlock because it can be called from
180 * interrupt context (get_char()).
181 */
182 spinlock_t inbuf_lock;
183
184 /* Protect the operations on the out_vq. */
185 spinlock_t outvq_lock;
186
187 /* The IO vqs for this port */
188 struct virtqueue *in_vq, *out_vq;
189
190 /* File in the debugfs directory that exposes this port's information */
191 struct dentry *debugfs_file;
192
193 /*
194 * Keep count of the bytes sent, received and discarded for
195 * this port for accounting and debugging purposes. These
196 * counts are not reset across port open / close events.
197 */
198 struct port_stats stats;
199
200 /*
201 * The entries in this struct will be valid if this port is
202 * hooked up to an hvc console
203 */
204 struct console cons;
205
206 /* Each port associates with a separate char device */
207 struct cdev *cdev;
208 struct device *dev;
209
210 /* Reference-counting to handle port hot-unplugs and file operations */
211 struct kref kref;
212
213 /* A waitqueue for poll() or blocking read operations */
214 wait_queue_head_t waitqueue;
215
216 /* The 'name' of the port that we expose via sysfs properties */
217 char *name;
218
219 /* We can notify apps of host connect / disconnect events via SIGIO */
220 struct fasync_struct *async_queue;
221
222 /* The 'id' to identify the port with the Host */
223 u32 id;
224
225 bool outvq_full;
226
227 /* Is the host device open */
228 bool host_connected;
229
230 /* We should allow only one process to open a port */
231 bool guest_connected;
232};
233
234static struct port *find_port_by_vtermno(u32 vtermno)
235{
236 struct port *port;
237 struct console *cons;
238 unsigned long flags;
239
240 spin_lock_irqsave(&pdrvdata_lock, flags);
241 list_for_each_entry(cons, &pdrvdata.consoles, list) {
242 if (cons->vtermno == vtermno) {
243 port = container_of(cons, struct port, cons);
244 goto out;
245 }
246 }
247 port = NULL;
248out:
249 spin_unlock_irqrestore(lock: &pdrvdata_lock, flags);
250 return port;
251}
252
253static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev,
254 dev_t dev)
255{
256 struct port *port;
257 unsigned long flags;
258
259 spin_lock_irqsave(&portdev->ports_lock, flags);
260 list_for_each_entry(port, &portdev->ports, list) {
261 if (port->cdev->dev == dev) {
262 kref_get(kref: &port->kref);
263 goto out;
264 }
265 }
266 port = NULL;
267out:
268 spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
269
270 return port;
271}
272
273static struct port *find_port_by_devt(dev_t dev)
274{
275 struct ports_device *portdev;
276 struct port *port;
277 unsigned long flags;
278
279 spin_lock_irqsave(&pdrvdata_lock, flags);
280 list_for_each_entry(portdev, &pdrvdata.portdevs, list) {
281 port = find_port_by_devt_in_portdev(portdev, dev);
282 if (port)
283 goto out;
284 }
285 port = NULL;
286out:
287 spin_unlock_irqrestore(lock: &pdrvdata_lock, flags);
288 return port;
289}
290
291static struct port *find_port_by_id(struct ports_device *portdev, u32 id)
292{
293 struct port *port;
294 unsigned long flags;
295
296 spin_lock_irqsave(&portdev->ports_lock, flags);
297 list_for_each_entry(port, &portdev->ports, list)
298 if (port->id == id)
299 goto out;
300 port = NULL;
301out:
302 spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
303
304 return port;
305}
306
307static struct port *find_port_by_vq(struct ports_device *portdev,
308 struct virtqueue *vq)
309{
310 struct port *port;
311 unsigned long flags;
312
313 spin_lock_irqsave(&portdev->ports_lock, flags);
314 list_for_each_entry(port, &portdev->ports, list)
315 if (port->in_vq == vq || port->out_vq == vq)
316 goto out;
317 port = NULL;
318out:
319 spin_unlock_irqrestore(lock: &portdev->ports_lock, flags);
320 return port;
321}
322
323static bool is_console_port(struct port *port)
324{
325 if (port->cons.hvc)
326 return true;
327 return false;
328}
329
330static bool is_rproc_serial(const struct virtio_device *vdev)
331{
332 return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL;
333}
334
335static inline bool use_multiport(struct ports_device *portdev)
336{
337 /*
338 * This condition can be true when put_chars is called from
339 * early_init
340 */
341 if (!portdev->vdev)
342 return false;
343 return __virtio_test_bit(vdev: portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);
344}
345
346static DEFINE_SPINLOCK(dma_bufs_lock);
347static LIST_HEAD(pending_free_dma_bufs);
348
349static void free_buf(struct port_buffer *buf, bool can_sleep)
350{
351 unsigned int i;
352
353 for (i = 0; i < buf->sgpages; i++) {
354 struct page *page = sg_page(sg: &buf->sg[i]);
355 if (!page)
356 break;
357 put_page(page);
358 }
359
360 if (!buf->dev) {
361 kfree(objp: buf->buf);
362 } else if (is_rproc_enabled) {
363 unsigned long flags;
364
365 /* dma_free_coherent requires interrupts to be enabled. */
366 if (!can_sleep) {
367 /* queue up dma-buffers to be freed later */
368 spin_lock_irqsave(&dma_bufs_lock, flags);
369 list_add_tail(new: &buf->list, head: &pending_free_dma_bufs);
370 spin_unlock_irqrestore(lock: &dma_bufs_lock, flags);
371 return;
372 }
373 dma_free_coherent(dev: buf->dev, size: buf->size, cpu_addr: buf->buf, dma_handle: buf->dma);
374
375 /* Release device refcnt and allow it to be freed */
376 put_device(dev: buf->dev);
377 }
378
379 kfree(objp: buf);
380}
381
382static void reclaim_dma_bufs(void)
383{
384 unsigned long flags;
385 struct port_buffer *buf, *tmp;
386 LIST_HEAD(tmp_list);
387
388 if (list_empty(head: &pending_free_dma_bufs))
389 return;
390
391 /* Create a copy of the pending_free_dma_bufs while holding the lock */
392 spin_lock_irqsave(&dma_bufs_lock, flags);
393 list_cut_position(list: &tmp_list, head: &pending_free_dma_bufs,
394 entry: pending_free_dma_bufs.prev);
395 spin_unlock_irqrestore(lock: &dma_bufs_lock, flags);
396
397 /* Release the dma buffers, without irqs enabled */
398 list_for_each_entry_safe(buf, tmp, &tmp_list, list) {
399 list_del(entry: &buf->list);
400 free_buf(buf, can_sleep: true);
401 }
402}
403
404static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size,
405 int pages)
406{
407 struct port_buffer *buf;
408
409 reclaim_dma_bufs();
410
411 /*
412 * Allocate buffer and the sg list. The sg list array is allocated
413 * directly after the port_buffer struct.
414 */
415 buf = kmalloc(struct_size(buf, sg, pages), GFP_KERNEL);
416 if (!buf)
417 goto fail;
418
419 buf->sgpages = pages;
420 if (pages > 0) {
421 buf->dev = NULL;
422 buf->buf = NULL;
423 return buf;
424 }
425
426 if (is_rproc_serial(vdev)) {
427 /*
428 * Allocate DMA memory from ancestor. When a virtio
429 * device is created by remoteproc, the DMA memory is
430 * associated with the parent device:
431 * virtioY => remoteprocX#vdevYbuffer.
432 */
433 buf->dev = vdev->dev.parent;
434 if (!buf->dev)
435 goto free_buf;
436
437 /* Increase device refcnt to avoid freeing it */
438 get_device(dev: buf->dev);
439 buf->buf = dma_alloc_coherent(dev: buf->dev, size: buf_size, dma_handle: &buf->dma,
440 GFP_KERNEL);
441 } else {
442 buf->dev = NULL;
443 buf->buf = kmalloc(buf_size, GFP_KERNEL);
444 }
445
446 if (!buf->buf)
447 goto free_buf;
448 buf->len = 0;
449 buf->offset = 0;
450 buf->size = buf_size;
451 return buf;
452
453free_buf:
454 kfree(objp: buf);
455fail:
456 return NULL;
457}
458
459/* Callers should take appropriate locks */
460static struct port_buffer *get_inbuf(struct port *port)
461{
462 struct port_buffer *buf;
463 unsigned int len;
464
465 if (port->inbuf)
466 return port->inbuf;
467
468 buf = virtqueue_get_buf(vq: port->in_vq, len: &len);
469 if (buf) {
470 buf->len = min_t(size_t, len, buf->size);
471 buf->offset = 0;
472 port->stats.bytes_received += len;
473 }
474 return buf;
475}
476
477/*
478 * Create a scatter-gather list representing our input buffer and put
479 * it in the queue.
480 *
481 * Callers should take appropriate locks.
482 */
483static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf)
484{
485 struct scatterlist sg[1];
486 int ret;
487
488 sg_init_one(sg, buf->buf, buf->size);
489
490 ret = virtqueue_add_inbuf(vq, sg, num: 1, data: buf, GFP_ATOMIC);
491 virtqueue_kick(vq);
492 if (!ret)
493 ret = vq->num_free;
494 return ret;
495}
496
497/* Discard any unread data this port has. Callers lockers. */
498static void discard_port_data(struct port *port)
499{
500 struct port_buffer *buf;
501 unsigned int err;
502
503 if (!port->portdev) {
504 /* Device has been unplugged. vqs are already gone. */
505 return;
506 }
507 buf = get_inbuf(port);
508
509 err = 0;
510 while (buf) {
511 port->stats.bytes_discarded += buf->len - buf->offset;
512 if (add_inbuf(vq: port->in_vq, buf) < 0) {
513 err++;
514 free_buf(buf, can_sleep: false);
515 }
516 port->inbuf = NULL;
517 buf = get_inbuf(port);
518 }
519 if (err)
520 dev_warn(port->dev, "Errors adding %d buffers back to vq\n",
521 err);
522}
523
524static bool port_has_data(struct port *port)
525{
526 unsigned long flags;
527 bool ret;
528
529 ret = false;
530 spin_lock_irqsave(&port->inbuf_lock, flags);
531 port->inbuf = get_inbuf(port);
532 if (port->inbuf)
533 ret = true;
534
535 spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
536 return ret;
537}
538
539static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
540 unsigned int event, unsigned int value)
541{
542 struct scatterlist sg[1];
543 struct virtqueue *vq;
544 unsigned int len;
545
546 if (!use_multiport(portdev))
547 return 0;
548
549 vq = portdev->c_ovq;
550
551 spin_lock(lock: &portdev->c_ovq_lock);
552
553 portdev->cpkt.id = cpu_to_virtio32(vdev: portdev->vdev, val: port_id);
554 portdev->cpkt.event = cpu_to_virtio16(vdev: portdev->vdev, val: event);
555 portdev->cpkt.value = cpu_to_virtio16(vdev: portdev->vdev, val: value);
556
557 sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control));
558
559 if (virtqueue_add_outbuf(vq, sg, num: 1, data: &portdev->cpkt, GFP_ATOMIC) == 0) {
560 virtqueue_kick(vq);
561 while (!virtqueue_get_buf(vq, len: &len)
562 && !virtqueue_is_broken(vq))
563 cpu_relax();
564 }
565
566 spin_unlock(lock: &portdev->c_ovq_lock);
567 return 0;
568}
569
570static ssize_t send_control_msg(struct port *port, unsigned int event,
571 unsigned int value)
572{
573 /* Did the port get unplugged before userspace closed it? */
574 if (port->portdev)
575 return __send_control_msg(portdev: port->portdev, port_id: port->id, event, value);
576 return 0;
577}
578
579
580/* Callers must take the port->outvq_lock */
581static void reclaim_consumed_buffers(struct port *port)
582{
583 struct port_buffer *buf;
584 unsigned int len;
585
586 if (!port->portdev) {
587 /* Device has been unplugged. vqs are already gone. */
588 return;
589 }
590 while ((buf = virtqueue_get_buf(vq: port->out_vq, len: &len))) {
591 free_buf(buf, can_sleep: false);
592 port->outvq_full = false;
593 }
594}
595
596static ssize_t __send_to_port(struct port *port, struct scatterlist *sg,
597 int nents, size_t in_count,
598 void *data, bool nonblock)
599{
600 struct virtqueue *out_vq;
601 int err;
602 unsigned long flags;
603 unsigned int len;
604
605 out_vq = port->out_vq;
606
607 spin_lock_irqsave(&port->outvq_lock, flags);
608
609 reclaim_consumed_buffers(port);
610
611 err = virtqueue_add_outbuf(vq: out_vq, sg, num: nents, data, GFP_ATOMIC);
612
613 /* Tell Host to go! */
614 virtqueue_kick(vq: out_vq);
615
616 if (err) {
617 in_count = 0;
618 goto done;
619 }
620
621 if (out_vq->num_free == 0)
622 port->outvq_full = true;
623
624 if (nonblock)
625 goto done;
626
627 /*
628 * Wait till the host acknowledges it pushed out the data we
629 * sent. This is done for data from the hvc_console; the tty
630 * operations are performed with spinlocks held so we can't
631 * sleep here. An alternative would be to copy the data to a
632 * buffer and relax the spinning requirement. The downside is
633 * we need to kmalloc a GFP_ATOMIC buffer each time the
634 * console driver writes something out.
635 */
636 while (!virtqueue_get_buf(vq: out_vq, len: &len)
637 && !virtqueue_is_broken(vq: out_vq))
638 cpu_relax();
639done:
640 spin_unlock_irqrestore(lock: &port->outvq_lock, flags);
641
642 port->stats.bytes_sent += in_count;
643 /*
644 * We're expected to return the amount of data we wrote -- all
645 * of it
646 */
647 return in_count;
648}
649
650/*
651 * Give out the data that's requested from the buffer that we have
652 * queued up.
653 */
654static ssize_t fill_readbuf(struct port *port, u8 __user *out_buf,
655 size_t out_count, bool to_user)
656{
657 struct port_buffer *buf;
658 unsigned long flags;
659
660 if (!out_count || !port_has_data(port))
661 return 0;
662
663 buf = port->inbuf;
664 out_count = min(out_count, buf->len - buf->offset);
665
666 if (to_user) {
667 ssize_t ret;
668
669 ret = copy_to_user(to: out_buf, from: buf->buf + buf->offset, n: out_count);
670 if (ret)
671 return -EFAULT;
672 } else {
673 memcpy(to: (__force u8 *)out_buf, from: buf->buf + buf->offset,
674 len: out_count);
675 }
676
677 buf->offset += out_count;
678
679 if (buf->offset == buf->len) {
680 /*
681 * We're done using all the data in this buffer.
682 * Re-queue so that the Host can send us more data.
683 */
684 spin_lock_irqsave(&port->inbuf_lock, flags);
685 port->inbuf = NULL;
686
687 if (add_inbuf(vq: port->in_vq, buf) < 0)
688 dev_warn(port->dev, "failed add_buf\n");
689
690 spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
691 }
692 /* Return the number of bytes actually copied */
693 return out_count;
694}
695
696/* The condition that must be true for polling to end */
697static bool will_read_block(struct port *port)
698{
699 if (!port->guest_connected) {
700 /* Port got hot-unplugged. Let's exit. */
701 return false;
702 }
703 return !port_has_data(port) && port->host_connected;
704}
705
706static bool will_write_block(struct port *port)
707{
708 bool ret;
709
710 if (!port->guest_connected) {
711 /* Port got hot-unplugged. Let's exit. */
712 return false;
713 }
714 if (!port->host_connected)
715 return true;
716
717 spin_lock_irq(lock: &port->outvq_lock);
718 /*
719 * Check if the Host has consumed any buffers since we last
720 * sent data (this is only applicable for nonblocking ports).
721 */
722 reclaim_consumed_buffers(port);
723 ret = port->outvq_full;
724 spin_unlock_irq(lock: &port->outvq_lock);
725
726 return ret;
727}
728
729static ssize_t port_fops_read(struct file *filp, char __user *ubuf,
730 size_t count, loff_t *offp)
731{
732 struct port *port;
733 ssize_t ret;
734
735 port = filp->private_data;
736
737 /* Port is hot-unplugged. */
738 if (!port->guest_connected)
739 return -ENODEV;
740
741 if (!port_has_data(port)) {
742 /*
743 * If nothing's connected on the host just return 0 in
744 * case of list_empty; this tells the userspace app
745 * that there's no connection
746 */
747 if (!port->host_connected)
748 return 0;
749 if (filp->f_flags & O_NONBLOCK)
750 return -EAGAIN;
751
752 ret = wait_event_freezable(port->waitqueue,
753 !will_read_block(port));
754 if (ret < 0)
755 return ret;
756 }
757 /* Port got hot-unplugged while we were waiting above. */
758 if (!port->guest_connected)
759 return -ENODEV;
760 /*
761 * We could've received a disconnection message while we were
762 * waiting for more data.
763 *
764 * This check is not clubbed in the if() statement above as we
765 * might receive some data as well as the host could get
766 * disconnected after we got woken up from our wait. So we
767 * really want to give off whatever data we have and only then
768 * check for host_connected.
769 */
770 if (!port_has_data(port) && !port->host_connected)
771 return 0;
772
773 return fill_readbuf(port, out_buf: ubuf, out_count: count, to_user: true);
774}
775
776static int wait_port_writable(struct port *port, bool nonblock)
777{
778 int ret;
779
780 if (will_write_block(port)) {
781 if (nonblock)
782 return -EAGAIN;
783
784 ret = wait_event_freezable(port->waitqueue,
785 !will_write_block(port));
786 if (ret < 0)
787 return ret;
788 }
789 /* Port got hot-unplugged. */
790 if (!port->guest_connected)
791 return -ENODEV;
792
793 return 0;
794}
795
796static ssize_t port_fops_write(struct file *filp, const char __user *ubuf,
797 size_t count, loff_t *offp)
798{
799 struct port *port;
800 struct port_buffer *buf;
801 ssize_t ret;
802 bool nonblock;
803 struct scatterlist sg[1];
804
805 /* Userspace could be out to fool us */
806 if (!count)
807 return 0;
808
809 port = filp->private_data;
810
811 nonblock = filp->f_flags & O_NONBLOCK;
812
813 ret = wait_port_writable(port, nonblock);
814 if (ret < 0)
815 return ret;
816
817 count = min((size_t)(32 * 1024), count);
818
819 buf = alloc_buf(vdev: port->portdev->vdev, buf_size: count, pages: 0);
820 if (!buf)
821 return -ENOMEM;
822
823 ret = copy_from_user(to: buf->buf, from: ubuf, n: count);
824 if (ret) {
825 ret = -EFAULT;
826 goto free_buf;
827 }
828
829 /*
830 * We now ask send_buf() to not spin for generic ports -- we
831 * can re-use the same code path that non-blocking file
832 * descriptors take for blocking file descriptors since the
833 * wait is already done and we're certain the write will go
834 * through to the host.
835 */
836 nonblock = true;
837 sg_init_one(sg, buf->buf, count);
838 ret = __send_to_port(port, sg, nents: 1, in_count: count, data: buf, nonblock);
839
840 if (nonblock && ret > 0)
841 goto out;
842
843free_buf:
844 free_buf(buf, can_sleep: true);
845out:
846 return ret;
847}
848
849struct sg_list {
850 unsigned int n;
851 unsigned int size;
852 size_t len;
853 struct scatterlist *sg;
854};
855
856static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
857 struct splice_desc *sd)
858{
859 struct sg_list *sgl = sd->u.data;
860 unsigned int offset, len;
861
862 if (sgl->n == sgl->size)
863 return 0;
864
865 /* Try lock this page */
866 if (pipe_buf_try_steal(pipe, buf)) {
867 /* Get reference and unlock page for moving */
868 get_page(page: buf->page);
869 unlock_page(page: buf->page);
870
871 len = min(buf->len, sd->len);
872 sg_set_page(sg: &(sgl->sg[sgl->n]), page: buf->page, len, offset: buf->offset);
873 } else {
874 /* Failback to copying a page */
875 struct page *page = alloc_page(GFP_KERNEL);
876 char *src;
877
878 if (!page)
879 return -ENOMEM;
880
881 offset = sd->pos & ~PAGE_MASK;
882
883 len = sd->len;
884 if (len + offset > PAGE_SIZE)
885 len = PAGE_SIZE - offset;
886
887 src = kmap_local_page(page: buf->page);
888 memcpy(page_address(page) + offset, from: src + buf->offset, len);
889 kunmap_local(src);
890
891 sg_set_page(sg: &(sgl->sg[sgl->n]), page, len, offset);
892 }
893 sgl->n++;
894 sgl->len += len;
895
896 return len;
897}
898
899/* Faster zero-copy write by splicing */
900static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
901 struct file *filp, loff_t *ppos,
902 size_t len, unsigned int flags)
903{
904 struct port *port = filp->private_data;
905 struct sg_list sgl;
906 ssize_t ret;
907 struct port_buffer *buf;
908 struct splice_desc sd = {
909 .total_len = len,
910 .flags = flags,
911 .pos = *ppos,
912 .u.data = &sgl,
913 };
914 unsigned int occupancy;
915
916 /*
917 * Rproc_serial does not yet support splice. To support splice
918 * pipe_to_sg() must allocate dma-buffers and copy content from
919 * regular pages to dma pages. And alloc_buf and free_buf must
920 * support allocating and freeing such a list of dma-buffers.
921 */
922 if (is_rproc_serial(vdev: port->out_vq->vdev))
923 return -EINVAL;
924
925 pipe_lock(pipe);
926 ret = 0;
927 if (pipe_is_empty(pipe))
928 goto error_out;
929
930 ret = wait_port_writable(port, nonblock: filp->f_flags & O_NONBLOCK);
931 if (ret < 0)
932 goto error_out;
933
934 occupancy = pipe_buf_usage(pipe);
935 buf = alloc_buf(vdev: port->portdev->vdev, buf_size: 0, pages: occupancy);
936
937 if (!buf) {
938 ret = -ENOMEM;
939 goto error_out;
940 }
941
942 sgl.n = 0;
943 sgl.len = 0;
944 sgl.size = occupancy;
945 sgl.sg = buf->sg;
946 sg_init_table(sgl.sg, sgl.size);
947 ret = __splice_from_pipe(pipe, sd: &sd, actor: pipe_to_sg);
948 pipe_unlock(pipe);
949 if (likely(ret > 0))
950 ret = __send_to_port(port, sg: buf->sg, nents: sgl.n, in_count: sgl.len, data: buf, nonblock: true);
951
952 if (unlikely(ret <= 0))
953 free_buf(buf, can_sleep: true);
954 return ret;
955
956error_out:
957 pipe_unlock(pipe);
958 return ret;
959}
960
961static __poll_t port_fops_poll(struct file *filp, poll_table *wait)
962{
963 struct port *port;
964 __poll_t ret;
965
966 port = filp->private_data;
967 poll_wait(filp, wait_address: &port->waitqueue, p: wait);
968
969 if (!port->guest_connected) {
970 /* Port got unplugged */
971 return EPOLLHUP;
972 }
973 ret = 0;
974 if (!will_read_block(port))
975 ret |= EPOLLIN | EPOLLRDNORM;
976 if (!will_write_block(port))
977 ret |= EPOLLOUT;
978 if (!port->host_connected)
979 ret |= EPOLLHUP;
980
981 return ret;
982}
983
984static void remove_port(struct kref *kref);
985
986static int port_fops_release(struct inode *inode, struct file *filp)
987{
988 struct port *port;
989
990 port = filp->private_data;
991
992 /* Notify host of port being closed */
993 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 0);
994
995 spin_lock_irq(lock: &port->inbuf_lock);
996 port->guest_connected = false;
997
998 discard_port_data(port);
999
1000 spin_unlock_irq(lock: &port->inbuf_lock);
1001
1002 spin_lock_irq(lock: &port->outvq_lock);
1003 reclaim_consumed_buffers(port);
1004 spin_unlock_irq(lock: &port->outvq_lock);
1005
1006 reclaim_dma_bufs();
1007 /*
1008 * Locks aren't necessary here as a port can't be opened after
1009 * unplug, and if a port isn't unplugged, a kref would already
1010 * exist for the port. Plus, taking ports_lock here would
1011 * create a dependency on other locks taken by functions
1012 * inside remove_port if we're the last holder of the port,
1013 * creating many problems.
1014 */
1015 kref_put(kref: &port->kref, release: remove_port);
1016
1017 return 0;
1018}
1019
1020static int port_fops_open(struct inode *inode, struct file *filp)
1021{
1022 struct cdev *cdev = inode->i_cdev;
1023 struct port *port;
1024 int ret;
1025
1026 /* We get the port with a kref here */
1027 port = find_port_by_devt(dev: cdev->dev);
1028 if (!port) {
1029 /* Port was unplugged before we could proceed */
1030 return -ENXIO;
1031 }
1032 filp->private_data = port;
1033
1034 /*
1035 * Don't allow opening of console port devices -- that's done
1036 * via /dev/hvc
1037 */
1038 if (is_console_port(port)) {
1039 ret = -ENXIO;
1040 goto out;
1041 }
1042
1043 /* Allow only one process to open a particular port at a time */
1044 spin_lock_irq(lock: &port->inbuf_lock);
1045 if (port->guest_connected) {
1046 spin_unlock_irq(lock: &port->inbuf_lock);
1047 ret = -EBUSY;
1048 goto out;
1049 }
1050
1051 port->guest_connected = true;
1052 spin_unlock_irq(lock: &port->inbuf_lock);
1053
1054 spin_lock_irq(lock: &port->outvq_lock);
1055 /*
1056 * There might be a chance that we missed reclaiming a few
1057 * buffers in the window of the port getting previously closed
1058 * and opening now.
1059 */
1060 reclaim_consumed_buffers(port);
1061 spin_unlock_irq(lock: &port->outvq_lock);
1062
1063 nonseekable_open(inode, filp);
1064
1065 /* Notify host of port being opened */
1066 send_control_msg(port: filp->private_data, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
1067
1068 return 0;
1069out:
1070 kref_put(kref: &port->kref, release: remove_port);
1071 return ret;
1072}
1073
1074static int port_fops_fasync(int fd, struct file *filp, int mode)
1075{
1076 struct port *port;
1077
1078 port = filp->private_data;
1079 return fasync_helper(fd, filp, mode, &port->async_queue);
1080}
1081
1082/*
1083 * The file operations that we support: programs in the guest can open
1084 * a console device, read from it, write to it, poll for data and
1085 * close it. The devices are at
1086 * /dev/vport<device number>p<port number>
1087 */
1088static const struct file_operations port_fops = {
1089 .owner = THIS_MODULE,
1090 .open = port_fops_open,
1091 .read = port_fops_read,
1092 .write = port_fops_write,
1093 .splice_write = port_fops_splice_write,
1094 .poll = port_fops_poll,
1095 .release = port_fops_release,
1096 .fasync = port_fops_fasync,
1097};
1098
1099/*
1100 * The put_chars() callback is pretty straightforward.
1101 *
1102 * We turn the characters into a scatter-gather list, add it to the
1103 * output queue and then kick the Host. Then we sit here waiting for
1104 * it to finish: inefficient in theory, but in practice
1105 * implementations will do it immediately.
1106 */
1107static ssize_t put_chars(u32 vtermno, const u8 *buf, size_t count)
1108{
1109 struct port *port;
1110 struct scatterlist sg[1];
1111 void *data;
1112 int ret;
1113
1114 port = find_port_by_vtermno(vtermno);
1115 if (!port)
1116 return -EPIPE;
1117
1118 data = kmemdup(buf, count, GFP_ATOMIC);
1119 if (!data)
1120 return -ENOMEM;
1121
1122 sg_init_one(sg, data, count);
1123 ret = __send_to_port(port, sg, nents: 1, in_count: count, data, nonblock: false);
1124 kfree(objp: data);
1125 return ret;
1126}
1127
1128/*
1129 * get_chars() is the callback from the hvc_console infrastructure
1130 * when an interrupt is received.
1131 *
1132 * We call out to fill_readbuf that gets us the required data from the
1133 * buffers that are queued up.
1134 */
1135static ssize_t get_chars(u32 vtermno, u8 *buf, size_t count)
1136{
1137 struct port *port;
1138
1139 port = find_port_by_vtermno(vtermno);
1140 if (!port)
1141 return -EPIPE;
1142
1143 /* If we don't have an input queue yet, we can't get input. */
1144 BUG_ON(!port->in_vq);
1145
1146 return fill_readbuf(port, out_buf: (__force u8 __user *)buf, out_count: count, to_user: false);
1147}
1148
1149static void resize_console(struct port *port)
1150{
1151 struct virtio_device *vdev;
1152
1153 /* The port could have been hot-unplugged */
1154 if (!port || !is_console_port(port))
1155 return;
1156
1157 vdev = port->portdev->vdev;
1158
1159 /* Don't test F_SIZE at all if we're rproc: not a valid feature! */
1160 if (!is_rproc_serial(vdev) &&
1161 virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE))
1162 hvc_resize(hp: port->cons.hvc, ws: port->cons.ws);
1163}
1164
1165/* We set the configuration at this point, since we now have a tty */
1166static int notifier_add_vio(struct hvc_struct *hp, int data)
1167{
1168 struct port *port;
1169
1170 port = find_port_by_vtermno(vtermno: hp->vtermno);
1171 if (!port)
1172 return -EINVAL;
1173
1174 hp->irq_requested = 1;
1175 resize_console(port);
1176
1177 return 0;
1178}
1179
1180static void notifier_del_vio(struct hvc_struct *hp, int data)
1181{
1182 hp->irq_requested = 0;
1183}
1184
1185/* The operations for console ports. */
1186static const struct hv_ops hv_ops = {
1187 .get_chars = get_chars,
1188 .put_chars = put_chars,
1189 .notifier_add = notifier_add_vio,
1190 .notifier_del = notifier_del_vio,
1191 .notifier_hangup = notifier_del_vio,
1192};
1193
1194static int init_port_console(struct port *port)
1195{
1196 int ret;
1197
1198 /*
1199 * The Host's telling us this port is a console port. Hook it
1200 * up with an hvc console.
1201 *
1202 * To set up and manage our virtual console, we call
1203 * hvc_alloc().
1204 *
1205 * The first argument of hvc_alloc() is the virtual console
1206 * number. The second argument is the parameter for the
1207 * notification mechanism (like irq number). We currently
1208 * leave this as zero, virtqueues have implicit notifications.
1209 *
1210 * The third argument is a "struct hv_ops" containing the
1211 * put_chars() get_chars(), notifier_add() and notifier_del()
1212 * pointers. The final argument is the output buffer size: we
1213 * can do any size, so we put PAGE_SIZE here.
1214 */
1215 ret = ida_alloc_min(ida: &vtermno_ida, min: 1, GFP_KERNEL);
1216 if (ret < 0)
1217 return ret;
1218
1219 port->cons.vtermno = ret;
1220 port->cons.hvc = hvc_alloc(vtermno: port->cons.vtermno, data: 0, ops: &hv_ops, PAGE_SIZE);
1221 if (IS_ERR(ptr: port->cons.hvc)) {
1222 ret = PTR_ERR(ptr: port->cons.hvc);
1223 dev_err(port->dev,
1224 "error %d allocating hvc for port\n", ret);
1225 port->cons.hvc = NULL;
1226 ida_free(&vtermno_ida, id: port->cons.vtermno);
1227 return ret;
1228 }
1229 spin_lock_irq(lock: &pdrvdata_lock);
1230 list_add_tail(new: &port->cons.list, head: &pdrvdata.consoles);
1231 spin_unlock_irq(lock: &pdrvdata_lock);
1232 port->guest_connected = true;
1233
1234 /* Notify host of port being opened */
1235 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
1236
1237 return 0;
1238}
1239
1240static ssize_t show_port_name(struct device *dev,
1241 struct device_attribute *attr, char *buffer)
1242{
1243 struct port *port;
1244
1245 port = dev_get_drvdata(dev);
1246
1247 return sprintf(buf: buffer, fmt: "%s\n", port->name);
1248}
1249
1250static DEVICE_ATTR(name, S_IRUGO, show_port_name, NULL);
1251
1252static struct attribute *port_sysfs_entries[] = {
1253 &dev_attr_name.attr,
1254 NULL
1255};
1256
1257static const struct attribute_group port_attribute_group = {
1258 .name = NULL, /* put in device directory */
1259 .attrs = port_sysfs_entries,
1260};
1261
1262static int port_debugfs_show(struct seq_file *s, void *data)
1263{
1264 struct port *port = s->private;
1265
1266 seq_printf(m: s, fmt: "name: %s\n", port->name ? port->name : "");
1267 seq_printf(m: s, fmt: "guest_connected: %d\n", port->guest_connected);
1268 seq_printf(m: s, fmt: "host_connected: %d\n", port->host_connected);
1269 seq_printf(m: s, fmt: "outvq_full: %d\n", port->outvq_full);
1270 seq_printf(m: s, fmt: "bytes_sent: %lu\n", port->stats.bytes_sent);
1271 seq_printf(m: s, fmt: "bytes_received: %lu\n", port->stats.bytes_received);
1272 seq_printf(m: s, fmt: "bytes_discarded: %lu\n", port->stats.bytes_discarded);
1273 seq_printf(m: s, fmt: "is_console: %s\n", str_yes_no(v: is_console_port(port)));
1274 seq_printf(m: s, fmt: "console_vtermno: %u\n", port->cons.vtermno);
1275
1276 return 0;
1277}
1278
1279DEFINE_SHOW_ATTRIBUTE(port_debugfs);
1280
1281static void set_console_size(struct port *port, u16 rows, u16 cols)
1282{
1283 if (!port || !is_console_port(port))
1284 return;
1285
1286 port->cons.ws.ws_row = rows;
1287 port->cons.ws.ws_col = cols;
1288}
1289
1290static int fill_queue(struct virtqueue *vq, spinlock_t *lock)
1291{
1292 struct port_buffer *buf;
1293 int nr_added_bufs;
1294 int ret;
1295
1296 nr_added_bufs = 0;
1297 do {
1298 buf = alloc_buf(vdev: vq->vdev, PAGE_SIZE, pages: 0);
1299 if (!buf)
1300 return -ENOMEM;
1301
1302 spin_lock_irq(lock);
1303 ret = add_inbuf(vq, buf);
1304 if (ret < 0) {
1305 spin_unlock_irq(lock);
1306 free_buf(buf, can_sleep: true);
1307 return ret;
1308 }
1309 nr_added_bufs++;
1310 spin_unlock_irq(lock);
1311 } while (ret > 0);
1312
1313 return nr_added_bufs;
1314}
1315
1316static void send_sigio_to_port(struct port *port)
1317{
1318 if (port->async_queue && port->guest_connected)
1319 kill_fasync(&port->async_queue, SIGIO, POLL_OUT);
1320}
1321
1322static int add_port(struct ports_device *portdev, u32 id)
1323{
1324 struct port *port;
1325 dev_t devt;
1326 int err;
1327
1328 port = kmalloc(sizeof(*port), GFP_KERNEL);
1329 if (!port) {
1330 err = -ENOMEM;
1331 goto fail;
1332 }
1333 kref_init(kref: &port->kref);
1334
1335 port->portdev = portdev;
1336 port->id = id;
1337
1338 port->name = NULL;
1339 port->inbuf = NULL;
1340 port->cons.hvc = NULL;
1341 port->async_queue = NULL;
1342
1343 port->cons.ws.ws_row = port->cons.ws.ws_col = 0;
1344 port->cons.vtermno = 0;
1345
1346 port->host_connected = port->guest_connected = false;
1347 port->stats = (struct port_stats) { 0 };
1348
1349 port->outvq_full = false;
1350
1351 port->in_vq = portdev->in_vqs[port->id];
1352 port->out_vq = portdev->out_vqs[port->id];
1353
1354 port->cdev = cdev_alloc();
1355 if (!port->cdev) {
1356 dev_err(&port->portdev->vdev->dev, "Error allocating cdev\n");
1357 err = -ENOMEM;
1358 goto free_port;
1359 }
1360 port->cdev->ops = &port_fops;
1361
1362 devt = MKDEV(portdev->chr_major, id);
1363 err = cdev_add(port->cdev, devt, 1);
1364 if (err < 0) {
1365 dev_err(&port->portdev->vdev->dev,
1366 "Error %d adding cdev for port %u\n", err, id);
1367 goto free_cdev;
1368 }
1369 port->dev = device_create(&port_class, &port->portdev->vdev->dev,
1370 devt, port, "vport%up%u",
1371 port->portdev->vdev->index, id);
1372 if (IS_ERR(port->dev)) {
1373 err = PTR_ERR(port->dev);
1374 dev_err(&port->portdev->vdev->dev,
1375 "Error %d creating device for port %u\n",
1376 err, id);
1377 goto free_cdev;
1378 }
1379
1380 spin_lock_init(&port->inbuf_lock);
1381 spin_lock_init(&port->outvq_lock);
1382 init_waitqueue_head(&port->waitqueue);
1383
1384 /* We can safely ignore ENOSPC because it means
1385 * the queue already has buffers. Buffers are removed
1386 * only by virtcons_remove(), not by unplug_port()
1387 */
1388 err = fill_queue(port->in_vq, &port->inbuf_lock);
1389 if (err < 0 && err != -ENOSPC) {
1390 dev_err(port->dev, "Error allocating inbufs\n");
1391 goto free_device;
1392 }
1393
1394 if (is_rproc_serial(port->portdev->vdev))
1395 /*
1396 * For rproc_serial assume remote processor is connected.
1397 * rproc_serial does not want the console port, only
1398 * the generic port implementation.
1399 */
1400 port->host_connected = true;
1401 else if (!use_multiport(port->portdev)) {
1402 /*
1403 * If we're not using multiport support,
1404 * this has to be a console port.
1405 */
1406 err = init_port_console(port);
1407 if (err)
1408 goto free_inbufs;
1409 }
1410
1411 spin_lock_irq(&portdev->ports_lock);
1412 list_add_tail(&port->list, &port->portdev->ports);
1413 spin_unlock_irq(&portdev->ports_lock);
1414
1415 /*
1416 * Tell the Host we're set so that it can send us various
1417 * configuration parameters for this port (eg, port name,
1418 * caching, whether this is a console port, etc.)
1419 */
1420 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1421
1422 /*
1423 * Finally, create the debugfs file that we can use to
1424 * inspect a port's state at any time
1425 */
1426 port->debugfs_file = debugfs_create_file(dev_name(port->dev), 0444,
1427 pdrvdata.debugfs_dir,
1428 port, &port_debugfs_fops);
1429 return 0;
1430
1431free_inbufs:
1432free_device:
1433 device_destroy(&port_class, port->dev->devt);
1434free_cdev:
1435 cdev_del(port->cdev);
1436free_port:
1437 kfree(port);
1438fail:
1439 /* The host might want to notify management sw about port add failure */
1440 __send_control_msg(portdev, id, VIRTIO_CONSOLE_PORT_READY, 0);
1441 return err;
1442}
1443
1444/* No users remain, remove all port-specific data. */
1445static void remove_port(struct kref *kref)
1446{
1447 struct port *port;
1448
1449 port = container_of(kref, struct port, kref);
1450
1451 kfree(objp: port);
1452}
1453
1454static void remove_port_data(struct port *port)
1455{
1456 spin_lock_irq(lock: &port->inbuf_lock);
1457 /* Remove unused data this port might have received. */
1458 discard_port_data(port);
1459 spin_unlock_irq(lock: &port->inbuf_lock);
1460
1461 spin_lock_irq(lock: &port->outvq_lock);
1462 reclaim_consumed_buffers(port);
1463 spin_unlock_irq(lock: &port->outvq_lock);
1464}
1465
1466/*
1467 * Port got unplugged. Remove port from portdev's list and drop the
1468 * kref reference. If no userspace has this port opened, it will
1469 * result in immediate removal the port.
1470 */
1471static void unplug_port(struct port *port)
1472{
1473 spin_lock_irq(lock: &port->portdev->ports_lock);
1474 list_del(entry: &port->list);
1475 spin_unlock_irq(lock: &port->portdev->ports_lock);
1476
1477 spin_lock_irq(lock: &port->inbuf_lock);
1478 if (port->guest_connected) {
1479 /* Let the app know the port is going down. */
1480 send_sigio_to_port(port);
1481
1482 /* Do this after sigio is actually sent */
1483 port->guest_connected = false;
1484 port->host_connected = false;
1485
1486 wake_up_interruptible(&port->waitqueue);
1487 }
1488 spin_unlock_irq(lock: &port->inbuf_lock);
1489
1490 if (is_console_port(port)) {
1491 spin_lock_irq(lock: &pdrvdata_lock);
1492 list_del(entry: &port->cons.list);
1493 spin_unlock_irq(lock: &pdrvdata_lock);
1494 hvc_remove(hp: port->cons.hvc);
1495 ida_free(&vtermno_ida, id: port->cons.vtermno);
1496 }
1497
1498 remove_port_data(port);
1499
1500 /*
1501 * We should just assume the device itself has gone off --
1502 * else a close on an open port later will try to send out a
1503 * control message.
1504 */
1505 port->portdev = NULL;
1506
1507 sysfs_remove_group(kobj: &port->dev->kobj, grp: &port_attribute_group);
1508 device_destroy(cls: &port_class, devt: port->dev->devt);
1509 cdev_del(port->cdev);
1510
1511 debugfs_remove(dentry: port->debugfs_file);
1512 kfree(objp: port->name);
1513
1514 /*
1515 * Locks around here are not necessary - a port can't be
1516 * opened after we removed the port struct from ports_list
1517 * above.
1518 */
1519 kref_put(kref: &port->kref, release: remove_port);
1520}
1521
1522/* Any private messages that the Host and Guest want to share */
1523static void handle_control_message(struct virtio_device *vdev,
1524 struct ports_device *portdev,
1525 struct port_buffer *buf)
1526{
1527 struct virtio_console_control *cpkt;
1528 struct port *port;
1529 size_t name_size;
1530 int err;
1531
1532 cpkt = (struct virtio_console_control *)(buf->buf + buf->offset);
1533
1534 port = find_port_by_id(portdev, id: virtio32_to_cpu(vdev, val: cpkt->id));
1535 if (!port &&
1536 cpkt->event != cpu_to_virtio16(vdev, VIRTIO_CONSOLE_PORT_ADD)) {
1537 /* No valid header at start of buffer. Drop it. */
1538 dev_dbg(&portdev->vdev->dev,
1539 "Invalid index %u in control packet\n", cpkt->id);
1540 return;
1541 }
1542
1543 switch (virtio16_to_cpu(vdev, val: cpkt->event)) {
1544 case VIRTIO_CONSOLE_PORT_ADD:
1545 if (port) {
1546 dev_dbg(&portdev->vdev->dev,
1547 "Port %u already added\n", port->id);
1548 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, value: 1);
1549 break;
1550 }
1551 if (virtio32_to_cpu(vdev, val: cpkt->id) >=
1552 portdev->max_nr_ports) {
1553 dev_warn(&portdev->vdev->dev,
1554 "Request for adding port with "
1555 "out-of-bound id %u, max. supported id: %u\n",
1556 cpkt->id, portdev->max_nr_ports - 1);
1557 break;
1558 }
1559 add_port(portdev, id: virtio32_to_cpu(vdev, val: cpkt->id));
1560 break;
1561 case VIRTIO_CONSOLE_PORT_REMOVE:
1562 unplug_port(port);
1563 break;
1564 case VIRTIO_CONSOLE_CONSOLE_PORT:
1565 if (!cpkt->value)
1566 break;
1567 if (is_console_port(port))
1568 break;
1569
1570 init_port_console(port);
1571 complete(&early_console_added);
1572 /*
1573 * Could remove the port here in case init fails - but
1574 * have to notify the host first.
1575 */
1576 break;
1577 case VIRTIO_CONSOLE_RESIZE: {
1578 struct {
1579 __virtio16 cols;
1580 __virtio16 rows;
1581 } size;
1582
1583 if (!is_console_port(port))
1584 break;
1585
1586 memcpy(to: &size, from: buf->buf + buf->offset + sizeof(*cpkt),
1587 len: sizeof(size));
1588 set_console_size(port, rows: virtio16_to_cpu(vdev, val: size.rows),
1589 cols: virtio16_to_cpu(vdev, val: size.cols));
1590
1591 port->cons.hvc->irq_requested = 1;
1592 resize_console(port);
1593 break;
1594 }
1595 case VIRTIO_CONSOLE_PORT_OPEN:
1596 port->host_connected = virtio16_to_cpu(vdev, val: cpkt->value);
1597 wake_up_interruptible(&port->waitqueue);
1598 /*
1599 * If the host port got closed and the host had any
1600 * unconsumed buffers, we'll be able to reclaim them
1601 * now.
1602 */
1603 spin_lock_irq(lock: &port->outvq_lock);
1604 reclaim_consumed_buffers(port);
1605 spin_unlock_irq(lock: &port->outvq_lock);
1606
1607 /*
1608 * If the guest is connected, it'll be interested in
1609 * knowing the host connection state changed.
1610 */
1611 spin_lock_irq(lock: &port->inbuf_lock);
1612 send_sigio_to_port(port);
1613 spin_unlock_irq(lock: &port->inbuf_lock);
1614 break;
1615 case VIRTIO_CONSOLE_PORT_NAME:
1616 /*
1617 * If we woke up after hibernation, we can get this
1618 * again. Skip it in that case.
1619 */
1620 if (port->name)
1621 break;
1622
1623 /*
1624 * Skip the size of the header and the cpkt to get the size
1625 * of the name that was sent
1626 */
1627 name_size = buf->len - buf->offset - sizeof(*cpkt) + 1;
1628
1629 port->name = kmalloc(name_size, GFP_KERNEL);
1630 if (!port->name) {
1631 dev_err(port->dev,
1632 "Not enough space to store port name\n");
1633 break;
1634 }
1635 strscpy(port->name, buf->buf + buf->offset + sizeof(*cpkt),
1636 name_size);
1637
1638 /*
1639 * Since we only have one sysfs attribute, 'name',
1640 * create it only if we have a name for the port.
1641 */
1642 err = sysfs_create_group(kobj: &port->dev->kobj,
1643 grp: &port_attribute_group);
1644 if (err) {
1645 dev_err(port->dev,
1646 "Error %d creating sysfs device attributes\n",
1647 err);
1648 } else {
1649 /*
1650 * Generate a udev event so that appropriate
1651 * symlinks can be created based on udev
1652 * rules.
1653 */
1654 kobject_uevent(kobj: &port->dev->kobj, action: KOBJ_CHANGE);
1655 }
1656 break;
1657 }
1658}
1659
1660static void control_work_handler(struct work_struct *work)
1661{
1662 struct ports_device *portdev;
1663 struct virtqueue *vq;
1664 struct port_buffer *buf;
1665 unsigned int len;
1666
1667 portdev = container_of(work, struct ports_device, control_work);
1668 vq = portdev->c_ivq;
1669
1670 spin_lock(lock: &portdev->c_ivq_lock);
1671 while ((buf = virtqueue_get_buf(vq, len: &len))) {
1672 spin_unlock(lock: &portdev->c_ivq_lock);
1673
1674 buf->len = min_t(size_t, len, buf->size);
1675 buf->offset = 0;
1676
1677 handle_control_message(vdev: vq->vdev, portdev, buf);
1678
1679 spin_lock(lock: &portdev->c_ivq_lock);
1680 if (add_inbuf(vq: portdev->c_ivq, buf) < 0) {
1681 dev_warn(&portdev->vdev->dev,
1682 "Error adding buffer to queue\n");
1683 free_buf(buf, can_sleep: false);
1684 }
1685 }
1686 spin_unlock(lock: &portdev->c_ivq_lock);
1687}
1688
1689static void flush_bufs(struct virtqueue *vq, bool can_sleep)
1690{
1691 struct port_buffer *buf;
1692 unsigned int len;
1693
1694 while ((buf = virtqueue_get_buf(vq, len: &len)))
1695 free_buf(buf, can_sleep);
1696}
1697
1698static void out_intr(struct virtqueue *vq)
1699{
1700 struct port *port;
1701
1702 port = find_port_by_vq(portdev: vq->vdev->priv, vq);
1703 if (!port) {
1704 flush_bufs(vq, can_sleep: false);
1705 return;
1706 }
1707
1708 wake_up_interruptible(&port->waitqueue);
1709}
1710
1711static void in_intr(struct virtqueue *vq)
1712{
1713 struct port *port;
1714 unsigned long flags;
1715
1716 port = find_port_by_vq(portdev: vq->vdev->priv, vq);
1717 if (!port) {
1718 flush_bufs(vq, can_sleep: false);
1719 return;
1720 }
1721
1722 spin_lock_irqsave(&port->inbuf_lock, flags);
1723 port->inbuf = get_inbuf(port);
1724
1725 /*
1726 * Normally the port should not accept data when the port is
1727 * closed. For generic serial ports, the host won't (shouldn't)
1728 * send data till the guest is connected. But this condition
1729 * can be reached when a console port is not yet connected (no
1730 * tty is spawned) and the other side sends out data over the
1731 * vring, or when a remote devices start sending data before
1732 * the ports are opened.
1733 *
1734 * A generic serial port will discard data if not connected,
1735 * while console ports and rproc-serial ports accepts data at
1736 * any time. rproc-serial is initiated with guest_connected to
1737 * false because port_fops_open expects this. Console ports are
1738 * hooked up with an HVC console and is initialized with
1739 * guest_connected to true.
1740 */
1741
1742 if (!port->guest_connected && !is_rproc_serial(vdev: port->portdev->vdev))
1743 discard_port_data(port);
1744
1745 /* Send a SIGIO indicating new data in case the process asked for it */
1746 send_sigio_to_port(port);
1747
1748 spin_unlock_irqrestore(lock: &port->inbuf_lock, flags);
1749
1750 wake_up_interruptible(&port->waitqueue);
1751
1752 if (is_console_port(port) && hvc_poll(hp: port->cons.hvc))
1753 hvc_kick();
1754}
1755
1756static void control_intr(struct virtqueue *vq)
1757{
1758 struct ports_device *portdev;
1759
1760 portdev = vq->vdev->priv;
1761 schedule_work(work: &portdev->control_work);
1762}
1763
1764static void config_intr(struct virtio_device *vdev)
1765{
1766 struct ports_device *portdev;
1767
1768 portdev = vdev->priv;
1769
1770 if (!use_multiport(portdev))
1771 schedule_work(work: &portdev->config_work);
1772}
1773
1774static void config_work_handler(struct work_struct *work)
1775{
1776 struct ports_device *portdev;
1777
1778 portdev = container_of(work, struct ports_device, config_work);
1779 if (!use_multiport(portdev)) {
1780 struct virtio_device *vdev;
1781 struct port *port;
1782 u16 rows, cols;
1783
1784 vdev = portdev->vdev;
1785 virtio_cread(vdev, struct virtio_console_config, cols, &cols);
1786 virtio_cread(vdev, struct virtio_console_config, rows, &rows);
1787
1788 port = find_port_by_id(portdev, id: 0);
1789 set_console_size(port, rows, cols);
1790
1791 /*
1792 * We'll use this way of resizing only for legacy
1793 * support. For newer userspace
1794 * (VIRTIO_CONSOLE_F_MULTPORT+), use control messages
1795 * to indicate console size changes so that it can be
1796 * done per-port.
1797 */
1798 resize_console(port);
1799 }
1800}
1801
1802static int init_vqs(struct ports_device *portdev)
1803{
1804 struct virtqueue_info *vqs_info;
1805 struct virtqueue **vqs;
1806 u32 i, j, nr_ports, nr_queues;
1807 int err;
1808
1809 nr_ports = portdev->max_nr_ports;
1810 nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
1811
1812 vqs = kmalloc_array(nr_queues, sizeof(struct virtqueue *), GFP_KERNEL);
1813 vqs_info = kcalloc(nr_queues, sizeof(*vqs_info), GFP_KERNEL);
1814 portdev->in_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
1815 GFP_KERNEL);
1816 portdev->out_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
1817 GFP_KERNEL);
1818 if (!vqs || !vqs_info || !portdev->in_vqs || !portdev->out_vqs) {
1819 err = -ENOMEM;
1820 goto free;
1821 }
1822
1823 /*
1824 * For backward compat (newer host but older guest), the host
1825 * spawns a console port first and also inits the vqs for port
1826 * 0 before others.
1827 */
1828 j = 0;
1829 vqs_info[j].callback = in_intr;
1830 vqs_info[j + 1].callback = out_intr;
1831 vqs_info[j].name = "input";
1832 vqs_info[j + 1].name = "output";
1833 j += 2;
1834
1835 if (use_multiport(portdev)) {
1836 vqs_info[j].callback = control_intr;
1837 vqs_info[j].name = "control-i";
1838 vqs_info[j + 1].name = "control-o";
1839
1840 for (i = 1; i < nr_ports; i++) {
1841 j += 2;
1842 vqs_info[j].callback = in_intr;
1843 vqs_info[j + 1].callback = out_intr;
1844 vqs_info[j].name = "input";
1845 vqs_info[j + 1].name = "output";
1846 }
1847 }
1848 /* Find the queues. */
1849 err = virtio_find_vqs(vdev: portdev->vdev, nvqs: nr_queues, vqs, vqs_info, NULL);
1850 if (err)
1851 goto free;
1852
1853 j = 0;
1854 portdev->in_vqs[0] = vqs[0];
1855 portdev->out_vqs[0] = vqs[1];
1856 j += 2;
1857 if (use_multiport(portdev)) {
1858 portdev->c_ivq = vqs[j];
1859 portdev->c_ovq = vqs[j + 1];
1860
1861 for (i = 1; i < nr_ports; i++) {
1862 j += 2;
1863 portdev->in_vqs[i] = vqs[j];
1864 portdev->out_vqs[i] = vqs[j + 1];
1865 }
1866 }
1867 kfree(objp: vqs_info);
1868 kfree(objp: vqs);
1869
1870 return 0;
1871
1872free:
1873 kfree(objp: portdev->out_vqs);
1874 kfree(objp: portdev->in_vqs);
1875 kfree(objp: vqs_info);
1876 kfree(objp: vqs);
1877
1878 return err;
1879}
1880
1881static const struct file_operations portdev_fops = {
1882 .owner = THIS_MODULE,
1883};
1884
1885static void remove_vqs(struct ports_device *portdev)
1886{
1887 struct virtqueue *vq;
1888
1889 virtio_device_for_each_vq(portdev->vdev, vq) {
1890 struct port_buffer *buf;
1891
1892 flush_bufs(vq, can_sleep: true);
1893 while ((buf = virtqueue_detach_unused_buf(vq)))
1894 free_buf(buf, can_sleep: true);
1895 cond_resched();
1896 }
1897 portdev->vdev->config->del_vqs(portdev->vdev);
1898 kfree(objp: portdev->in_vqs);
1899 kfree(objp: portdev->out_vqs);
1900}
1901
1902static void virtcons_remove(struct virtio_device *vdev)
1903{
1904 struct ports_device *portdev;
1905 struct port *port, *port2;
1906
1907 portdev = vdev->priv;
1908
1909 spin_lock_irq(lock: &pdrvdata_lock);
1910 list_del(entry: &portdev->list);
1911 spin_unlock_irq(lock: &pdrvdata_lock);
1912
1913 /* Device is going away, exit any polling for buffers */
1914 virtio_break_device(dev: vdev);
1915 if (use_multiport(portdev))
1916 flush_work(work: &portdev->control_work);
1917 else
1918 flush_work(work: &portdev->config_work);
1919
1920 /* Disable interrupts for vqs */
1921 virtio_reset_device(dev: vdev);
1922 /* Finish up work that's lined up */
1923 if (use_multiport(portdev))
1924 cancel_work_sync(work: &portdev->control_work);
1925 else
1926 cancel_work_sync(work: &portdev->config_work);
1927
1928 list_for_each_entry_safe(port, port2, &portdev->ports, list)
1929 unplug_port(port);
1930
1931 unregister_chrdev(major: portdev->chr_major, name: "virtio-portsdev");
1932
1933 /*
1934 * When yanking out a device, we immediately lose the
1935 * (device-side) queues. So there's no point in keeping the
1936 * guest side around till we drop our final reference. This
1937 * also means that any ports which are in an open state will
1938 * have to just stop using the port, as the vqs are going
1939 * away.
1940 */
1941 remove_vqs(portdev);
1942 kfree(objp: portdev);
1943}
1944
1945/*
1946 * Once we're further in boot, we get probed like any other virtio
1947 * device.
1948 *
1949 * If the host also supports multiple console ports, we check the
1950 * config space to see how many ports the host has spawned. We
1951 * initialize each port found.
1952 */
1953static int virtcons_probe(struct virtio_device *vdev)
1954{
1955 struct ports_device *portdev;
1956 int err;
1957 bool multiport;
1958
1959 /* We only need a config space if features are offered */
1960 if (!vdev->config->get &&
1961 (virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE)
1962 || virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT))) {
1963 dev_err(&vdev->dev, "%s failure: config access disabled\n",
1964 __func__);
1965 return -EINVAL;
1966 }
1967
1968 portdev = kmalloc(sizeof(*portdev), GFP_KERNEL);
1969 if (!portdev) {
1970 err = -ENOMEM;
1971 goto fail;
1972 }
1973
1974 /* Attach this portdev to this virtio_device, and vice-versa. */
1975 portdev->vdev = vdev;
1976 vdev->priv = portdev;
1977
1978 portdev->chr_major = register_chrdev(major: 0, name: "virtio-portsdev",
1979 fops: &portdev_fops);
1980 if (portdev->chr_major < 0) {
1981 dev_err(&vdev->dev,
1982 "Error %d registering chrdev for device %u\n",
1983 portdev->chr_major, vdev->index);
1984 err = portdev->chr_major;
1985 goto free;
1986 }
1987
1988 multiport = false;
1989 portdev->max_nr_ports = 1;
1990
1991 /* Don't test MULTIPORT at all if we're rproc: not a valid feature! */
1992 if (!is_rproc_serial(vdev) &&
1993 virtio_cread_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT,
1994 struct virtio_console_config, max_nr_ports,
1995 &portdev->max_nr_ports) == 0) {
1996 if (portdev->max_nr_ports == 0 ||
1997 portdev->max_nr_ports > VIRTCONS_MAX_PORTS) {
1998 dev_err(&vdev->dev,
1999 "Invalidate max_nr_ports %d",
2000 portdev->max_nr_ports);
2001 err = -EINVAL;
2002 goto free;
2003 }
2004 multiport = true;
2005 }
2006
2007 spin_lock_init(&portdev->ports_lock);
2008 INIT_LIST_HEAD(list: &portdev->ports);
2009 INIT_LIST_HEAD(list: &portdev->list);
2010
2011 INIT_WORK(&portdev->config_work, &config_work_handler);
2012 INIT_WORK(&portdev->control_work, &control_work_handler);
2013
2014 if (multiport) {
2015 spin_lock_init(&portdev->c_ivq_lock);
2016 spin_lock_init(&portdev->c_ovq_lock);
2017 }
2018
2019 err = init_vqs(portdev);
2020 if (err < 0) {
2021 dev_err(&vdev->dev, "Error %d initializing vqs\n", err);
2022 goto free_chrdev;
2023 }
2024
2025 virtio_device_ready(dev: portdev->vdev);
2026
2027 if (multiport) {
2028 err = fill_queue(vq: portdev->c_ivq, lock: &portdev->c_ivq_lock);
2029 if (err < 0) {
2030 dev_err(&vdev->dev,
2031 "Error allocating buffers for control queue\n");
2032 /*
2033 * The host might want to notify mgmt sw about device
2034 * add failure.
2035 */
2036 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2037 VIRTIO_CONSOLE_DEVICE_READY, value: 0);
2038 /* Device was functional: we need full cleanup. */
2039 virtcons_remove(vdev);
2040 return err;
2041 }
2042 } else {
2043 /*
2044 * For backward compatibility: Create a console port
2045 * if we're running on older host.
2046 */
2047 add_port(portdev, id: 0);
2048 }
2049
2050 spin_lock_irq(lock: &pdrvdata_lock);
2051 list_add_tail(new: &portdev->list, head: &pdrvdata.portdevs);
2052 spin_unlock_irq(lock: &pdrvdata_lock);
2053
2054 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2055 VIRTIO_CONSOLE_DEVICE_READY, value: 1);
2056
2057 return 0;
2058
2059free_chrdev:
2060 unregister_chrdev(major: portdev->chr_major, name: "virtio-portsdev");
2061free:
2062 kfree(objp: portdev);
2063fail:
2064 return err;
2065}
2066
2067static const struct virtio_device_id id_table[] = {
2068 { VIRTIO_ID_CONSOLE, VIRTIO_DEV_ANY_ID },
2069 { 0 },
2070};
2071MODULE_DEVICE_TABLE(virtio, id_table);
2072
2073static const unsigned int features[] = {
2074 VIRTIO_CONSOLE_F_SIZE,
2075 VIRTIO_CONSOLE_F_MULTIPORT,
2076};
2077
2078static const struct virtio_device_id rproc_serial_id_table[] = {
2079#if IS_ENABLED(CONFIG_REMOTEPROC)
2080 { VIRTIO_ID_RPROC_SERIAL, VIRTIO_DEV_ANY_ID },
2081#endif
2082 { 0 },
2083};
2084MODULE_DEVICE_TABLE(virtio, rproc_serial_id_table);
2085
2086static const unsigned int rproc_serial_features[] = {
2087};
2088
2089#ifdef CONFIG_PM_SLEEP
2090static int virtcons_freeze(struct virtio_device *vdev)
2091{
2092 struct ports_device *portdev;
2093 struct port *port;
2094
2095 portdev = vdev->priv;
2096
2097 virtio_reset_device(dev: vdev);
2098
2099 if (use_multiport(portdev))
2100 virtqueue_disable_cb(vq: portdev->c_ivq);
2101 cancel_work_sync(work: &portdev->control_work);
2102 cancel_work_sync(work: &portdev->config_work);
2103 /*
2104 * Once more: if control_work_handler() was running, it would
2105 * enable the cb as the last step.
2106 */
2107 if (use_multiport(portdev))
2108 virtqueue_disable_cb(vq: portdev->c_ivq);
2109
2110 list_for_each_entry(port, &portdev->ports, list) {
2111 virtqueue_disable_cb(vq: port->in_vq);
2112 virtqueue_disable_cb(vq: port->out_vq);
2113 /*
2114 * We'll ask the host later if the new invocation has
2115 * the port opened or closed.
2116 */
2117 port->host_connected = false;
2118 remove_port_data(port);
2119 }
2120 remove_vqs(portdev);
2121
2122 return 0;
2123}
2124
2125static int virtcons_restore(struct virtio_device *vdev)
2126{
2127 struct ports_device *portdev;
2128 struct port *port;
2129 int ret;
2130
2131 portdev = vdev->priv;
2132
2133 ret = init_vqs(portdev);
2134 if (ret)
2135 return ret;
2136
2137 virtio_device_ready(dev: portdev->vdev);
2138
2139 if (use_multiport(portdev))
2140 fill_queue(vq: portdev->c_ivq, lock: &portdev->c_ivq_lock);
2141
2142 list_for_each_entry(port, &portdev->ports, list) {
2143 port->in_vq = portdev->in_vqs[port->id];
2144 port->out_vq = portdev->out_vqs[port->id];
2145
2146 fill_queue(vq: port->in_vq, lock: &port->inbuf_lock);
2147
2148 /* Get port open/close status on the host */
2149 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, value: 1);
2150
2151 /*
2152 * If a port was open at the time of suspending, we
2153 * have to let the host know that it's still open.
2154 */
2155 if (port->guest_connected)
2156 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, value: 1);
2157 }
2158 return 0;
2159}
2160#endif
2161
2162static struct virtio_driver virtio_console = {
2163 .feature_table = features,
2164 .feature_table_size = ARRAY_SIZE(features),
2165 .driver.name = KBUILD_MODNAME,
2166 .id_table = id_table,
2167 .probe = virtcons_probe,
2168 .remove = virtcons_remove,
2169 .config_changed = config_intr,
2170#ifdef CONFIG_PM_SLEEP
2171 .freeze = virtcons_freeze,
2172 .restore = virtcons_restore,
2173#endif
2174};
2175
2176static struct virtio_driver virtio_rproc_serial = {
2177 .feature_table = rproc_serial_features,
2178 .feature_table_size = ARRAY_SIZE(rproc_serial_features),
2179 .driver.name = "virtio_rproc_serial",
2180 .id_table = rproc_serial_id_table,
2181 .probe = virtcons_probe,
2182 .remove = virtcons_remove,
2183};
2184
2185static int __init virtio_console_init(void)
2186{
2187 int err;
2188
2189 err = class_register(class: &port_class);
2190 if (err)
2191 return err;
2192
2193 pdrvdata.debugfs_dir = debugfs_create_dir(name: "virtio-ports", NULL);
2194 INIT_LIST_HEAD(list: &pdrvdata.consoles);
2195 INIT_LIST_HEAD(list: &pdrvdata.portdevs);
2196
2197 err = register_virtio_driver(&virtio_console);
2198 if (err < 0) {
2199 pr_err("Error %d registering virtio driver\n", err);
2200 goto free;
2201 }
2202 err = register_virtio_driver(&virtio_rproc_serial);
2203 if (err < 0) {
2204 pr_err("Error %d registering virtio rproc serial driver\n",
2205 err);
2206 goto unregister;
2207 }
2208 return 0;
2209unregister:
2210 unregister_virtio_driver(drv: &virtio_console);
2211free:
2212 debugfs_remove_recursive(dentry: pdrvdata.debugfs_dir);
2213 class_unregister(class: &port_class);
2214 return err;
2215}
2216
2217static void __exit virtio_console_fini(void)
2218{
2219 reclaim_dma_bufs();
2220
2221 unregister_virtio_driver(drv: &virtio_console);
2222 unregister_virtio_driver(drv: &virtio_rproc_serial);
2223
2224 class_unregister(class: &port_class);
2225 debugfs_remove_recursive(dentry: pdrvdata.debugfs_dir);
2226}
2227module_init(virtio_console_init);
2228module_exit(virtio_console_fini);
2229
2230MODULE_DESCRIPTION("Virtio console driver");
2231MODULE_LICENSE("GPL");
2232