1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/net/sunrpc/svcsock.c
4 *
5 * These are the RPC server socket internals.
6 *
7 * The server scheduling algorithm does not always distribute the load
8 * evenly when servicing a single client. May need to modify the
9 * svc_xprt_enqueue procedure...
10 *
11 * TCP support is largely untested and may be a little slow. The problem
12 * is that we currently do two separate recvfrom's, one for the 4-byte
13 * record length, and the second for the actual record. This could possibly
14 * be improved by always reading a minimum size of around 100 bytes and
15 * tucking any superfluous bytes away in a temporary store. Still, that
16 * leaves write requests out in the rain. An alternative may be to peek at
17 * the first skb in the queue, and if it matches the next TCP sequence
18 * number, to extract the record marker. Yuck.
19 *
20 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
21 */
22
23#include <linux/kernel.h>
24#include <linux/sched.h>
25#include <linux/module.h>
26#include <linux/errno.h>
27#include <linux/fcntl.h>
28#include <linux/net.h>
29#include <linux/in.h>
30#include <linux/inet.h>
31#include <linux/udp.h>
32#include <linux/tcp.h>
33#include <linux/unistd.h>
34#include <linux/slab.h>
35#include <linux/netdevice.h>
36#include <linux/skbuff.h>
37#include <linux/file.h>
38#include <linux/freezer.h>
39#include <linux/bvec.h>
40
41#include <net/sock.h>
42#include <net/checksum.h>
43#include <net/ip.h>
44#include <net/ipv6.h>
45#include <net/udp.h>
46#include <net/tcp.h>
47#include <net/tcp_states.h>
48#include <net/tls_prot.h>
49#include <net/handshake.h>
50#include <linux/uaccess.h>
51#include <linux/highmem.h>
52#include <asm/ioctls.h>
53#include <linux/key.h>
54
55#include <linux/sunrpc/types.h>
56#include <linux/sunrpc/clnt.h>
57#include <linux/sunrpc/xdr.h>
58#include <linux/sunrpc/msg_prot.h>
59#include <linux/sunrpc/svcsock.h>
60#include <linux/sunrpc/stats.h>
61#include <linux/sunrpc/xprt.h>
62
63#include <trace/events/sock.h>
64#include <trace/events/sunrpc.h>
65
66#include "socklib.h"
67#include "sunrpc.h"
68
69#define RPCDBG_FACILITY RPCDBG_SVCXPRT
70
71/* To-do: to avoid tying up an nfsd thread while waiting for a
72 * handshake request, the request could instead be deferred.
73 */
74enum {
75 SVC_HANDSHAKE_TO = 5U * HZ
76};
77
78static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *,
79 int flags);
80static int svc_udp_recvfrom(struct svc_rqst *);
81static int svc_udp_sendto(struct svc_rqst *);
82static void svc_sock_detach(struct svc_xprt *);
83static void svc_tcp_sock_detach(struct svc_xprt *);
84static void svc_sock_free(struct svc_xprt *);
85
86static struct svc_xprt *svc_create_socket(struct svc_serv *, int,
87 struct net *, struct sockaddr *,
88 int, int);
89#ifdef CONFIG_DEBUG_LOCK_ALLOC
90static struct lock_class_key svc_key[2];
91static struct lock_class_key svc_slock_key[2];
92
93static void svc_reclassify_socket(struct socket *sock)
94{
95 struct sock *sk = sock->sk;
96
97 if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
98 return;
99
100 switch (sk->sk_family) {
101 case AF_INET:
102 sock_lock_init_class_and_name(sk, "slock-AF_INET-NFSD",
103 &svc_slock_key[0],
104 "sk_xprt.xpt_lock-AF_INET-NFSD",
105 &svc_key[0]);
106 break;
107
108 case AF_INET6:
109 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NFSD",
110 &svc_slock_key[1],
111 "sk_xprt.xpt_lock-AF_INET6-NFSD",
112 &svc_key[1]);
113 break;
114
115 default:
116 BUG();
117 }
118}
119#else
120static void svc_reclassify_socket(struct socket *sock)
121{
122}
123#endif
124
125/**
126 * svc_tcp_release_ctxt - Release transport-related resources
127 * @xprt: the transport which owned the context
128 * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt
129 *
130 */
131static void svc_tcp_release_ctxt(struct svc_xprt *xprt, void *ctxt)
132{
133}
134
135/**
136 * svc_udp_release_ctxt - Release transport-related resources
137 * @xprt: the transport which owned the context
138 * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt
139 *
140 */
141static void svc_udp_release_ctxt(struct svc_xprt *xprt, void *ctxt)
142{
143 struct sk_buff *skb = ctxt;
144
145 if (skb)
146 consume_skb(skb);
147}
148
149union svc_pktinfo_u {
150 struct in_pktinfo pkti;
151 struct in6_pktinfo pkti6;
152};
153#define SVC_PKTINFO_SPACE \
154 CMSG_SPACE(sizeof(union svc_pktinfo_u))
155
156static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh)
157{
158 struct svc_sock *svsk =
159 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
160 switch (svsk->sk_sk->sk_family) {
161 case AF_INET: {
162 struct in_pktinfo *pki = CMSG_DATA(cmh);
163
164 cmh->cmsg_level = SOL_IP;
165 cmh->cmsg_type = IP_PKTINFO;
166 pki->ipi_ifindex = 0;
167 pki->ipi_spec_dst.s_addr =
168 svc_daddr_in(rqst: rqstp)->sin_addr.s_addr;
169 cmh->cmsg_len = CMSG_LEN(sizeof(*pki));
170 }
171 break;
172
173 case AF_INET6: {
174 struct in6_pktinfo *pki = CMSG_DATA(cmh);
175 struct sockaddr_in6 *daddr = svc_daddr_in6(rqst: rqstp);
176
177 cmh->cmsg_level = SOL_IPV6;
178 cmh->cmsg_type = IPV6_PKTINFO;
179 pki->ipi6_ifindex = daddr->sin6_scope_id;
180 pki->ipi6_addr = daddr->sin6_addr;
181 cmh->cmsg_len = CMSG_LEN(sizeof(*pki));
182 }
183 break;
184 }
185}
186
187static int svc_sock_result_payload(struct svc_rqst *rqstp, unsigned int offset,
188 unsigned int length)
189{
190 return 0;
191}
192
193/*
194 * Report socket names for nfsdfs
195 */
196static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)
197{
198 const struct sock *sk = svsk->sk_sk;
199 const char *proto_name = sk->sk_protocol == IPPROTO_UDP ?
200 "udp" : "tcp";
201 int len;
202
203 switch (sk->sk_family) {
204 case PF_INET:
205 len = snprintf(buf, size: remaining, fmt: "ipv4 %s %pI4 %d\n",
206 proto_name,
207 &inet_sk(sk)->inet_rcv_saddr,
208 inet_sk(sk)->inet_num);
209 break;
210#if IS_ENABLED(CONFIG_IPV6)
211 case PF_INET6:
212 len = snprintf(buf, size: remaining, fmt: "ipv6 %s %pI6 %d\n",
213 proto_name,
214 &sk->sk_v6_rcv_saddr,
215 inet_sk(sk)->inet_num);
216 break;
217#endif
218 default:
219 len = snprintf(buf, size: remaining, fmt: "*unknown-%d*\n",
220 sk->sk_family);
221 }
222
223 if (len >= remaining) {
224 *buf = '\0';
225 return -ENAMETOOLONG;
226 }
227 return len;
228}
229
230static int
231svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg,
232 struct cmsghdr *cmsg, int ret)
233{
234 u8 content_type = tls_get_record_type(sk: sock->sk, msg: cmsg);
235 u8 level, description;
236
237 switch (content_type) {
238 case 0:
239 break;
240 case TLS_RECORD_TYPE_DATA:
241 /* TLS sets EOR at the end of each application data
242 * record, even though there might be more frames
243 * waiting to be decrypted.
244 */
245 msg->msg_flags &= ~MSG_EOR;
246 break;
247 case TLS_RECORD_TYPE_ALERT:
248 tls_alert_recv(sk: sock->sk, msg, level: &level, description: &description);
249 ret = (level == TLS_ALERT_LEVEL_FATAL) ?
250 -ENOTCONN : -EAGAIN;
251 break;
252 default:
253 /* discard this record type */
254 ret = -EAGAIN;
255 }
256 return ret;
257}
258
259static int
260svc_tcp_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags)
261{
262 union {
263 struct cmsghdr cmsg;
264 u8 buf[CMSG_SPACE(sizeof(u8))];
265 } u;
266 u8 alert[2];
267 struct kvec alert_kvec = {
268 .iov_base = alert,
269 .iov_len = sizeof(alert),
270 };
271 struct msghdr msg = {
272 .msg_flags = *msg_flags,
273 .msg_control = &u,
274 .msg_controllen = sizeof(u),
275 };
276 int ret;
277
278 iov_iter_kvec(i: &msg.msg_iter, ITER_DEST, kvec: &alert_kvec, nr_segs: 1,
279 count: alert_kvec.iov_len);
280 ret = sock_recvmsg(sock, msg: &msg, MSG_DONTWAIT);
281 if (ret > 0 &&
282 tls_get_record_type(sk: sock->sk, msg: &u.cmsg) == TLS_RECORD_TYPE_ALERT) {
283 iov_iter_revert(i: &msg.msg_iter, bytes: ret);
284 ret = svc_tcp_sock_process_cmsg(sock, msg: &msg, cmsg: &u.cmsg, ret: -EAGAIN);
285 }
286 return ret;
287}
288
289static int
290svc_tcp_sock_recvmsg(struct svc_sock *svsk, struct msghdr *msg)
291{
292 int ret;
293 struct socket *sock = svsk->sk_sock;
294
295 ret = sock_recvmsg(sock, msg, MSG_DONTWAIT);
296 if (msg->msg_flags & MSG_CTRUNC) {
297 msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR);
298 if (ret == 0 || ret == -EIO)
299 ret = svc_tcp_sock_recv_cmsg(sock, msg_flags: &msg->msg_flags);
300 }
301 return ret;
302}
303
304#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
305static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek)
306{
307 struct bvec_iter bi = {
308 .bi_size = size + seek,
309 };
310 struct bio_vec bv;
311
312 bvec_iter_advance(bvec, &bi, seek & PAGE_MASK);
313 for_each_bvec(bv, bvec, bi, bi)
314 flush_dcache_page(bv.bv_page);
315}
316#else
317static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size,
318 size_t seek)
319{
320}
321#endif
322
323/*
324 * Read from @rqstp's transport socket. The incoming message fills whole
325 * pages in @rqstp's rq_pages array until the last page of the message
326 * has been received into a partial page.
327 */
328static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen,
329 size_t seek)
330{
331 struct svc_sock *svsk =
332 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
333 struct bio_vec *bvec = rqstp->rq_bvec;
334 struct msghdr msg = { NULL };
335 unsigned int i;
336 ssize_t len;
337 size_t t;
338
339 clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
340
341 for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE)
342 bvec_set_page(bv: &bvec[i], page: rqstp->rq_pages[i], PAGE_SIZE, offset: 0);
343 rqstp->rq_respages = &rqstp->rq_pages[i];
344 rqstp->rq_next_page = rqstp->rq_respages + 1;
345
346 iov_iter_bvec(i: &msg.msg_iter, ITER_DEST, bvec, nr_segs: i, count: buflen);
347 if (seek) {
348 iov_iter_advance(i: &msg.msg_iter, bytes: seek);
349 buflen -= seek;
350 }
351 len = svc_tcp_sock_recvmsg(svsk, msg: &msg);
352 if (len > 0)
353 svc_flush_bvec(bvec, size: len, seek);
354
355 /* If we read a full record, then assume there may be more
356 * data to read (stream based sockets only!)
357 */
358 if (len == buflen)
359 set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
360
361 return len;
362}
363
364/*
365 * Set socket snd and rcv buffer lengths
366 */
367static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs)
368{
369 unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg;
370 struct socket *sock = svsk->sk_sock;
371
372 nreqs = min(nreqs, INT_MAX / 2 / max_mesg);
373
374 lock_sock(sk: sock->sk);
375 sock->sk->sk_sndbuf = nreqs * max_mesg * 2;
376 sock->sk->sk_rcvbuf = nreqs * max_mesg * 2;
377 sock->sk->sk_write_space(sock->sk);
378 release_sock(sk: sock->sk);
379}
380
381static void svc_sock_secure_port(struct svc_rqst *rqstp)
382{
383 if (svc_port_is_privileged(sin: svc_addr(rqst: rqstp)))
384 set_bit(nr: RQ_SECURE, addr: &rqstp->rq_flags);
385 else
386 clear_bit(nr: RQ_SECURE, addr: &rqstp->rq_flags);
387}
388
389/*
390 * INET callback when data has been received on the socket.
391 */
392static void svc_data_ready(struct sock *sk)
393{
394 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data;
395
396 trace_sk_data_ready(sk);
397
398 if (svsk) {
399 /* Refer to svc_setup_socket() for details. */
400 rmb();
401 svsk->sk_odata(sk);
402 trace_svcsock_data_ready(xprt: &svsk->sk_xprt, result: 0);
403 if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags))
404 return;
405 if (!test_and_set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags))
406 svc_xprt_enqueue(xprt: &svsk->sk_xprt);
407 }
408}
409
410/*
411 * INET callback when space is newly available on the socket.
412 */
413static void svc_write_space(struct sock *sk)
414{
415 struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data);
416
417 if (svsk) {
418 /* Refer to svc_setup_socket() for details. */
419 rmb();
420 trace_svcsock_write_space(xprt: &svsk->sk_xprt, result: 0);
421 svsk->sk_owspace(sk);
422 svc_xprt_enqueue(xprt: &svsk->sk_xprt);
423 }
424}
425
426static int svc_tcp_has_wspace(struct svc_xprt *xprt)
427{
428 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
429
430 if (test_bit(XPT_LISTENER, &xprt->xpt_flags))
431 return 1;
432 return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);
433}
434
435static void svc_tcp_kill_temp_xprt(struct svc_xprt *xprt)
436{
437 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
438
439 sock_no_linger(sk: svsk->sk_sock->sk);
440}
441
442/**
443 * svc_tcp_handshake_done - Handshake completion handler
444 * @data: address of xprt to wake
445 * @status: status of handshake
446 * @peerid: serial number of key containing the remote peer's identity
447 *
448 * If a security policy is specified as an export option, we don't
449 * have a specific export here to check. So we set a "TLS session
450 * is present" flag on the xprt and let an upper layer enforce local
451 * security policy.
452 */
453static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid)
454{
455 struct svc_xprt *xprt = data;
456 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
457
458 if (!status) {
459 if (peerid != TLS_NO_PEERID)
460 set_bit(nr: XPT_PEER_AUTH, addr: &xprt->xpt_flags);
461 set_bit(nr: XPT_TLS_SESSION, addr: &xprt->xpt_flags);
462 }
463 clear_bit(nr: XPT_HANDSHAKE, addr: &xprt->xpt_flags);
464 complete_all(&svsk->sk_handshake_done);
465}
466
467/**
468 * svc_tcp_handshake - Perform a transport-layer security handshake
469 * @xprt: connected transport endpoint
470 *
471 */
472static void svc_tcp_handshake(struct svc_xprt *xprt)
473{
474 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
475 struct sock *sk = svsk->sk_sock->sk;
476 struct tls_handshake_args args = {
477 .ta_sock = svsk->sk_sock,
478 .ta_done = svc_tcp_handshake_done,
479 .ta_data = xprt,
480 };
481 int ret;
482
483 trace_svc_tls_upcall(xprt);
484
485 clear_bit(nr: XPT_TLS_SESSION, addr: &xprt->xpt_flags);
486 init_completion(x: &svsk->sk_handshake_done);
487
488 ret = tls_server_hello_x509(args: &args, GFP_KERNEL);
489 if (ret) {
490 trace_svc_tls_not_started(xprt);
491 goto out_failed;
492 }
493
494 ret = wait_for_completion_interruptible_timeout(x: &svsk->sk_handshake_done,
495 timeout: SVC_HANDSHAKE_TO);
496 if (ret <= 0) {
497 if (tls_handshake_cancel(sk)) {
498 trace_svc_tls_timed_out(xprt);
499 goto out_close;
500 }
501 }
502
503 if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) {
504 trace_svc_tls_unavailable(xprt);
505 goto out_close;
506 }
507
508 /* Mark the transport ready in case the remote sent RPC
509 * traffic before the kernel received the handshake
510 * completion downcall.
511 */
512 set_bit(nr: XPT_DATA, addr: &xprt->xpt_flags);
513 svc_xprt_enqueue(xprt);
514 return;
515
516out_close:
517 set_bit(nr: XPT_CLOSE, addr: &xprt->xpt_flags);
518out_failed:
519 clear_bit(nr: XPT_HANDSHAKE, addr: &xprt->xpt_flags);
520 set_bit(nr: XPT_DATA, addr: &xprt->xpt_flags);
521 svc_xprt_enqueue(xprt);
522}
523
524/*
525 * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo
526 */
527static int svc_udp_get_dest_address4(struct svc_rqst *rqstp,
528 struct cmsghdr *cmh)
529{
530 struct in_pktinfo *pki = CMSG_DATA(cmh);
531 struct sockaddr_in *daddr = svc_daddr_in(rqst: rqstp);
532
533 if (cmh->cmsg_type != IP_PKTINFO)
534 return 0;
535
536 daddr->sin_family = AF_INET;
537 daddr->sin_addr.s_addr = pki->ipi_spec_dst.s_addr;
538 return 1;
539}
540
541/*
542 * See net/ipv6/datagram.c : ip6_datagram_recv_ctl
543 */
544static int svc_udp_get_dest_address6(struct svc_rqst *rqstp,
545 struct cmsghdr *cmh)
546{
547 struct in6_pktinfo *pki = CMSG_DATA(cmh);
548 struct sockaddr_in6 *daddr = svc_daddr_in6(rqst: rqstp);
549
550 if (cmh->cmsg_type != IPV6_PKTINFO)
551 return 0;
552
553 daddr->sin6_family = AF_INET6;
554 daddr->sin6_addr = pki->ipi6_addr;
555 daddr->sin6_scope_id = pki->ipi6_ifindex;
556 return 1;
557}
558
559/*
560 * Copy the UDP datagram's destination address to the rqstp structure.
561 * The 'destination' address in this case is the address to which the
562 * peer sent the datagram, i.e. our local address. For multihomed
563 * hosts, this can change from msg to msg. Note that only the IP
564 * address changes, the port number should remain the same.
565 */
566static int svc_udp_get_dest_address(struct svc_rqst *rqstp,
567 struct cmsghdr *cmh)
568{
569 switch (cmh->cmsg_level) {
570 case SOL_IP:
571 return svc_udp_get_dest_address4(rqstp, cmh);
572 case SOL_IPV6:
573 return svc_udp_get_dest_address6(rqstp, cmh);
574 }
575
576 return 0;
577}
578
579/**
580 * svc_udp_recvfrom - Receive a datagram from a UDP socket.
581 * @rqstp: request structure into which to receive an RPC Call
582 *
583 * Called in a loop when XPT_DATA has been set.
584 *
585 * Returns:
586 * On success, the number of bytes in a received RPC Call, or
587 * %0 if a complete RPC Call message was not ready to return
588 */
589static int svc_udp_recvfrom(struct svc_rqst *rqstp)
590{
591 struct svc_sock *svsk =
592 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
593 struct svc_serv *serv = svsk->sk_xprt.xpt_server;
594 struct sk_buff *skb;
595 union {
596 struct cmsghdr hdr;
597 long all[SVC_PKTINFO_SPACE / sizeof(long)];
598 } buffer;
599 struct cmsghdr *cmh = &buffer.hdr;
600 struct msghdr msg = {
601 .msg_name = svc_addr(rqst: rqstp),
602 .msg_control = cmh,
603 .msg_controllen = sizeof(buffer),
604 .msg_flags = MSG_DONTWAIT,
605 };
606 size_t len;
607 int err;
608
609 if (test_and_clear_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags))
610 /* udp sockets need large rcvbuf as all pending
611 * requests are still in that buffer. sndbuf must
612 * also be large enough that there is enough space
613 * for one reply per thread. We count all threads
614 * rather than threads in a particular pool, which
615 * provides an upper bound on the number of threads
616 * which will access the socket.
617 */
618 svc_sock_setbufsize(svsk, nreqs: serv->sv_nrthreads + 3);
619
620 clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
621 err = kernel_recvmsg(sock: svsk->sk_sock, msg: &msg, NULL,
622 num: 0, len: 0, MSG_PEEK | MSG_DONTWAIT);
623 if (err < 0)
624 goto out_recv_err;
625 skb = skb_recv_udp(sk: svsk->sk_sk, MSG_DONTWAIT, err: &err);
626 if (!skb)
627 goto out_recv_err;
628
629 len = svc_addr_len(sa: svc_addr(rqst: rqstp));
630 rqstp->rq_addrlen = len;
631 if (skb->tstamp == 0) {
632 skb->tstamp = ktime_get_real();
633 /* Don't enable netstamp, sunrpc doesn't
634 need that much accuracy */
635 }
636 sock_write_timestamp(sk: svsk->sk_sk, kt: skb->tstamp);
637 set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); /* there may be more data... */
638
639 len = skb->len;
640 rqstp->rq_arg.len = len;
641 trace_svcsock_udp_recv(xprt: &svsk->sk_xprt, result: len);
642
643 rqstp->rq_prot = IPPROTO_UDP;
644
645 if (!svc_udp_get_dest_address(rqstp, cmh))
646 goto out_cmsg_err;
647 rqstp->rq_daddrlen = svc_addr_len(sa: svc_daddr(rqst: rqstp));
648
649 if (skb_is_nonlinear(skb)) {
650 /* we have to copy */
651 local_bh_disable();
652 if (csum_partial_copy_to_xdr(xdr: &rqstp->rq_arg, skb))
653 goto out_bh_enable;
654 local_bh_enable();
655 consume_skb(skb);
656 } else {
657 /* we can use it in-place */
658 rqstp->rq_arg.head[0].iov_base = skb->data;
659 rqstp->rq_arg.head[0].iov_len = len;
660 if (skb_checksum_complete(skb))
661 goto out_free;
662 rqstp->rq_xprt_ctxt = skb;
663 }
664
665 rqstp->rq_arg.page_base = 0;
666 if (len <= rqstp->rq_arg.head[0].iov_len) {
667 rqstp->rq_arg.head[0].iov_len = len;
668 rqstp->rq_arg.page_len = 0;
669 rqstp->rq_respages = rqstp->rq_pages+1;
670 } else {
671 rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len;
672 rqstp->rq_respages = rqstp->rq_pages + 1 +
673 DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE);
674 }
675 rqstp->rq_next_page = rqstp->rq_respages+1;
676
677 if (serv->sv_stats)
678 serv->sv_stats->netudpcnt++;
679
680 svc_sock_secure_port(rqstp);
681 svc_xprt_received(xprt: rqstp->rq_xprt);
682 return len;
683
684out_recv_err:
685 if (err != -EAGAIN) {
686 /* possibly an icmp error */
687 set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
688 }
689 trace_svcsock_udp_recv_err(xprt: &svsk->sk_xprt, result: err);
690 goto out_clear_busy;
691out_cmsg_err:
692 net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n",
693 cmh->cmsg_level, cmh->cmsg_type);
694 goto out_free;
695out_bh_enable:
696 local_bh_enable();
697out_free:
698 kfree_skb(skb);
699out_clear_busy:
700 svc_xprt_received(xprt: rqstp->rq_xprt);
701 return 0;
702}
703
704/**
705 * svc_udp_sendto - Send out a reply on a UDP socket
706 * @rqstp: completed svc_rqst
707 *
708 * xpt_mutex ensures @rqstp's whole message is written to the socket
709 * without interruption.
710 *
711 * Returns the number of bytes sent, or a negative errno.
712 */
713static int svc_udp_sendto(struct svc_rqst *rqstp)
714{
715 struct svc_xprt *xprt = rqstp->rq_xprt;
716 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
717 struct xdr_buf *xdr = &rqstp->rq_res;
718 union {
719 struct cmsghdr hdr;
720 long all[SVC_PKTINFO_SPACE / sizeof(long)];
721 } buffer;
722 struct cmsghdr *cmh = &buffer.hdr;
723 struct msghdr msg = {
724 .msg_name = &rqstp->rq_addr,
725 .msg_namelen = rqstp->rq_addrlen,
726 .msg_control = cmh,
727 .msg_flags = MSG_SPLICE_PAGES,
728 .msg_controllen = sizeof(buffer),
729 };
730 unsigned int count;
731 int err;
732
733 svc_udp_release_ctxt(xprt, ctxt: rqstp->rq_xprt_ctxt);
734 rqstp->rq_xprt_ctxt = NULL;
735
736 svc_set_cmsg_data(rqstp, cmh);
737
738 mutex_lock(lock: &xprt->xpt_mutex);
739
740 if (svc_xprt_is_dead(xprt))
741 goto out_notconn;
742
743 count = xdr_buf_to_bvec(bvec: rqstp->rq_bvec, bvec_size: rqstp->rq_maxpages, xdr);
744
745 iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec,
746 nr_segs: count, count: rqstp->rq_res.len);
747 err = sock_sendmsg(sock: svsk->sk_sock, msg: &msg);
748 if (err == -ECONNREFUSED) {
749 /* ICMP error on earlier request. */
750 iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec,
751 nr_segs: count, count: rqstp->rq_res.len);
752 err = sock_sendmsg(sock: svsk->sk_sock, msg: &msg);
753 }
754
755 trace_svcsock_udp_send(xprt, result: err);
756
757 mutex_unlock(lock: &xprt->xpt_mutex);
758 return err;
759
760out_notconn:
761 mutex_unlock(lock: &xprt->xpt_mutex);
762 return -ENOTCONN;
763}
764
765static int svc_udp_has_wspace(struct svc_xprt *xprt)
766{
767 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
768 struct svc_serv *serv = xprt->xpt_server;
769 unsigned long required;
770
771 /*
772 * Set the SOCK_NOSPACE flag before checking the available
773 * sock space.
774 */
775 set_bit(nr: SOCK_NOSPACE, addr: &svsk->sk_sock->flags);
776 required = atomic_read(v: &svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg;
777 if (required*2 > sock_wspace(sk: svsk->sk_sk))
778 return 0;
779 clear_bit(nr: SOCK_NOSPACE, addr: &svsk->sk_sock->flags);
780 return 1;
781}
782
783static struct svc_xprt *svc_udp_accept(struct svc_xprt *xprt)
784{
785 BUG();
786 return NULL;
787}
788
789static void svc_udp_kill_temp_xprt(struct svc_xprt *xprt)
790{
791}
792
793static struct svc_xprt *svc_udp_create(struct svc_serv *serv,
794 struct net *net,
795 struct sockaddr *sa, int salen,
796 int flags)
797{
798 return svc_create_socket(serv, IPPROTO_UDP, net, sa, salen, flags);
799}
800
801static const struct svc_xprt_ops svc_udp_ops = {
802 .xpo_create = svc_udp_create,
803 .xpo_recvfrom = svc_udp_recvfrom,
804 .xpo_sendto = svc_udp_sendto,
805 .xpo_result_payload = svc_sock_result_payload,
806 .xpo_release_ctxt = svc_udp_release_ctxt,
807 .xpo_detach = svc_sock_detach,
808 .xpo_free = svc_sock_free,
809 .xpo_has_wspace = svc_udp_has_wspace,
810 .xpo_accept = svc_udp_accept,
811 .xpo_kill_temp_xprt = svc_udp_kill_temp_xprt,
812};
813
814static struct svc_xprt_class svc_udp_class = {
815 .xcl_name = "udp",
816 .xcl_owner = THIS_MODULE,
817 .xcl_ops = &svc_udp_ops,
818 .xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP,
819 .xcl_ident = XPRT_TRANSPORT_UDP,
820};
821
822static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)
823{
824 svc_xprt_init(sock_net(sk: svsk->sk_sock->sk), &svc_udp_class,
825 &svsk->sk_xprt, serv);
826 clear_bit(nr: XPT_CACHE_AUTH, addr: &svsk->sk_xprt.xpt_flags);
827 svsk->sk_sk->sk_data_ready = svc_data_ready;
828 svsk->sk_sk->sk_write_space = svc_write_space;
829
830 /* initialise setting must have enough space to
831 * receive and respond to one request.
832 * svc_udp_recvfrom will re-adjust if necessary
833 */
834 svc_sock_setbufsize(svsk, nreqs: 3);
835
836 /* data might have come in before data_ready set up */
837 set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
838 set_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags);
839 set_bit(nr: XPT_RPCB_UNREG, addr: &svsk->sk_xprt.xpt_flags);
840
841 /* make sure we get destination address info */
842 switch (svsk->sk_sk->sk_family) {
843 case AF_INET:
844 ip_sock_set_pktinfo(sk: svsk->sk_sock->sk);
845 break;
846 case AF_INET6:
847 ip6_sock_set_recvpktinfo(sk: svsk->sk_sock->sk);
848 break;
849 default:
850 BUG();
851 }
852}
853
854/*
855 * A data_ready event on a listening socket means there's a connection
856 * pending. Do not use state_change as a substitute for it.
857 */
858static void svc_tcp_listen_data_ready(struct sock *sk)
859{
860 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data;
861
862 trace_sk_data_ready(sk);
863
864 /*
865 * This callback may called twice when a new connection
866 * is established as a child socket inherits everything
867 * from a parent LISTEN socket.
868 * 1) data_ready method of the parent socket will be called
869 * when one of child sockets become ESTABLISHED.
870 * 2) data_ready method of the child socket may be called
871 * when it receives data before the socket is accepted.
872 * In case of 2, we should ignore it silently and DO NOT
873 * dereference svsk.
874 */
875 if (sk->sk_state != TCP_LISTEN)
876 return;
877
878 if (svsk) {
879 /* Refer to svc_setup_socket() for details. */
880 rmb();
881 svsk->sk_odata(sk);
882 set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags);
883 svc_xprt_enqueue(xprt: &svsk->sk_xprt);
884 }
885}
886
887/*
888 * A state change on a connected socket means it's dying or dead.
889 */
890static void svc_tcp_state_change(struct sock *sk)
891{
892 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data;
893
894 if (svsk) {
895 /* Refer to svc_setup_socket() for details. */
896 rmb();
897 svsk->sk_ostate(sk);
898 trace_svcsock_tcp_state(xprt: &svsk->sk_xprt, socket: svsk->sk_sock);
899 if (sk->sk_state != TCP_ESTABLISHED)
900 svc_xprt_deferred_close(xprt: &svsk->sk_xprt);
901 }
902}
903
904/*
905 * Accept a TCP connection
906 */
907static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt)
908{
909 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
910 struct sockaddr_storage addr;
911 struct sockaddr *sin = (struct sockaddr *) &addr;
912 struct svc_serv *serv = svsk->sk_xprt.xpt_server;
913 struct socket *sock = svsk->sk_sock;
914 struct socket *newsock;
915 struct svc_sock *newsvsk;
916 int err, slen;
917
918 if (!sock)
919 return NULL;
920
921 clear_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags);
922 err = kernel_accept(sock, newsock: &newsock, O_NONBLOCK);
923 if (err < 0) {
924 if (err != -EAGAIN)
925 trace_svcsock_accept_err(xprt, service: serv->sv_name, status: err);
926 return NULL;
927 }
928 if (IS_ERR(ptr: sock_alloc_file(sock: newsock, O_NONBLOCK, NULL)))
929 return NULL;
930
931 set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags);
932
933 err = kernel_getpeername(sock: newsock, addr: sin);
934 if (err < 0) {
935 trace_svcsock_getpeername_err(xprt, service: serv->sv_name, status: err);
936 goto failed; /* aborted connection or whatever */
937 }
938 slen = err;
939
940 /* Reset the inherited callbacks before calling svc_setup_socket */
941 newsock->sk->sk_state_change = svsk->sk_ostate;
942 newsock->sk->sk_data_ready = svsk->sk_odata;
943 newsock->sk->sk_write_space = svsk->sk_owspace;
944
945 /* make sure that a write doesn't block forever when
946 * low on memory
947 */
948 newsock->sk->sk_sndtimeo = HZ*30;
949
950 newsvsk = svc_setup_socket(serv, newsock,
951 flags: (SVC_SOCK_ANONYMOUS | SVC_SOCK_TEMPORARY));
952 if (IS_ERR(ptr: newsvsk))
953 goto failed;
954 svc_xprt_set_remote(xprt: &newsvsk->sk_xprt, sa: sin, salen: slen);
955 err = kernel_getsockname(sock: newsock, addr: sin);
956 slen = err;
957 if (unlikely(err < 0))
958 slen = offsetof(struct sockaddr, sa_data);
959 svc_xprt_set_local(xprt: &newsvsk->sk_xprt, sa: sin, salen: slen);
960
961 if (sock_is_loopback(sk: newsock->sk))
962 set_bit(nr: XPT_LOCAL, addr: &newsvsk->sk_xprt.xpt_flags);
963 else
964 clear_bit(nr: XPT_LOCAL, addr: &newsvsk->sk_xprt.xpt_flags);
965 if (serv->sv_stats)
966 serv->sv_stats->nettcpconn++;
967
968 return &newsvsk->sk_xprt;
969
970failed:
971 sockfd_put(newsock);
972 return NULL;
973}
974
975static size_t svc_tcp_restore_pages(struct svc_sock *svsk,
976 struct svc_rqst *rqstp)
977{
978 size_t len = svsk->sk_datalen;
979 unsigned int i, npages;
980
981 if (!len)
982 return 0;
983 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
984 for (i = 0; i < npages; i++) {
985 if (rqstp->rq_pages[i] != NULL)
986 put_page(page: rqstp->rq_pages[i]);
987 BUG_ON(svsk->sk_pages[i] == NULL);
988 rqstp->rq_pages[i] = svsk->sk_pages[i];
989 svsk->sk_pages[i] = NULL;
990 }
991 rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]);
992 return len;
993}
994
995static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)
996{
997 unsigned int i, len, npages;
998
999 if (svsk->sk_datalen == 0)
1000 return;
1001 len = svsk->sk_datalen;
1002 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1003 for (i = 0; i < npages; i++) {
1004 svsk->sk_pages[i] = rqstp->rq_pages[i];
1005 rqstp->rq_pages[i] = NULL;
1006 }
1007}
1008
1009static void svc_tcp_clear_pages(struct svc_sock *svsk)
1010{
1011 unsigned int i, len, npages;
1012
1013 if (svsk->sk_datalen == 0)
1014 goto out;
1015 len = svsk->sk_datalen;
1016 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1017 for (i = 0; i < npages; i++) {
1018 if (svsk->sk_pages[i] == NULL) {
1019 WARN_ON_ONCE(1);
1020 continue;
1021 }
1022 put_page(page: svsk->sk_pages[i]);
1023 svsk->sk_pages[i] = NULL;
1024 }
1025out:
1026 svsk->sk_tcplen = 0;
1027 svsk->sk_datalen = 0;
1028}
1029
1030/*
1031 * Receive fragment record header into sk_marker.
1032 */
1033static ssize_t svc_tcp_read_marker(struct svc_sock *svsk,
1034 struct svc_rqst *rqstp)
1035{
1036 ssize_t want, len;
1037
1038 /* If we haven't gotten the record length yet,
1039 * get the next four bytes.
1040 */
1041 if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {
1042 struct msghdr msg = { NULL };
1043 struct kvec iov;
1044
1045 want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;
1046 iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen;
1047 iov.iov_len = want;
1048 iov_iter_kvec(i: &msg.msg_iter, ITER_DEST, kvec: &iov, nr_segs: 1, count: want);
1049 len = svc_tcp_sock_recvmsg(svsk, msg: &msg);
1050 if (len < 0)
1051 return len;
1052 svsk->sk_tcplen += len;
1053 if (len < want) {
1054 /* call again to read the remaining bytes */
1055 goto err_short;
1056 }
1057 trace_svcsock_marker(xprt: &svsk->sk_xprt, marker: svsk->sk_marker);
1058 if (svc_sock_reclen(svsk) + svsk->sk_datalen >
1059 svsk->sk_xprt.xpt_server->sv_max_mesg)
1060 goto err_too_large;
1061 }
1062 return svc_sock_reclen(svsk);
1063
1064err_too_large:
1065 net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n",
1066 __func__, svsk->sk_xprt.xpt_server->sv_name,
1067 svc_sock_reclen(svsk));
1068 svc_xprt_deferred_close(xprt: &svsk->sk_xprt);
1069err_short:
1070 return -EAGAIN;
1071}
1072
1073static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)
1074{
1075 struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt;
1076 struct rpc_rqst *req = NULL;
1077 struct kvec *src, *dst;
1078 __be32 *p = (__be32 *)rqstp->rq_arg.head[0].iov_base;
1079 __be32 xid = *p;
1080
1081 if (!bc_xprt)
1082 return -EAGAIN;
1083 spin_lock(lock: &bc_xprt->queue_lock);
1084 req = xprt_lookup_rqst(xprt: bc_xprt, xid);
1085 if (!req)
1086 goto unlock_eagain;
1087
1088 memcpy(to: &req->rq_private_buf, from: &req->rq_rcv_buf, len: sizeof(struct xdr_buf));
1089 /*
1090 * XXX!: cheating for now! Only copying HEAD.
1091 * But we know this is good enough for now (in fact, for any
1092 * callback reply in the forseeable future).
1093 */
1094 dst = &req->rq_private_buf.head[0];
1095 src = &rqstp->rq_arg.head[0];
1096 if (dst->iov_len < src->iov_len)
1097 goto unlock_eagain; /* whatever; just giving up. */
1098 memcpy(to: dst->iov_base, from: src->iov_base, len: src->iov_len);
1099 xprt_complete_rqst(task: req->rq_task, copied: rqstp->rq_arg.len);
1100 rqstp->rq_arg.len = 0;
1101 spin_unlock(lock: &bc_xprt->queue_lock);
1102 return 0;
1103unlock_eagain:
1104 spin_unlock(lock: &bc_xprt->queue_lock);
1105 return -EAGAIN;
1106}
1107
1108static void svc_tcp_fragment_received(struct svc_sock *svsk)
1109{
1110 /* If we have more data, signal svc_xprt_enqueue() to try again */
1111 svsk->sk_tcplen = 0;
1112 svsk->sk_marker = xdr_zero;
1113}
1114
1115/**
1116 * svc_tcp_recvfrom - Receive data from a TCP socket
1117 * @rqstp: request structure into which to receive an RPC Call
1118 *
1119 * Called in a loop when XPT_DATA has been set.
1120 *
1121 * Read the 4-byte stream record marker, then use the record length
1122 * in that marker to set up exactly the resources needed to receive
1123 * the next RPC message into @rqstp.
1124 *
1125 * Returns:
1126 * On success, the number of bytes in a received RPC Call, or
1127 * %0 if a complete RPC Call message was not ready to return
1128 *
1129 * The zero return case handles partial receives and callback Replies.
1130 * The state of a partial receive is preserved in the svc_sock for
1131 * the next call to svc_tcp_recvfrom.
1132 */
1133static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
1134{
1135 struct svc_sock *svsk =
1136 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
1137 struct svc_serv *serv = svsk->sk_xprt.xpt_server;
1138 size_t want, base;
1139 ssize_t len;
1140 __be32 *p;
1141 __be32 calldir;
1142
1143 clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
1144 len = svc_tcp_read_marker(svsk, rqstp);
1145 if (len < 0)
1146 goto error;
1147
1148 base = svc_tcp_restore_pages(svsk, rqstp);
1149 want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr));
1150 len = svc_tcp_read_msg(rqstp, buflen: base + want, seek: base);
1151 if (len >= 0) {
1152 trace_svcsock_tcp_recv(xprt: &svsk->sk_xprt, result: len);
1153 svsk->sk_tcplen += len;
1154 svsk->sk_datalen += len;
1155 }
1156 if (len != want || !svc_sock_final_rec(svsk))
1157 goto err_incomplete;
1158 if (svsk->sk_datalen < 8)
1159 goto err_nuts;
1160
1161 rqstp->rq_arg.len = svsk->sk_datalen;
1162 rqstp->rq_arg.page_base = 0;
1163 if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
1164 rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
1165 rqstp->rq_arg.page_len = 0;
1166 } else
1167 rqstp->rq_arg.page_len = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;
1168
1169 rqstp->rq_xprt_ctxt = NULL;
1170 rqstp->rq_prot = IPPROTO_TCP;
1171 if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))
1172 set_bit(nr: RQ_LOCAL, addr: &rqstp->rq_flags);
1173 else
1174 clear_bit(nr: RQ_LOCAL, addr: &rqstp->rq_flags);
1175
1176 p = (__be32 *)rqstp->rq_arg.head[0].iov_base;
1177 calldir = p[1];
1178 if (calldir)
1179 len = receive_cb_reply(svsk, rqstp);
1180
1181 /* Reset TCP read info */
1182 svsk->sk_datalen = 0;
1183 svc_tcp_fragment_received(svsk);
1184
1185 if (len < 0)
1186 goto error;
1187
1188 svc_xprt_copy_addrs(rqstp, xprt: &svsk->sk_xprt);
1189 if (serv->sv_stats)
1190 serv->sv_stats->nettcpcnt++;
1191
1192 svc_sock_secure_port(rqstp);
1193 svc_xprt_received(xprt: rqstp->rq_xprt);
1194 return rqstp->rq_arg.len;
1195
1196err_incomplete:
1197 svc_tcp_save_pages(svsk, rqstp);
1198 if (len < 0 && len != -EAGAIN)
1199 goto err_delete;
1200 if (len == want)
1201 svc_tcp_fragment_received(svsk);
1202 else
1203 trace_svcsock_tcp_recv_short(xprt: &svsk->sk_xprt,
1204 expected: svc_sock_reclen(svsk),
1205 received: svsk->sk_tcplen - sizeof(rpc_fraghdr));
1206 goto err_noclose;
1207error:
1208 if (len != -EAGAIN)
1209 goto err_delete;
1210 trace_svcsock_tcp_recv_eagain(xprt: &svsk->sk_xprt, result: 0);
1211 goto err_noclose;
1212err_nuts:
1213 svsk->sk_datalen = 0;
1214err_delete:
1215 trace_svcsock_tcp_recv_err(xprt: &svsk->sk_xprt, result: len);
1216 svc_xprt_deferred_close(xprt: &svsk->sk_xprt);
1217err_noclose:
1218 svc_xprt_received(xprt: rqstp->rq_xprt);
1219 return 0; /* record not complete */
1220}
1221
1222/*
1223 * MSG_SPLICE_PAGES is used exclusively to reduce the number of
1224 * copy operations in this path. Therefore the caller must ensure
1225 * that the pages backing @xdr are unchanging.
1226 */
1227static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,
1228 rpc_fraghdr marker)
1229{
1230 struct msghdr msg = {
1231 .msg_flags = MSG_SPLICE_PAGES,
1232 };
1233 unsigned int count;
1234 void *buf;
1235 int ret;
1236
1237 /* The stream record marker is copied into a temporary page
1238 * fragment buffer so that it can be included in rq_bvec.
1239 */
1240 buf = page_frag_alloc(nc: &svsk->sk_frag_cache, fragsz: sizeof(marker),
1241 GFP_KERNEL);
1242 if (!buf)
1243 return -ENOMEM;
1244 memcpy(to: buf, from: &marker, len: sizeof(marker));
1245 bvec_set_virt(bv: rqstp->rq_bvec, vaddr: buf, len: sizeof(marker));
1246
1247 count = xdr_buf_to_bvec(bvec: rqstp->rq_bvec + 1, bvec_size: rqstp->rq_maxpages,
1248 xdr: &rqstp->rq_res);
1249
1250 iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec,
1251 nr_segs: 1 + count, count: sizeof(marker) + rqstp->rq_res.len);
1252 ret = sock_sendmsg(sock: svsk->sk_sock, msg: &msg);
1253 page_frag_free(addr: buf);
1254 return ret;
1255}
1256
1257/**
1258 * svc_tcp_sendto - Send out a reply on a TCP socket
1259 * @rqstp: completed svc_rqst
1260 *
1261 * xpt_mutex ensures @rqstp's whole message is written to the socket
1262 * without interruption.
1263 *
1264 * Returns the number of bytes sent, or a negative errno.
1265 */
1266static int svc_tcp_sendto(struct svc_rqst *rqstp)
1267{
1268 struct svc_xprt *xprt = rqstp->rq_xprt;
1269 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
1270 struct xdr_buf *xdr = &rqstp->rq_res;
1271 rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT |
1272 (u32)xdr->len);
1273 int sent;
1274
1275 svc_tcp_release_ctxt(xprt, ctxt: rqstp->rq_xprt_ctxt);
1276 rqstp->rq_xprt_ctxt = NULL;
1277
1278 mutex_lock(lock: &xprt->xpt_mutex);
1279 if (svc_xprt_is_dead(xprt))
1280 goto out_notconn;
1281 sent = svc_tcp_sendmsg(svsk, rqstp, marker);
1282 trace_svcsock_tcp_send(xprt, result: sent);
1283 if (sent < 0 || sent != (xdr->len + sizeof(marker)))
1284 goto out_close;
1285 mutex_unlock(lock: &xprt->xpt_mutex);
1286 return sent;
1287
1288out_notconn:
1289 mutex_unlock(lock: &xprt->xpt_mutex);
1290 return -ENOTCONN;
1291out_close:
1292 pr_notice("rpc-srv/tcp: %s: %s %d when sending %zu bytes - shutting down socket\n",
1293 xprt->xpt_server->sv_name,
1294 (sent < 0) ? "got error" : "sent",
1295 sent, xdr->len + sizeof(marker));
1296 svc_xprt_deferred_close(xprt);
1297 mutex_unlock(lock: &xprt->xpt_mutex);
1298 return -EAGAIN;
1299}
1300
1301static struct svc_xprt *svc_tcp_create(struct svc_serv *serv,
1302 struct net *net,
1303 struct sockaddr *sa, int salen,
1304 int flags)
1305{
1306 return svc_create_socket(serv, IPPROTO_TCP, net, sa, salen, flags);
1307}
1308
1309static const struct svc_xprt_ops svc_tcp_ops = {
1310 .xpo_create = svc_tcp_create,
1311 .xpo_recvfrom = svc_tcp_recvfrom,
1312 .xpo_sendto = svc_tcp_sendto,
1313 .xpo_result_payload = svc_sock_result_payload,
1314 .xpo_release_ctxt = svc_tcp_release_ctxt,
1315 .xpo_detach = svc_tcp_sock_detach,
1316 .xpo_free = svc_sock_free,
1317 .xpo_has_wspace = svc_tcp_has_wspace,
1318 .xpo_accept = svc_tcp_accept,
1319 .xpo_kill_temp_xprt = svc_tcp_kill_temp_xprt,
1320 .xpo_handshake = svc_tcp_handshake,
1321};
1322
1323static struct svc_xprt_class svc_tcp_class = {
1324 .xcl_name = "tcp",
1325 .xcl_owner = THIS_MODULE,
1326 .xcl_ops = &svc_tcp_ops,
1327 .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP,
1328 .xcl_ident = XPRT_TRANSPORT_TCP,
1329};
1330
1331void svc_init_xprt_sock(void)
1332{
1333 svc_reg_xprt_class(&svc_tcp_class);
1334 svc_reg_xprt_class(&svc_udp_class);
1335}
1336
1337void svc_cleanup_xprt_sock(void)
1338{
1339 svc_unreg_xprt_class(&svc_tcp_class);
1340 svc_unreg_xprt_class(&svc_udp_class);
1341}
1342
1343static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv)
1344{
1345 struct sock *sk = svsk->sk_sk;
1346
1347 svc_xprt_init(sock_net(sk: svsk->sk_sock->sk), &svc_tcp_class,
1348 &svsk->sk_xprt, serv);
1349 set_bit(nr: XPT_CACHE_AUTH, addr: &svsk->sk_xprt.xpt_flags);
1350 set_bit(nr: XPT_CONG_CTRL, addr: &svsk->sk_xprt.xpt_flags);
1351 if (sk->sk_state == TCP_LISTEN) {
1352 strcpy(svsk->sk_xprt.xpt_remotebuf, "listener");
1353 set_bit(nr: XPT_LISTENER, addr: &svsk->sk_xprt.xpt_flags);
1354 set_bit(nr: XPT_RPCB_UNREG, addr: &svsk->sk_xprt.xpt_flags);
1355 sk->sk_data_ready = svc_tcp_listen_data_ready;
1356 set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags);
1357 } else {
1358 sk->sk_state_change = svc_tcp_state_change;
1359 sk->sk_data_ready = svc_data_ready;
1360 sk->sk_write_space = svc_write_space;
1361
1362 svsk->sk_marker = xdr_zero;
1363 svsk->sk_tcplen = 0;
1364 svsk->sk_datalen = 0;
1365 memset(s: &svsk->sk_pages[0], c: 0,
1366 n: svsk->sk_maxpages * sizeof(struct page *));
1367
1368 tcp_sock_set_nodelay(sk);
1369
1370 set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags);
1371 switch (sk->sk_state) {
1372 case TCP_SYN_RECV:
1373 case TCP_ESTABLISHED:
1374 break;
1375 default:
1376 svc_xprt_deferred_close(xprt: &svsk->sk_xprt);
1377 }
1378 }
1379}
1380
1381void svc_sock_update_bufs(struct svc_serv *serv)
1382{
1383 /*
1384 * The number of server threads has changed. Update
1385 * rcvbuf and sndbuf accordingly on all sockets
1386 */
1387 struct svc_sock *svsk;
1388
1389 spin_lock_bh(lock: &serv->sv_lock);
1390 list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list)
1391 set_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags);
1392 spin_unlock_bh(lock: &serv->sv_lock);
1393}
1394
1395/*
1396 * Initialize socket for RPC use and create svc_sock struct
1397 */
1398static struct svc_sock *svc_setup_socket(struct svc_serv *serv,
1399 struct socket *sock,
1400 int flags)
1401{
1402 struct svc_sock *svsk;
1403 struct sock *inet;
1404 int pmap_register = !(flags & SVC_SOCK_ANONYMOUS);
1405 unsigned long pages;
1406
1407 pages = svc_serv_maxpages(serv);
1408 svsk = kzalloc(struct_size(svsk, sk_pages, pages), GFP_KERNEL);
1409 if (!svsk)
1410 return ERR_PTR(error: -ENOMEM);
1411 svsk->sk_maxpages = pages;
1412
1413 inet = sock->sk;
1414
1415 if (pmap_register) {
1416 int err;
1417
1418 err = svc_register(serv, sock_net(sk: sock->sk), inet->sk_family,
1419 inet->sk_protocol,
1420 ntohs(inet_sk(inet)->inet_sport));
1421 if (err < 0) {
1422 kfree(objp: svsk);
1423 return ERR_PTR(error: err);
1424 }
1425 }
1426
1427 svsk->sk_sock = sock;
1428 svsk->sk_sk = inet;
1429 svsk->sk_ostate = inet->sk_state_change;
1430 svsk->sk_odata = inet->sk_data_ready;
1431 svsk->sk_owspace = inet->sk_write_space;
1432 /*
1433 * This barrier is necessary in order to prevent race condition
1434 * with svc_data_ready(), svc_tcp_listen_data_ready(), and others
1435 * when calling callbacks above.
1436 */
1437 wmb();
1438 inet->sk_user_data = svsk;
1439
1440 /* Initialize the socket */
1441 if (sock->type == SOCK_DGRAM)
1442 svc_udp_init(svsk, serv);
1443 else
1444 svc_tcp_init(svsk, serv);
1445
1446 trace_svcsock_new(svsk, socket: sock);
1447 return svsk;
1448}
1449
1450/**
1451 * svc_addsock - add a listener socket to an RPC service
1452 * @serv: pointer to RPC service to which to add a new listener
1453 * @net: caller's network namespace
1454 * @fd: file descriptor of the new listener
1455 * @name_return: pointer to buffer to fill in with name of listener
1456 * @len: size of the buffer
1457 * @cred: credential
1458 *
1459 * Fills in socket name and returns positive length of name if successful.
1460 * Name is terminated with '\n'. On error, returns a negative errno
1461 * value.
1462 */
1463int svc_addsock(struct svc_serv *serv, struct net *net, const int fd,
1464 char *name_return, const size_t len, const struct cred *cred)
1465{
1466 int err = 0;
1467 struct socket *so = sockfd_lookup(fd, err: &err);
1468 struct svc_sock *svsk = NULL;
1469 struct sockaddr_storage addr;
1470 struct sockaddr *sin = (struct sockaddr *)&addr;
1471 int salen;
1472
1473 if (!so)
1474 return err;
1475 err = -EINVAL;
1476 if (sock_net(sk: so->sk) != net)
1477 goto out;
1478 err = -EAFNOSUPPORT;
1479 if ((so->sk->sk_family != PF_INET) && (so->sk->sk_family != PF_INET6))
1480 goto out;
1481 err = -EPROTONOSUPPORT;
1482 if (so->sk->sk_protocol != IPPROTO_TCP &&
1483 so->sk->sk_protocol != IPPROTO_UDP)
1484 goto out;
1485 err = -EISCONN;
1486 if (so->state > SS_UNCONNECTED)
1487 goto out;
1488 err = -ENOENT;
1489 if (!try_module_get(THIS_MODULE))
1490 goto out;
1491 svsk = svc_setup_socket(serv, sock: so, SVC_SOCK_DEFAULTS);
1492 if (IS_ERR(ptr: svsk)) {
1493 module_put(THIS_MODULE);
1494 err = PTR_ERR(ptr: svsk);
1495 goto out;
1496 }
1497 salen = kernel_getsockname(sock: svsk->sk_sock, addr: sin);
1498 if (salen >= 0)
1499 svc_xprt_set_local(xprt: &svsk->sk_xprt, sa: sin, salen);
1500 svsk->sk_xprt.xpt_cred = get_cred(cred);
1501 svc_add_new_perm_xprt(serv, xprt: &svsk->sk_xprt);
1502 return svc_one_sock_name(svsk, buf: name_return, remaining: len);
1503out:
1504 sockfd_put(so);
1505 return err;
1506}
1507EXPORT_SYMBOL_GPL(svc_addsock);
1508
1509/*
1510 * Create socket for RPC service.
1511 */
1512static struct svc_xprt *svc_create_socket(struct svc_serv *serv,
1513 int protocol,
1514 struct net *net,
1515 struct sockaddr *sin, int len,
1516 int flags)
1517{
1518 struct svc_sock *svsk;
1519 struct socket *sock;
1520 int error;
1521 int type;
1522 struct sockaddr_storage addr;
1523 struct sockaddr *newsin = (struct sockaddr *)&addr;
1524 int newlen;
1525 int family;
1526
1527 if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) {
1528 printk(KERN_WARNING "svc: only UDP and TCP "
1529 "sockets supported\n");
1530 return ERR_PTR(error: -EINVAL);
1531 }
1532
1533 type = (protocol == IPPROTO_UDP)? SOCK_DGRAM : SOCK_STREAM;
1534 switch (sin->sa_family) {
1535 case AF_INET6:
1536 family = PF_INET6;
1537 break;
1538 case AF_INET:
1539 family = PF_INET;
1540 break;
1541 default:
1542 return ERR_PTR(error: -EINVAL);
1543 }
1544
1545 error = __sock_create(net, family, type, proto: protocol, res: &sock, kern: 1);
1546 if (error < 0)
1547 return ERR_PTR(error);
1548
1549 svc_reclassify_socket(sock);
1550
1551 /*
1552 * If this is an PF_INET6 listener, we want to avoid
1553 * getting requests from IPv4 remotes. Those should
1554 * be shunted to a PF_INET listener via rpcbind.
1555 */
1556 if (family == PF_INET6)
1557 ip6_sock_set_v6only(sk: sock->sk);
1558 if (type == SOCK_STREAM)
1559 sock->sk->sk_reuse = SK_CAN_REUSE; /* allow address reuse */
1560 error = kernel_bind(sock, addr: sin, addrlen: len);
1561 if (error < 0)
1562 goto bummer;
1563
1564 error = kernel_getsockname(sock, addr: newsin);
1565 if (error < 0)
1566 goto bummer;
1567 newlen = error;
1568
1569 if (protocol == IPPROTO_TCP) {
1570 sk_net_refcnt_upgrade(sk: sock->sk);
1571 if ((error = kernel_listen(sock, SOMAXCONN)) < 0)
1572 goto bummer;
1573 }
1574
1575 svsk = svc_setup_socket(serv, sock, flags);
1576 if (IS_ERR(ptr: svsk)) {
1577 error = PTR_ERR(ptr: svsk);
1578 goto bummer;
1579 }
1580 svc_xprt_set_local(xprt: &svsk->sk_xprt, sa: newsin, salen: newlen);
1581 return (struct svc_xprt *)svsk;
1582bummer:
1583 sock_release(sock);
1584 return ERR_PTR(error);
1585}
1586
1587/*
1588 * Detach the svc_sock from the socket so that no
1589 * more callbacks occur.
1590 */
1591static void svc_sock_detach(struct svc_xprt *xprt)
1592{
1593 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
1594 struct sock *sk = svsk->sk_sk;
1595
1596 /* put back the old socket callbacks */
1597 lock_sock(sk);
1598 sk->sk_state_change = svsk->sk_ostate;
1599 sk->sk_data_ready = svsk->sk_odata;
1600 sk->sk_write_space = svsk->sk_owspace;
1601 sk->sk_user_data = NULL;
1602 release_sock(sk);
1603}
1604
1605/*
1606 * Disconnect the socket, and reset the callbacks
1607 */
1608static void svc_tcp_sock_detach(struct svc_xprt *xprt)
1609{
1610 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
1611
1612 tls_handshake_close(sock: svsk->sk_sock);
1613
1614 svc_sock_detach(xprt);
1615
1616 if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) {
1617 svc_tcp_clear_pages(svsk);
1618 kernel_sock_shutdown(sock: svsk->sk_sock, how: SHUT_RDWR);
1619 }
1620}
1621
1622/*
1623 * Free the svc_sock's socket resources and the svc_sock itself.
1624 */
1625static void svc_sock_free(struct svc_xprt *xprt)
1626{
1627 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
1628 struct socket *sock = svsk->sk_sock;
1629
1630 trace_svcsock_free(svsk, socket: sock);
1631
1632 tls_handshake_cancel(sk: sock->sk);
1633 if (sock->file)
1634 sockfd_put(sock);
1635 else
1636 sock_release(sock);
1637
1638 page_frag_cache_drain(nc: &svsk->sk_frag_cache);
1639 kfree(objp: svsk);
1640}
1641