1
2
3
4
5
6
7
8#include <linux/sched.h>
9#include <linux/errno.h>
10#include <linux/freezer.h>
11#include <linux/kthread.h>
12#include <linux/slab.h>
13#include <net/sock.h>
14#include <linux/sunrpc/addr.h>
15#include <linux/sunrpc/stats.h>
16#include <linux/sunrpc/svc_xprt.h>
17#include <linux/sunrpc/svcsock.h>
18#include <linux/sunrpc/xprt.h>
19#include <linux/module.h>
20#include <linux/netdevice.h>
21#include <trace/events/sunrpc.h>
22
23#define RPCDBG_FACILITY RPCDBG_SVCXPRT
24
25static unsigned int svc_rpc_per_connection_limit __read_mostly;
26module_param(svc_rpc_per_connection_limit, uint, 0644);
27
28
29static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt);
30static int svc_deferred_recv(struct svc_rqst *rqstp);
31static struct cache_deferred_req *svc_defer(struct cache_req *req);
32static void svc_age_temp_xprts(struct timer_list *t);
33static void svc_delete_xprt(struct svc_xprt *xprt);
34
35
36
37
38
39
40static int svc_conn_age_period = 6*60;
41
42
43static DEFINE_SPINLOCK(svc_xprt_class_lock);
44static LIST_HEAD(svc_xprt_class_list);
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76int svc_reg_xprt_class(struct svc_xprt_class *xcl)
77{
78 struct svc_xprt_class *cl;
79 int res = -EEXIST;
80
81 dprintk("svc: Adding svc transport class '%s'\n", xcl->xcl_name);
82
83 INIT_LIST_HEAD(&xcl->xcl_list);
84 spin_lock(&svc_xprt_class_lock);
85
86 list_for_each_entry(cl, &svc_xprt_class_list, xcl_list) {
87 if (strcmp(xcl->xcl_name, cl->xcl_name) == 0)
88 goto out;
89 }
90 list_add_tail(&xcl->xcl_list, &svc_xprt_class_list);
91 res = 0;
92out:
93 spin_unlock(&svc_xprt_class_lock);
94 return res;
95}
96EXPORT_SYMBOL_GPL(svc_reg_xprt_class);
97
98void svc_unreg_xprt_class(struct svc_xprt_class *xcl)
99{
100 dprintk("svc: Removing svc transport class '%s'\n", xcl->xcl_name);
101 spin_lock(&svc_xprt_class_lock);
102 list_del_init(&xcl->xcl_list);
103 spin_unlock(&svc_xprt_class_lock);
104}
105EXPORT_SYMBOL_GPL(svc_unreg_xprt_class);
106
107
108
109
110
111
112
113
114
115
116
117
118
119int svc_print_xprts(char *buf, int maxlen)
120{
121 struct svc_xprt_class *xcl;
122 char tmpstr[80];
123 int len = 0;
124 buf[0] = '\0';
125
126 spin_lock(&svc_xprt_class_lock);
127 list_for_each_entry(xcl, &svc_xprt_class_list, xcl_list) {
128 int slen;
129
130 slen = snprintf(tmpstr, sizeof(tmpstr), "%s %d\n",
131 xcl->xcl_name, xcl->xcl_max_payload);
132 if (slen >= sizeof(tmpstr) || len + slen >= maxlen)
133 break;
134 len += slen;
135 strcat(buf, tmpstr);
136 }
137 spin_unlock(&svc_xprt_class_lock);
138
139 return len;
140}
141
142
143
144
145
146
147
148
149void svc_xprt_deferred_close(struct svc_xprt *xprt)
150{
151 if (!test_and_set_bit(XPT_CLOSE, &xprt->xpt_flags))
152 svc_xprt_enqueue(xprt);
153}
154EXPORT_SYMBOL_GPL(svc_xprt_deferred_close);
155
156static void svc_xprt_free(struct kref *kref)
157{
158 struct svc_xprt *xprt =
159 container_of(kref, struct svc_xprt, xpt_ref);
160 struct module *owner = xprt->xpt_class->xcl_owner;
161 if (test_bit(XPT_CACHE_AUTH, &xprt->xpt_flags))
162 svcauth_unix_info_release(xprt);
163 put_cred(xprt->xpt_cred);
164 put_net(xprt->xpt_net);
165
166 if (xprt->xpt_bc_xprt)
167 xprt_put(xprt->xpt_bc_xprt);
168 if (xprt->xpt_bc_xps)
169 xprt_switch_put(xprt->xpt_bc_xps);
170 trace_svc_xprt_free(xprt);
171 xprt->xpt_ops->xpo_free(xprt);
172 module_put(owner);
173}
174
175void svc_xprt_put(struct svc_xprt *xprt)
176{
177 kref_put(&xprt->xpt_ref, svc_xprt_free);
178}
179EXPORT_SYMBOL_GPL(svc_xprt_put);
180
181
182
183
184
185void svc_xprt_init(struct net *net, struct svc_xprt_class *xcl,
186 struct svc_xprt *xprt, struct svc_serv *serv)
187{
188 memset(xprt, 0, sizeof(*xprt));
189 xprt->xpt_class = xcl;
190 xprt->xpt_ops = xcl->xcl_ops;
191 kref_init(&xprt->xpt_ref);
192 xprt->xpt_server = serv;
193 INIT_LIST_HEAD(&xprt->xpt_list);
194 INIT_LIST_HEAD(&xprt->xpt_ready);
195 INIT_LIST_HEAD(&xprt->xpt_deferred);
196 INIT_LIST_HEAD(&xprt->xpt_users);
197 mutex_init(&xprt->xpt_mutex);
198 spin_lock_init(&xprt->xpt_lock);
199 set_bit(XPT_BUSY, &xprt->xpt_flags);
200 xprt->xpt_net = get_net(net);
201 strcpy(xprt->xpt_remotebuf, "uninitialized");
202}
203EXPORT_SYMBOL_GPL(svc_xprt_init);
204
205static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl,
206 struct svc_serv *serv,
207 struct net *net,
208 const int family,
209 const unsigned short port,
210 int flags)
211{
212 struct sockaddr_in sin = {
213 .sin_family = AF_INET,
214 .sin_addr.s_addr = htonl(INADDR_ANY),
215 .sin_port = htons(port),
216 };
217#if IS_ENABLED(CONFIG_IPV6)
218 struct sockaddr_in6 sin6 = {
219 .sin6_family = AF_INET6,
220 .sin6_addr = IN6ADDR_ANY_INIT,
221 .sin6_port = htons(port),
222 };
223#endif
224 struct svc_xprt *xprt;
225 struct sockaddr *sap;
226 size_t len;
227
228 switch (family) {
229 case PF_INET:
230 sap = (struct sockaddr *)&sin;
231 len = sizeof(sin);
232 break;
233#if IS_ENABLED(CONFIG_IPV6)
234 case PF_INET6:
235 sap = (struct sockaddr *)&sin6;
236 len = sizeof(sin6);
237 break;
238#endif
239 default:
240 return ERR_PTR(-EAFNOSUPPORT);
241 }
242
243 xprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags);
244 if (IS_ERR(xprt))
245 trace_svc_xprt_create_err(serv->sv_program->pg_name,
246 xcl->xcl_name, sap, xprt);
247 return xprt;
248}
249
250
251
252
253
254
255
256
257
258
259
260void svc_xprt_received(struct svc_xprt *xprt)
261{
262 if (!test_bit(XPT_BUSY, &xprt->xpt_flags)) {
263 WARN_ONCE(1, "xprt=0x%p already busy!", xprt);
264 return;
265 }
266
267 trace_svc_xprt_received(xprt);
268
269
270
271
272 svc_xprt_get(xprt);
273 smp_mb__before_atomic();
274 clear_bit(XPT_BUSY, &xprt->xpt_flags);
275 xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
276 svc_xprt_put(xprt);
277}
278EXPORT_SYMBOL_GPL(svc_xprt_received);
279
280void svc_add_new_perm_xprt(struct svc_serv *serv, struct svc_xprt *new)
281{
282 clear_bit(XPT_TEMP, &new->xpt_flags);
283 spin_lock_bh(&serv->sv_lock);
284 list_add(&new->xpt_list, &serv->sv_permsocks);
285 spin_unlock_bh(&serv->sv_lock);
286 svc_xprt_received(new);
287}
288
289static int _svc_create_xprt(struct svc_serv *serv, const char *xprt_name,
290 struct net *net, const int family,
291 const unsigned short port, int flags,
292 const struct cred *cred)
293{
294 struct svc_xprt_class *xcl;
295
296 spin_lock(&svc_xprt_class_lock);
297 list_for_each_entry(xcl, &svc_xprt_class_list, xcl_list) {
298 struct svc_xprt *newxprt;
299 unsigned short newport;
300
301 if (strcmp(xprt_name, xcl->xcl_name))
302 continue;
303
304 if (!try_module_get(xcl->xcl_owner))
305 goto err;
306
307 spin_unlock(&svc_xprt_class_lock);
308 newxprt = __svc_xpo_create(xcl, serv, net, family, port, flags);
309 if (IS_ERR(newxprt)) {
310 module_put(xcl->xcl_owner);
311 return PTR_ERR(newxprt);
312 }
313 newxprt->xpt_cred = get_cred(cred);
314 svc_add_new_perm_xprt(serv, newxprt);
315 newport = svc_xprt_local_port(newxprt);
316 return newport;
317 }
318 err:
319 spin_unlock(&svc_xprt_class_lock);
320
321
322 return -EPROTONOSUPPORT;
323}
324
325int svc_create_xprt(struct svc_serv *serv, const char *xprt_name,
326 struct net *net, const int family,
327 const unsigned short port, int flags,
328 const struct cred *cred)
329{
330 int err;
331
332 err = _svc_create_xprt(serv, xprt_name, net, family, port, flags, cred);
333 if (err == -EPROTONOSUPPORT) {
334 request_module("svc%s", xprt_name);
335 err = _svc_create_xprt(serv, xprt_name, net, family, port, flags, cred);
336 }
337 return err;
338}
339EXPORT_SYMBOL_GPL(svc_create_xprt);
340
341
342
343
344void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt)
345{
346 memcpy(&rqstp->rq_addr, &xprt->xpt_remote, xprt->xpt_remotelen);
347 rqstp->rq_addrlen = xprt->xpt_remotelen;
348
349
350
351
352
353 memcpy(&rqstp->rq_daddr, &xprt->xpt_local, xprt->xpt_locallen);
354 rqstp->rq_daddrlen = xprt->xpt_locallen;
355}
356EXPORT_SYMBOL_GPL(svc_xprt_copy_addrs);
357
358
359
360
361
362
363
364
365char *svc_print_addr(struct svc_rqst *rqstp, char *buf, size_t len)
366{
367 return __svc_print_addr(svc_addr(rqstp), buf, len);
368}
369EXPORT_SYMBOL_GPL(svc_print_addr);
370
371static bool svc_xprt_slots_in_range(struct svc_xprt *xprt)
372{
373 unsigned int limit = svc_rpc_per_connection_limit;
374 int nrqsts = atomic_read(&xprt->xpt_nr_rqsts);
375
376 return limit == 0 || (nrqsts >= 0 && nrqsts < limit);
377}
378
379static bool svc_xprt_reserve_slot(struct svc_rqst *rqstp, struct svc_xprt *xprt)
380{
381 if (!test_bit(RQ_DATA, &rqstp->rq_flags)) {
382 if (!svc_xprt_slots_in_range(xprt))
383 return false;
384 atomic_inc(&xprt->xpt_nr_rqsts);
385 set_bit(RQ_DATA, &rqstp->rq_flags);
386 }
387 return true;
388}
389
390static void svc_xprt_release_slot(struct svc_rqst *rqstp)
391{
392 struct svc_xprt *xprt = rqstp->rq_xprt;
393 if (test_and_clear_bit(RQ_DATA, &rqstp->rq_flags)) {
394 atomic_dec(&xprt->xpt_nr_rqsts);
395 smp_wmb();
396 svc_xprt_enqueue(xprt);
397 }
398}
399
400static bool svc_xprt_ready(struct svc_xprt *xprt)
401{
402 unsigned long xpt_flags;
403
404
405
406
407
408
409
410
411
412 smp_rmb();
413 xpt_flags = READ_ONCE(xprt->xpt_flags);
414
415 if (xpt_flags & (BIT(XPT_CONN) | BIT(XPT_CLOSE)))
416 return true;
417 if (xpt_flags & (BIT(XPT_DATA) | BIT(XPT_DEFERRED))) {
418 if (xprt->xpt_ops->xpo_has_wspace(xprt) &&
419 svc_xprt_slots_in_range(xprt))
420 return true;
421 trace_svc_xprt_no_write_space(xprt);
422 return false;
423 }
424 return false;
425}
426
427void svc_xprt_do_enqueue(struct svc_xprt *xprt)
428{
429 struct svc_pool *pool;
430 struct svc_rqst *rqstp = NULL;
431 int cpu;
432
433 if (!svc_xprt_ready(xprt))
434 return;
435
436
437
438
439
440
441 if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags))
442 return;
443
444 cpu = get_cpu();
445 pool = svc_pool_for_cpu(xprt->xpt_server, cpu);
446
447 atomic_long_inc(&pool->sp_stats.packets);
448
449 spin_lock_bh(&pool->sp_lock);
450 list_add_tail(&xprt->xpt_ready, &pool->sp_sockets);
451 pool->sp_stats.sockets_queued++;
452 spin_unlock_bh(&pool->sp_lock);
453
454
455 rcu_read_lock();
456 list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {
457 if (test_and_set_bit(RQ_BUSY, &rqstp->rq_flags))
458 continue;
459 atomic_long_inc(&pool->sp_stats.threads_woken);
460 rqstp->rq_qtime = ktime_get();
461 wake_up_process(rqstp->rq_task);
462 goto out_unlock;
463 }
464 set_bit(SP_CONGESTED, &pool->sp_flags);
465 rqstp = NULL;
466out_unlock:
467 rcu_read_unlock();
468 put_cpu();
469 trace_svc_xprt_do_enqueue(xprt, rqstp);
470}
471EXPORT_SYMBOL_GPL(svc_xprt_do_enqueue);
472
473
474
475
476
477
478void svc_xprt_enqueue(struct svc_xprt *xprt)
479{
480 if (test_bit(XPT_BUSY, &xprt->xpt_flags))
481 return;
482 xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
483}
484EXPORT_SYMBOL_GPL(svc_xprt_enqueue);
485
486
487
488
489static struct svc_xprt *svc_xprt_dequeue(struct svc_pool *pool)
490{
491 struct svc_xprt *xprt = NULL;
492
493 if (list_empty(&pool->sp_sockets))
494 goto out;
495
496 spin_lock_bh(&pool->sp_lock);
497 if (likely(!list_empty(&pool->sp_sockets))) {
498 xprt = list_first_entry(&pool->sp_sockets,
499 struct svc_xprt, xpt_ready);
500 list_del_init(&xprt->xpt_ready);
501 svc_xprt_get(xprt);
502 }
503 spin_unlock_bh(&pool->sp_lock);
504out:
505 return xprt;
506}
507
508
509
510
511
512
513
514
515
516
517
518void svc_reserve(struct svc_rqst *rqstp, int space)
519{
520 struct svc_xprt *xprt = rqstp->rq_xprt;
521
522 space += rqstp->rq_res.head[0].iov_len;
523
524 if (xprt && space < rqstp->rq_reserved) {
525 atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);
526 rqstp->rq_reserved = space;
527 smp_wmb();
528 svc_xprt_enqueue(xprt);
529 }
530}
531EXPORT_SYMBOL_GPL(svc_reserve);
532
533static void svc_xprt_release(struct svc_rqst *rqstp)
534{
535 struct svc_xprt *xprt = rqstp->rq_xprt;
536
537 xprt->xpt_ops->xpo_release_rqst(rqstp);
538
539 kfree(rqstp->rq_deferred);
540 rqstp->rq_deferred = NULL;
541
542 svc_free_res_pages(rqstp);
543 rqstp->rq_res.page_len = 0;
544 rqstp->rq_res.page_base = 0;
545
546
547
548
549
550
551 if ((rqstp->rq_res.len) > rqstp->rq_reserved)
552 printk(KERN_ERR "RPC request reserved %d but used %d\n",
553 rqstp->rq_reserved,
554 rqstp->rq_res.len);
555
556 rqstp->rq_res.head[0].iov_len = 0;
557 svc_reserve(rqstp, 0);
558 svc_xprt_release_slot(rqstp);
559 rqstp->rq_xprt = NULL;
560 svc_xprt_put(xprt);
561}
562
563
564
565
566
567
568
569
570void svc_wake_up(struct svc_serv *serv)
571{
572 struct svc_rqst *rqstp;
573 struct svc_pool *pool;
574
575 pool = &serv->sv_pools[0];
576
577 rcu_read_lock();
578 list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {
579
580 if (test_bit(RQ_BUSY, &rqstp->rq_flags))
581 continue;
582 rcu_read_unlock();
583 wake_up_process(rqstp->rq_task);
584 trace_svc_wake_up(rqstp->rq_task->pid);
585 return;
586 }
587 rcu_read_unlock();
588
589
590 set_bit(SP_TASK_PENDING, &pool->sp_flags);
591 smp_wmb();
592 trace_svc_wake_up(0);
593}
594EXPORT_SYMBOL_GPL(svc_wake_up);
595
596int svc_port_is_privileged(struct sockaddr *sin)
597{
598 switch (sin->sa_family) {
599 case AF_INET:
600 return ntohs(((struct sockaddr_in *)sin)->sin_port)
601 < PROT_SOCK;
602 case AF_INET6:
603 return ntohs(((struct sockaddr_in6 *)sin)->sin6_port)
604 < PROT_SOCK;
605 default:
606 return 0;
607 }
608}
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628static void svc_check_conn_limits(struct svc_serv *serv)
629{
630 unsigned int limit = serv->sv_maxconn ? serv->sv_maxconn :
631 (serv->sv_nrthreads+3) * 20;
632
633 if (serv->sv_tmpcnt > limit) {
634 struct svc_xprt *xprt = NULL;
635 spin_lock_bh(&serv->sv_lock);
636 if (!list_empty(&serv->sv_tempsocks)) {
637
638 net_notice_ratelimited("%s: too many open connections, consider increasing the %s\n",
639 serv->sv_name, serv->sv_maxconn ?
640 "max number of connections" :
641 "number of threads");
642
643
644
645
646 xprt = list_entry(serv->sv_tempsocks.prev,
647 struct svc_xprt,
648 xpt_list);
649 set_bit(XPT_CLOSE, &xprt->xpt_flags);
650 svc_xprt_get(xprt);
651 }
652 spin_unlock_bh(&serv->sv_lock);
653
654 if (xprt) {
655 svc_xprt_enqueue(xprt);
656 svc_xprt_put(xprt);
657 }
658 }
659}
660
661static int svc_alloc_arg(struct svc_rqst *rqstp)
662{
663 struct svc_serv *serv = rqstp->rq_server;
664 struct xdr_buf *arg = &rqstp->rq_arg;
665 unsigned long pages, filled;
666
667 pages = (serv->sv_max_mesg + 2 * PAGE_SIZE) >> PAGE_SHIFT;
668 if (pages > RPCSVC_MAXPAGES) {
669 pr_warn_once("svc: warning: pages=%lu > RPCSVC_MAXPAGES=%lu\n",
670 pages, RPCSVC_MAXPAGES);
671
672 pages = RPCSVC_MAXPAGES;
673 }
674
675 for (;;) {
676 filled = alloc_pages_bulk_array(GFP_KERNEL, pages,
677 rqstp->rq_pages);
678 if (filled == pages)
679 break;
680
681 set_current_state(TASK_INTERRUPTIBLE);
682 if (signalled() || kthread_should_stop()) {
683 set_current_state(TASK_RUNNING);
684 return -EINTR;
685 }
686 schedule_timeout(msecs_to_jiffies(500));
687 }
688 rqstp->rq_page_end = &rqstp->rq_pages[pages];
689 rqstp->rq_pages[pages] = NULL;
690
691
692 arg->head[0].iov_base = page_address(rqstp->rq_pages[0]);
693 arg->head[0].iov_len = PAGE_SIZE;
694 arg->pages = rqstp->rq_pages + 1;
695 arg->page_base = 0;
696
697 arg->page_len = (pages-2)*PAGE_SIZE;
698 arg->len = (pages-1)*PAGE_SIZE;
699 arg->tail[0].iov_len = 0;
700 return 0;
701}
702
703static bool
704rqst_should_sleep(struct svc_rqst *rqstp)
705{
706 struct svc_pool *pool = rqstp->rq_pool;
707
708
709 if (test_and_clear_bit(SP_TASK_PENDING, &pool->sp_flags))
710 return false;
711
712
713 if (!list_empty(&pool->sp_sockets))
714 return false;
715
716
717 if (signalled() || kthread_should_stop())
718 return false;
719
720
721 if (freezing(current))
722 return false;
723
724 return true;
725}
726
727static struct svc_xprt *svc_get_next_xprt(struct svc_rqst *rqstp, long timeout)
728{
729 struct svc_pool *pool = rqstp->rq_pool;
730 long time_left = 0;
731
732
733 WARN_ON_ONCE(rqstp->rq_xprt);
734
735 rqstp->rq_xprt = svc_xprt_dequeue(pool);
736 if (rqstp->rq_xprt)
737 goto out_found;
738
739
740
741
742
743 set_current_state(TASK_INTERRUPTIBLE);
744 smp_mb__before_atomic();
745 clear_bit(SP_CONGESTED, &pool->sp_flags);
746 clear_bit(RQ_BUSY, &rqstp->rq_flags);
747 smp_mb__after_atomic();
748
749 if (likely(rqst_should_sleep(rqstp)))
750 time_left = schedule_timeout(timeout);
751 else
752 __set_current_state(TASK_RUNNING);
753
754 try_to_freeze();
755
756 set_bit(RQ_BUSY, &rqstp->rq_flags);
757 smp_mb__after_atomic();
758 rqstp->rq_xprt = svc_xprt_dequeue(pool);
759 if (rqstp->rq_xprt)
760 goto out_found;
761
762 if (!time_left)
763 atomic_long_inc(&pool->sp_stats.threads_timedout);
764
765 if (signalled() || kthread_should_stop())
766 return ERR_PTR(-EINTR);
767 return ERR_PTR(-EAGAIN);
768out_found:
769
770
771
772 if (!test_bit(SP_CONGESTED, &pool->sp_flags))
773 rqstp->rq_chandle.thread_wait = 5*HZ;
774 else
775 rqstp->rq_chandle.thread_wait = 1*HZ;
776 trace_svc_xprt_dequeue(rqstp);
777 return rqstp->rq_xprt;
778}
779
780static void svc_add_new_temp_xprt(struct svc_serv *serv, struct svc_xprt *newxpt)
781{
782 spin_lock_bh(&serv->sv_lock);
783 set_bit(XPT_TEMP, &newxpt->xpt_flags);
784 list_add(&newxpt->xpt_list, &serv->sv_tempsocks);
785 serv->sv_tmpcnt++;
786 if (serv->sv_temptimer.function == NULL) {
787
788 serv->sv_temptimer.function = svc_age_temp_xprts;
789 mod_timer(&serv->sv_temptimer,
790 jiffies + svc_conn_age_period * HZ);
791 }
792 spin_unlock_bh(&serv->sv_lock);
793 svc_xprt_received(newxpt);
794}
795
796static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt)
797{
798 struct svc_serv *serv = rqstp->rq_server;
799 int len = 0;
800
801 if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) {
802 if (test_and_clear_bit(XPT_KILL_TEMP, &xprt->xpt_flags))
803 xprt->xpt_ops->xpo_kill_temp_xprt(xprt);
804 svc_delete_xprt(xprt);
805
806 goto out;
807 }
808 if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) {
809 struct svc_xprt *newxpt;
810
811
812
813
814 __module_get(xprt->xpt_class->xcl_owner);
815 svc_check_conn_limits(xprt->xpt_server);
816 newxpt = xprt->xpt_ops->xpo_accept(xprt);
817 if (newxpt) {
818 newxpt->xpt_cred = get_cred(xprt->xpt_cred);
819 svc_add_new_temp_xprt(serv, newxpt);
820 trace_svc_xprt_accept(newxpt, serv->sv_name);
821 } else {
822 module_put(xprt->xpt_class->xcl_owner);
823 }
824 svc_xprt_received(xprt);
825 } else if (svc_xprt_reserve_slot(rqstp, xprt)) {
826
827 dprintk("svc: server %p, pool %u, transport %p, inuse=%d\n",
828 rqstp, rqstp->rq_pool->sp_id, xprt,
829 kref_read(&xprt->xpt_ref));
830 rqstp->rq_deferred = svc_deferred_dequeue(xprt);
831 if (rqstp->rq_deferred)
832 len = svc_deferred_recv(rqstp);
833 else
834 len = xprt->xpt_ops->xpo_recvfrom(rqstp);
835 rqstp->rq_stime = ktime_get();
836 rqstp->rq_reserved = serv->sv_max_mesg;
837 atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);
838 } else
839 svc_xprt_received(xprt);
840out:
841 trace_svc_handle_xprt(xprt, len);
842 return len;
843}
844
845
846
847
848
849
850int svc_recv(struct svc_rqst *rqstp, long timeout)
851{
852 struct svc_xprt *xprt = NULL;
853 struct svc_serv *serv = rqstp->rq_server;
854 int len, err;
855
856 err = svc_alloc_arg(rqstp);
857 if (err)
858 goto out;
859
860 try_to_freeze();
861 cond_resched();
862 err = -EINTR;
863 if (signalled() || kthread_should_stop())
864 goto out;
865
866 xprt = svc_get_next_xprt(rqstp, timeout);
867 if (IS_ERR(xprt)) {
868 err = PTR_ERR(xprt);
869 goto out;
870 }
871
872 len = svc_handle_xprt(rqstp, xprt);
873
874
875 err = -EAGAIN;
876 if (len <= 0)
877 goto out_release;
878 trace_svc_xdr_recvfrom(&rqstp->rq_arg);
879
880 clear_bit(XPT_OLD, &xprt->xpt_flags);
881
882 xprt->xpt_ops->xpo_secure_port(rqstp);
883 rqstp->rq_chandle.defer = svc_defer;
884 rqstp->rq_xid = svc_getu32(&rqstp->rq_arg.head[0]);
885
886 if (serv->sv_stats)
887 serv->sv_stats->netcnt++;
888 return len;
889out_release:
890 rqstp->rq_res.len = 0;
891 svc_xprt_release(rqstp);
892out:
893 return err;
894}
895EXPORT_SYMBOL_GPL(svc_recv);
896
897
898
899
900void svc_drop(struct svc_rqst *rqstp)
901{
902 trace_svc_drop(rqstp);
903 svc_xprt_release(rqstp);
904}
905EXPORT_SYMBOL_GPL(svc_drop);
906
907
908
909
910int svc_send(struct svc_rqst *rqstp)
911{
912 struct svc_xprt *xprt;
913 int len = -EFAULT;
914 struct xdr_buf *xb;
915
916 xprt = rqstp->rq_xprt;
917 if (!xprt)
918 goto out;
919
920
921 xb = &rqstp->rq_res;
922 xb->len = xb->head[0].iov_len +
923 xb->page_len +
924 xb->tail[0].iov_len;
925 trace_svc_xdr_sendto(rqstp->rq_xid, xb);
926 trace_svc_stats_latency(rqstp);
927
928 len = xprt->xpt_ops->xpo_sendto(rqstp);
929
930 trace_svc_send(rqstp, len);
931 svc_xprt_release(rqstp);
932
933 if (len == -ECONNREFUSED || len == -ENOTCONN || len == -EAGAIN)
934 len = 0;
935out:
936 return len;
937}
938
939
940
941
942
943static void svc_age_temp_xprts(struct timer_list *t)
944{
945 struct svc_serv *serv = from_timer(serv, t, sv_temptimer);
946 struct svc_xprt *xprt;
947 struct list_head *le, *next;
948
949 dprintk("svc_age_temp_xprts\n");
950
951 if (!spin_trylock_bh(&serv->sv_lock)) {
952
953 dprintk("svc_age_temp_xprts: busy\n");
954 mod_timer(&serv->sv_temptimer, jiffies + HZ);
955 return;
956 }
957
958 list_for_each_safe(le, next, &serv->sv_tempsocks) {
959 xprt = list_entry(le, struct svc_xprt, xpt_list);
960
961
962
963 if (!test_and_set_bit(XPT_OLD, &xprt->xpt_flags))
964 continue;
965 if (kref_read(&xprt->xpt_ref) > 1 ||
966 test_bit(XPT_BUSY, &xprt->xpt_flags))
967 continue;
968 list_del_init(le);
969 set_bit(XPT_CLOSE, &xprt->xpt_flags);
970 dprintk("queuing xprt %p for closing\n", xprt);
971
972
973 svc_xprt_enqueue(xprt);
974 }
975 spin_unlock_bh(&serv->sv_lock);
976
977 mod_timer(&serv->sv_temptimer, jiffies + svc_conn_age_period * HZ);
978}
979
980
981
982
983
984
985
986void svc_age_temp_xprts_now(struct svc_serv *serv, struct sockaddr *server_addr)
987{
988 struct svc_xprt *xprt;
989 struct list_head *le, *next;
990 LIST_HEAD(to_be_closed);
991
992 spin_lock_bh(&serv->sv_lock);
993 list_for_each_safe(le, next, &serv->sv_tempsocks) {
994 xprt = list_entry(le, struct svc_xprt, xpt_list);
995 if (rpc_cmp_addr(server_addr, (struct sockaddr *)
996 &xprt->xpt_local)) {
997 dprintk("svc_age_temp_xprts_now: found %p\n", xprt);
998 list_move(le, &to_be_closed);
999 }
1000 }
1001 spin_unlock_bh(&serv->sv_lock);
1002
1003 while (!list_empty(&to_be_closed)) {
1004 le = to_be_closed.next;
1005 list_del_init(le);
1006 xprt = list_entry(le, struct svc_xprt, xpt_list);
1007 set_bit(XPT_CLOSE, &xprt->xpt_flags);
1008 set_bit(XPT_KILL_TEMP, &xprt->xpt_flags);
1009 dprintk("svc_age_temp_xprts_now: queuing xprt %p for closing\n",
1010 xprt);
1011 svc_xprt_enqueue(xprt);
1012 }
1013}
1014EXPORT_SYMBOL_GPL(svc_age_temp_xprts_now);
1015
1016static void call_xpt_users(struct svc_xprt *xprt)
1017{
1018 struct svc_xpt_user *u;
1019
1020 spin_lock(&xprt->xpt_lock);
1021 while (!list_empty(&xprt->xpt_users)) {
1022 u = list_first_entry(&xprt->xpt_users, struct svc_xpt_user, list);
1023 list_del_init(&u->list);
1024 u->callback(u);
1025 }
1026 spin_unlock(&xprt->xpt_lock);
1027}
1028
1029
1030
1031
1032static void svc_delete_xprt(struct svc_xprt *xprt)
1033{
1034 struct svc_serv *serv = xprt->xpt_server;
1035 struct svc_deferred_req *dr;
1036
1037 if (test_and_set_bit(XPT_DEAD, &xprt->xpt_flags))
1038 return;
1039
1040 trace_svc_xprt_detach(xprt);
1041 xprt->xpt_ops->xpo_detach(xprt);
1042 if (xprt->xpt_bc_xprt)
1043 xprt->xpt_bc_xprt->ops->close(xprt->xpt_bc_xprt);
1044
1045 spin_lock_bh(&serv->sv_lock);
1046 list_del_init(&xprt->xpt_list);
1047 WARN_ON_ONCE(!list_empty(&xprt->xpt_ready));
1048 if (test_bit(XPT_TEMP, &xprt->xpt_flags))
1049 serv->sv_tmpcnt--;
1050 spin_unlock_bh(&serv->sv_lock);
1051
1052 while ((dr = svc_deferred_dequeue(xprt)) != NULL)
1053 kfree(dr);
1054
1055 call_xpt_users(xprt);
1056 svc_xprt_put(xprt);
1057}
1058
1059void svc_close_xprt(struct svc_xprt *xprt)
1060{
1061 trace_svc_xprt_close(xprt);
1062 set_bit(XPT_CLOSE, &xprt->xpt_flags);
1063 if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags))
1064
1065 return;
1066
1067
1068
1069
1070
1071
1072 svc_delete_xprt(xprt);
1073}
1074EXPORT_SYMBOL_GPL(svc_close_xprt);
1075
1076static int svc_close_list(struct svc_serv *serv, struct list_head *xprt_list, struct net *net)
1077{
1078 struct svc_xprt *xprt;
1079 int ret = 0;
1080
1081 spin_lock_bh(&serv->sv_lock);
1082 list_for_each_entry(xprt, xprt_list, xpt_list) {
1083 if (xprt->xpt_net != net)
1084 continue;
1085 ret++;
1086 set_bit(XPT_CLOSE, &xprt->xpt_flags);
1087 svc_xprt_enqueue(xprt);
1088 }
1089 spin_unlock_bh(&serv->sv_lock);
1090 return ret;
1091}
1092
1093static struct svc_xprt *svc_dequeue_net(struct svc_serv *serv, struct net *net)
1094{
1095 struct svc_pool *pool;
1096 struct svc_xprt *xprt;
1097 struct svc_xprt *tmp;
1098 int i;
1099
1100 for (i = 0; i < serv->sv_nrpools; i++) {
1101 pool = &serv->sv_pools[i];
1102
1103 spin_lock_bh(&pool->sp_lock);
1104 list_for_each_entry_safe(xprt, tmp, &pool->sp_sockets, xpt_ready) {
1105 if (xprt->xpt_net != net)
1106 continue;
1107 list_del_init(&xprt->xpt_ready);
1108 spin_unlock_bh(&pool->sp_lock);
1109 return xprt;
1110 }
1111 spin_unlock_bh(&pool->sp_lock);
1112 }
1113 return NULL;
1114}
1115
1116static void svc_clean_up_xprts(struct svc_serv *serv, struct net *net)
1117{
1118 struct svc_xprt *xprt;
1119
1120 while ((xprt = svc_dequeue_net(serv, net))) {
1121 set_bit(XPT_CLOSE, &xprt->xpt_flags);
1122 svc_delete_xprt(xprt);
1123 }
1124}
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138void svc_close_net(struct svc_serv *serv, struct net *net)
1139{
1140 int delay = 0;
1141
1142 while (svc_close_list(serv, &serv->sv_permsocks, net) +
1143 svc_close_list(serv, &serv->sv_tempsocks, net)) {
1144
1145 svc_clean_up_xprts(serv, net);
1146 msleep(delay++);
1147 }
1148}
1149
1150
1151
1152
1153
1154static void svc_revisit(struct cache_deferred_req *dreq, int too_many)
1155{
1156 struct svc_deferred_req *dr =
1157 container_of(dreq, struct svc_deferred_req, handle);
1158 struct svc_xprt *xprt = dr->xprt;
1159
1160 spin_lock(&xprt->xpt_lock);
1161 set_bit(XPT_DEFERRED, &xprt->xpt_flags);
1162 if (too_many || test_bit(XPT_DEAD, &xprt->xpt_flags)) {
1163 spin_unlock(&xprt->xpt_lock);
1164 trace_svc_defer_drop(dr);
1165 svc_xprt_put(xprt);
1166 kfree(dr);
1167 return;
1168 }
1169 dr->xprt = NULL;
1170 list_add(&dr->handle.recent, &xprt->xpt_deferred);
1171 spin_unlock(&xprt->xpt_lock);
1172 trace_svc_defer_queue(dr);
1173 svc_xprt_enqueue(xprt);
1174 svc_xprt_put(xprt);
1175}
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186static struct cache_deferred_req *svc_defer(struct cache_req *req)
1187{
1188 struct svc_rqst *rqstp = container_of(req, struct svc_rqst, rq_chandle);
1189 struct svc_deferred_req *dr;
1190
1191 if (rqstp->rq_arg.page_len || !test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags))
1192 return NULL;
1193 if (rqstp->rq_deferred) {
1194 dr = rqstp->rq_deferred;
1195 rqstp->rq_deferred = NULL;
1196 } else {
1197 size_t skip;
1198 size_t size;
1199
1200 size = sizeof(struct svc_deferred_req) + rqstp->rq_arg.len;
1201 dr = kmalloc(size, GFP_KERNEL);
1202 if (dr == NULL)
1203 return NULL;
1204
1205 dr->handle.owner = rqstp->rq_server;
1206 dr->prot = rqstp->rq_prot;
1207 memcpy(&dr->addr, &rqstp->rq_addr, rqstp->rq_addrlen);
1208 dr->addrlen = rqstp->rq_addrlen;
1209 dr->daddr = rqstp->rq_daddr;
1210 dr->argslen = rqstp->rq_arg.len >> 2;
1211 dr->xprt_hlen = rqstp->rq_xprt_hlen;
1212
1213
1214 skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;
1215 memcpy(dr->args, rqstp->rq_arg.head[0].iov_base - skip,
1216 dr->argslen << 2);
1217 }
1218 trace_svc_defer(rqstp);
1219 svc_xprt_get(rqstp->rq_xprt);
1220 dr->xprt = rqstp->rq_xprt;
1221 set_bit(RQ_DROPME, &rqstp->rq_flags);
1222
1223 dr->handle.revisit = svc_revisit;
1224 return &dr->handle;
1225}
1226
1227
1228
1229
1230static noinline int svc_deferred_recv(struct svc_rqst *rqstp)
1231{
1232 struct svc_deferred_req *dr = rqstp->rq_deferred;
1233
1234 trace_svc_defer_recv(dr);
1235
1236
1237 rqstp->rq_arg.head[0].iov_base = dr->args + (dr->xprt_hlen>>2);
1238
1239 rqstp->rq_arg.head[0].iov_len = (dr->argslen<<2) - dr->xprt_hlen;
1240 rqstp->rq_arg.page_len = 0;
1241
1242 rqstp->rq_arg.len = dr->argslen<<2;
1243 rqstp->rq_prot = dr->prot;
1244 memcpy(&rqstp->rq_addr, &dr->addr, dr->addrlen);
1245 rqstp->rq_addrlen = dr->addrlen;
1246
1247 rqstp->rq_xprt_hlen = dr->xprt_hlen;
1248 rqstp->rq_daddr = dr->daddr;
1249 rqstp->rq_respages = rqstp->rq_pages;
1250 svc_xprt_received(rqstp->rq_xprt);
1251 return (dr->argslen<<2) - dr->xprt_hlen;
1252}
1253
1254
1255static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt)
1256{
1257 struct svc_deferred_req *dr = NULL;
1258
1259 if (!test_bit(XPT_DEFERRED, &xprt->xpt_flags))
1260 return NULL;
1261 spin_lock(&xprt->xpt_lock);
1262 if (!list_empty(&xprt->xpt_deferred)) {
1263 dr = list_entry(xprt->xpt_deferred.next,
1264 struct svc_deferred_req,
1265 handle.recent);
1266 list_del_init(&dr->handle.recent);
1267 } else
1268 clear_bit(XPT_DEFERRED, &xprt->xpt_flags);
1269 spin_unlock(&xprt->xpt_lock);
1270 return dr;
1271}
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289struct svc_xprt *svc_find_xprt(struct svc_serv *serv, const char *xcl_name,
1290 struct net *net, const sa_family_t af,
1291 const unsigned short port)
1292{
1293 struct svc_xprt *xprt;
1294 struct svc_xprt *found = NULL;
1295
1296
1297 if (serv == NULL || xcl_name == NULL)
1298 return found;
1299
1300 spin_lock_bh(&serv->sv_lock);
1301 list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {
1302 if (xprt->xpt_net != net)
1303 continue;
1304 if (strcmp(xprt->xpt_class->xcl_name, xcl_name))
1305 continue;
1306 if (af != AF_UNSPEC && af != xprt->xpt_local.ss_family)
1307 continue;
1308 if (port != 0 && port != svc_xprt_local_port(xprt))
1309 continue;
1310 found = xprt;
1311 svc_xprt_get(xprt);
1312 break;
1313 }
1314 spin_unlock_bh(&serv->sv_lock);
1315 return found;
1316}
1317EXPORT_SYMBOL_GPL(svc_find_xprt);
1318
1319static int svc_one_xprt_name(const struct svc_xprt *xprt,
1320 char *pos, int remaining)
1321{
1322 int len;
1323
1324 len = snprintf(pos, remaining, "%s %u\n",
1325 xprt->xpt_class->xcl_name,
1326 svc_xprt_local_port(xprt));
1327 if (len >= remaining)
1328 return -ENAMETOOLONG;
1329 return len;
1330}
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344int svc_xprt_names(struct svc_serv *serv, char *buf, const int buflen)
1345{
1346 struct svc_xprt *xprt;
1347 int len, totlen;
1348 char *pos;
1349
1350
1351 if (!serv)
1352 return 0;
1353
1354 spin_lock_bh(&serv->sv_lock);
1355
1356 pos = buf;
1357 totlen = 0;
1358 list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {
1359 len = svc_one_xprt_name(xprt, pos, buflen - totlen);
1360 if (len < 0) {
1361 *buf = '\0';
1362 totlen = len;
1363 }
1364 if (len <= 0)
1365 break;
1366
1367 pos += len;
1368 totlen += len;
1369 }
1370
1371 spin_unlock_bh(&serv->sv_lock);
1372 return totlen;
1373}
1374EXPORT_SYMBOL_GPL(svc_xprt_names);
1375
1376
1377
1378
1379static void *svc_pool_stats_start(struct seq_file *m, loff_t *pos)
1380{
1381 unsigned int pidx = (unsigned int)*pos;
1382 struct svc_serv *serv = m->private;
1383
1384 dprintk("svc_pool_stats_start, *pidx=%u\n", pidx);
1385
1386 if (!pidx)
1387 return SEQ_START_TOKEN;
1388 return (pidx > serv->sv_nrpools ? NULL : &serv->sv_pools[pidx-1]);
1389}
1390
1391static void *svc_pool_stats_next(struct seq_file *m, void *p, loff_t *pos)
1392{
1393 struct svc_pool *pool = p;
1394 struct svc_serv *serv = m->private;
1395
1396 dprintk("svc_pool_stats_next, *pos=%llu\n", *pos);
1397
1398 if (p == SEQ_START_TOKEN) {
1399 pool = &serv->sv_pools[0];
1400 } else {
1401 unsigned int pidx = (pool - &serv->sv_pools[0]);
1402 if (pidx < serv->sv_nrpools-1)
1403 pool = &serv->sv_pools[pidx+1];
1404 else
1405 pool = NULL;
1406 }
1407 ++*pos;
1408 return pool;
1409}
1410
1411static void svc_pool_stats_stop(struct seq_file *m, void *p)
1412{
1413}
1414
1415static int svc_pool_stats_show(struct seq_file *m, void *p)
1416{
1417 struct svc_pool *pool = p;
1418
1419 if (p == SEQ_START_TOKEN) {
1420 seq_puts(m, "# pool packets-arrived sockets-enqueued threads-woken threads-timedout\n");
1421 return 0;
1422 }
1423
1424 seq_printf(m, "%u %lu %lu %lu %lu\n",
1425 pool->sp_id,
1426 (unsigned long)atomic_long_read(&pool->sp_stats.packets),
1427 pool->sp_stats.sockets_queued,
1428 (unsigned long)atomic_long_read(&pool->sp_stats.threads_woken),
1429 (unsigned long)atomic_long_read(&pool->sp_stats.threads_timedout));
1430
1431 return 0;
1432}
1433
1434static const struct seq_operations svc_pool_stats_seq_ops = {
1435 .start = svc_pool_stats_start,
1436 .next = svc_pool_stats_next,
1437 .stop = svc_pool_stats_stop,
1438 .show = svc_pool_stats_show,
1439};
1440
1441int svc_pool_stats_open(struct svc_serv *serv, struct file *file)
1442{
1443 int err;
1444
1445 err = seq_open(file, &svc_pool_stats_seq_ops);
1446 if (!err)
1447 ((struct seq_file *) file->private_data)->private = serv;
1448 return err;
1449}
1450EXPORT_SYMBOL(svc_pool_stats_open);
1451
1452
1453