1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/splice.h>
24#include <linux/memcontrol.h>
25#include <linux/mm_inline.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
28#include <linux/export.h>
29#include <linux/syscalls.h>
30#include <linux/uio.h>
31#include <linux/security.h>
32#include <linux/gfp.h>
33#include <linux/socket.h>
34
35
36
37
38
39
40
41static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
42 struct pipe_buffer *buf)
43{
44 struct page *page = buf->page;
45 struct address_space *mapping;
46
47 lock_page(page);
48
49 mapping = page_mapping(page);
50 if (mapping) {
51 WARN_ON(!PageUptodate(page));
52
53
54
55
56
57
58
59
60
61 wait_on_page_writeback(page);
62
63 if (page_has_private(page) &&
64 !try_to_release_page(page, GFP_KERNEL))
65 goto out_unlock;
66
67
68
69
70
71 if (remove_mapping(mapping, page)) {
72 buf->flags |= PIPE_BUF_FLAG_LRU;
73 return 0;
74 }
75 }
76
77
78
79
80
81out_unlock:
82 unlock_page(page);
83 return 1;
84}
85
86static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
87 struct pipe_buffer *buf)
88{
89 page_cache_release(buf->page);
90 buf->flags &= ~PIPE_BUF_FLAG_LRU;
91}
92
93
94
95
96
97static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
98 struct pipe_buffer *buf)
99{
100 struct page *page = buf->page;
101 int err;
102
103 if (!PageUptodate(page)) {
104 lock_page(page);
105
106
107
108
109
110 if (!page->mapping) {
111 err = -ENODATA;
112 goto error;
113 }
114
115
116
117
118 if (!PageUptodate(page)) {
119 err = -EIO;
120 goto error;
121 }
122
123
124
125
126 unlock_page(page);
127 }
128
129 return 0;
130error:
131 unlock_page(page);
132 return err;
133}
134
135const struct pipe_buf_operations page_cache_pipe_buf_ops = {
136 .can_merge = 0,
137 .map = generic_pipe_buf_map,
138 .unmap = generic_pipe_buf_unmap,
139 .confirm = page_cache_pipe_buf_confirm,
140 .release = page_cache_pipe_buf_release,
141 .steal = page_cache_pipe_buf_steal,
142 .get = generic_pipe_buf_get,
143};
144
145static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
146 struct pipe_buffer *buf)
147{
148 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
149 return 1;
150
151 buf->flags |= PIPE_BUF_FLAG_LRU;
152 return generic_pipe_buf_steal(pipe, buf);
153}
154
155static const struct pipe_buf_operations user_page_pipe_buf_ops = {
156 .can_merge = 0,
157 .map = generic_pipe_buf_map,
158 .unmap = generic_pipe_buf_unmap,
159 .confirm = generic_pipe_buf_confirm,
160 .release = page_cache_pipe_buf_release,
161 .steal = user_page_pipe_buf_steal,
162 .get = generic_pipe_buf_get,
163};
164
165static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
166{
167 smp_mb();
168 if (waitqueue_active(&pipe->wait))
169 wake_up_interruptible(&pipe->wait);
170 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
171}
172
173
174
175
176
177
178
179
180
181
182
183
184ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
185 struct splice_pipe_desc *spd)
186{
187 unsigned int spd_pages = spd->nr_pages;
188 int ret, do_wakeup, page_nr;
189
190 ret = 0;
191 do_wakeup = 0;
192 page_nr = 0;
193
194 pipe_lock(pipe);
195
196 for (;;) {
197 if (!pipe->readers) {
198 send_sig(SIGPIPE, current, 0);
199 if (!ret)
200 ret = -EPIPE;
201 break;
202 }
203
204 if (pipe->nrbufs < pipe->buffers) {
205 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
206 struct pipe_buffer *buf = pipe->bufs + newbuf;
207
208 buf->page = spd->pages[page_nr];
209 buf->offset = spd->partial[page_nr].offset;
210 buf->len = spd->partial[page_nr].len;
211 buf->private = spd->partial[page_nr].private;
212 buf->ops = spd->ops;
213 if (spd->flags & SPLICE_F_GIFT)
214 buf->flags |= PIPE_BUF_FLAG_GIFT;
215
216 pipe->nrbufs++;
217 page_nr++;
218 ret += buf->len;
219
220 if (pipe->inode)
221 do_wakeup = 1;
222
223 if (!--spd->nr_pages)
224 break;
225 if (pipe->nrbufs < pipe->buffers)
226 continue;
227
228 break;
229 }
230
231 if (spd->flags & SPLICE_F_NONBLOCK) {
232 if (!ret)
233 ret = -EAGAIN;
234 break;
235 }
236
237 if (signal_pending(current)) {
238 if (!ret)
239 ret = -ERESTARTSYS;
240 break;
241 }
242
243 if (do_wakeup) {
244 smp_mb();
245 if (waitqueue_active(&pipe->wait))
246 wake_up_interruptible_sync(&pipe->wait);
247 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
248 do_wakeup = 0;
249 }
250
251 pipe->waiting_writers++;
252 pipe_wait(pipe);
253 pipe->waiting_writers--;
254 }
255
256 pipe_unlock(pipe);
257
258 if (do_wakeup)
259 wakeup_pipe_readers(pipe);
260
261 while (page_nr < spd_pages)
262 spd->spd_release(spd, page_nr++);
263
264 return ret;
265}
266
267void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
268{
269 page_cache_release(spd->pages[i]);
270}
271
272
273
274
275
276int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
277{
278 unsigned int buffers = ACCESS_ONCE(pipe->buffers);
279
280 spd->nr_pages_max = buffers;
281 if (buffers <= PIPE_DEF_BUFFERS)
282 return 0;
283
284 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
285 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
286
287 if (spd->pages && spd->partial)
288 return 0;
289
290 kfree(spd->pages);
291 kfree(spd->partial);
292 return -ENOMEM;
293}
294
295void splice_shrink_spd(struct splice_pipe_desc *spd)
296{
297 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
298 return;
299
300 kfree(spd->pages);
301 kfree(spd->partial);
302}
303
304static int
305__generic_file_splice_read(struct file *in, loff_t *ppos,
306 struct pipe_inode_info *pipe, size_t len,
307 unsigned int flags)
308{
309 struct address_space *mapping = in->f_mapping;
310 unsigned int loff, nr_pages, req_pages;
311 struct page *pages[PIPE_DEF_BUFFERS];
312 struct partial_page partial[PIPE_DEF_BUFFERS];
313 struct page *page;
314 pgoff_t index, end_index;
315 loff_t isize;
316 int error, page_nr;
317 struct splice_pipe_desc spd = {
318 .pages = pages,
319 .partial = partial,
320 .nr_pages_max = PIPE_DEF_BUFFERS,
321 .flags = flags,
322 .ops = &page_cache_pipe_buf_ops,
323 .spd_release = spd_release_page,
324 };
325
326 if (splice_grow_spd(pipe, &spd))
327 return -ENOMEM;
328
329 index = *ppos >> PAGE_CACHE_SHIFT;
330 loff = *ppos & ~PAGE_CACHE_MASK;
331 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
332 nr_pages = min(req_pages, spd.nr_pages_max);
333
334
335
336
337 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
338 index += spd.nr_pages;
339
340
341
342
343
344 if (spd.nr_pages < nr_pages)
345 page_cache_sync_readahead(mapping, &in->f_ra, in,
346 index, req_pages - spd.nr_pages);
347
348 error = 0;
349 while (spd.nr_pages < nr_pages) {
350
351
352
353
354 page = find_get_page(mapping, index);
355 if (!page) {
356
357
358
359 page = page_cache_alloc_cold(mapping);
360 if (!page)
361 break;
362
363 error = add_to_page_cache_lru(page, mapping, index,
364 GFP_KERNEL);
365 if (unlikely(error)) {
366 page_cache_release(page);
367 if (error == -EEXIST)
368 continue;
369 break;
370 }
371
372
373
374
375 unlock_page(page);
376 }
377
378 spd.pages[spd.nr_pages++] = page;
379 index++;
380 }
381
382
383
384
385
386 index = *ppos >> PAGE_CACHE_SHIFT;
387 nr_pages = spd.nr_pages;
388 spd.nr_pages = 0;
389 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
390 unsigned int this_len;
391
392 if (!len)
393 break;
394
395
396
397
398 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
399 page = spd.pages[page_nr];
400
401 if (PageReadahead(page))
402 page_cache_async_readahead(mapping, &in->f_ra, in,
403 page, index, req_pages - page_nr);
404
405
406
407
408 if (!PageUptodate(page)) {
409 lock_page(page);
410
411
412
413
414
415
416
417 if (!page->mapping) {
418 unlock_page(page);
419 page = find_or_create_page(mapping, index,
420 mapping_gfp_mask(mapping));
421
422 if (!page) {
423 error = -ENOMEM;
424 break;
425 }
426 page_cache_release(spd.pages[page_nr]);
427 spd.pages[page_nr] = page;
428 }
429
430
431
432 if (PageUptodate(page)) {
433 unlock_page(page);
434 goto fill_it;
435 }
436
437
438
439
440 error = mapping->a_ops->readpage(in, page);
441 if (unlikely(error)) {
442
443
444
445
446
447
448 if (error == AOP_TRUNCATED_PAGE)
449 error = 0;
450
451 break;
452 }
453 }
454fill_it:
455
456
457
458 isize = i_size_read(mapping->host);
459 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
460 if (unlikely(!isize || index > end_index))
461 break;
462
463
464
465
466
467 if (end_index == index) {
468 unsigned int plen;
469
470
471
472
473 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
474 if (plen <= loff)
475 break;
476
477
478
479
480 this_len = min(this_len, plen - loff);
481 len = this_len;
482 }
483
484 spd.partial[page_nr].offset = loff;
485 spd.partial[page_nr].len = this_len;
486 len -= this_len;
487 loff = 0;
488 spd.nr_pages++;
489 index++;
490 }
491
492
493
494
495
496 while (page_nr < nr_pages)
497 page_cache_release(spd.pages[page_nr++]);
498 in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
499
500 if (spd.nr_pages)
501 error = splice_to_pipe(pipe, &spd);
502
503 splice_shrink_spd(&spd);
504 return error;
505}
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
522 struct pipe_inode_info *pipe, size_t len,
523 unsigned int flags)
524{
525 loff_t isize, left;
526 int ret;
527
528 isize = i_size_read(in->f_mapping->host);
529 if (unlikely(*ppos >= isize))
530 return 0;
531
532 left = isize - *ppos;
533 if (unlikely(left < len))
534 len = left;
535
536 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
537 if (ret > 0) {
538 *ppos += ret;
539 file_accessed(in);
540 }
541
542 return ret;
543}
544EXPORT_SYMBOL(generic_file_splice_read);
545
546static const struct pipe_buf_operations default_pipe_buf_ops = {
547 .can_merge = 0,
548 .map = generic_pipe_buf_map,
549 .unmap = generic_pipe_buf_unmap,
550 .confirm = generic_pipe_buf_confirm,
551 .release = generic_pipe_buf_release,
552 .steal = generic_pipe_buf_steal,
553 .get = generic_pipe_buf_get,
554};
555
556static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
557 unsigned long vlen, loff_t offset)
558{
559 mm_segment_t old_fs;
560 loff_t pos = offset;
561 ssize_t res;
562
563 old_fs = get_fs();
564 set_fs(get_ds());
565
566 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
567 set_fs(old_fs);
568
569 return res;
570}
571
572static ssize_t kernel_write(struct file *file, const char *buf, size_t count,
573 loff_t pos)
574{
575 mm_segment_t old_fs;
576 ssize_t res;
577
578 old_fs = get_fs();
579 set_fs(get_ds());
580
581 res = vfs_write(file, (const char __user *)buf, count, &pos);
582 set_fs(old_fs);
583
584 return res;
585}
586
587ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
588 struct pipe_inode_info *pipe, size_t len,
589 unsigned int flags)
590{
591 unsigned int nr_pages;
592 unsigned int nr_freed;
593 size_t offset;
594 struct page *pages[PIPE_DEF_BUFFERS];
595 struct partial_page partial[PIPE_DEF_BUFFERS];
596 struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
597 ssize_t res;
598 size_t this_len;
599 int error;
600 int i;
601 struct splice_pipe_desc spd = {
602 .pages = pages,
603 .partial = partial,
604 .nr_pages_max = PIPE_DEF_BUFFERS,
605 .flags = flags,
606 .ops = &default_pipe_buf_ops,
607 .spd_release = spd_release_page,
608 };
609
610 if (splice_grow_spd(pipe, &spd))
611 return -ENOMEM;
612
613 res = -ENOMEM;
614 vec = __vec;
615 if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
616 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
617 if (!vec)
618 goto shrink_ret;
619 }
620
621 offset = *ppos & ~PAGE_CACHE_MASK;
622 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
623
624 for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
625 struct page *page;
626
627 page = alloc_page(GFP_USER);
628 error = -ENOMEM;
629 if (!page)
630 goto err;
631
632 this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
633 vec[i].iov_base = (void __user *) page_address(page);
634 vec[i].iov_len = this_len;
635 spd.pages[i] = page;
636 spd.nr_pages++;
637 len -= this_len;
638 offset = 0;
639 }
640
641 res = kernel_readv(in, vec, spd.nr_pages, *ppos);
642 if (res < 0) {
643 error = res;
644 goto err;
645 }
646
647 error = 0;
648 if (!res)
649 goto err;
650
651 nr_freed = 0;
652 for (i = 0; i < spd.nr_pages; i++) {
653 this_len = min_t(size_t, vec[i].iov_len, res);
654 spd.partial[i].offset = 0;
655 spd.partial[i].len = this_len;
656 if (!this_len) {
657 __free_page(spd.pages[i]);
658 spd.pages[i] = NULL;
659 nr_freed++;
660 }
661 res -= this_len;
662 }
663 spd.nr_pages -= nr_freed;
664
665 res = splice_to_pipe(pipe, &spd);
666 if (res > 0)
667 *ppos += res;
668
669shrink_ret:
670 if (vec != __vec)
671 kfree(vec);
672 splice_shrink_spd(&spd);
673 return res;
674
675err:
676 for (i = 0; i < spd.nr_pages; i++)
677 __free_page(spd.pages[i]);
678
679 res = error;
680 goto shrink_ret;
681}
682EXPORT_SYMBOL(default_file_splice_read);
683
684
685
686
687
688static int pipe_to_sendpage(struct pipe_inode_info *pipe,
689 struct pipe_buffer *buf, struct splice_desc *sd)
690{
691 struct file *file = sd->u.file;
692 loff_t pos = sd->pos;
693 int more;
694
695 if (!likely(file->f_op && file->f_op->sendpage))
696 return -EINVAL;
697
698 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
699 if (sd->len < sd->total_len)
700 more |= MSG_SENDPAGE_NOTLAST;
701 return file->f_op->sendpage(file, buf->page, buf->offset,
702 sd->len, &pos, more);
703}
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
726 struct splice_desc *sd)
727{
728 struct file *file = sd->u.file;
729 struct address_space *mapping = file->f_mapping;
730 unsigned int offset, this_len;
731 struct page *page;
732 void *fsdata;
733 int ret;
734
735 offset = sd->pos & ~PAGE_CACHE_MASK;
736
737 this_len = sd->len;
738 if (this_len + offset > PAGE_CACHE_SIZE)
739 this_len = PAGE_CACHE_SIZE - offset;
740
741 ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
742 AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
743 if (unlikely(ret))
744 goto out;
745
746 if (buf->page != page) {
747 char *src = buf->ops->map(pipe, buf, 1);
748 char *dst = kmap_atomic(page);
749
750 memcpy(dst + offset, src + buf->offset, this_len);
751 flush_dcache_page(page);
752 kunmap_atomic(dst);
753 buf->ops->unmap(pipe, buf, src);
754 }
755 ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
756 page, fsdata);
757out:
758 return ret;
759}
760EXPORT_SYMBOL(pipe_to_file);
761
762static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
763{
764 smp_mb();
765 if (waitqueue_active(&pipe->wait))
766 wake_up_interruptible(&pipe->wait);
767 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
768}
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
791 splice_actor *actor)
792{
793 int ret;
794
795 while (pipe->nrbufs) {
796 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
797 const struct pipe_buf_operations *ops = buf->ops;
798
799 sd->len = buf->len;
800 if (sd->len > sd->total_len)
801 sd->len = sd->total_len;
802
803 ret = buf->ops->confirm(pipe, buf);
804 if (unlikely(ret)) {
805 if (ret == -ENODATA)
806 ret = 0;
807 return ret;
808 }
809
810 ret = actor(pipe, buf, sd);
811 if (ret <= 0)
812 return ret;
813
814 buf->offset += ret;
815 buf->len -= ret;
816
817 sd->num_spliced += ret;
818 sd->len -= ret;
819 sd->pos += ret;
820 sd->total_len -= ret;
821
822 if (!buf->len) {
823 buf->ops = NULL;
824 ops->release(pipe, buf);
825 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
826 pipe->nrbufs--;
827 if (pipe->inode)
828 sd->need_wakeup = true;
829 }
830
831 if (!sd->total_len)
832 return 0;
833 }
834
835 return 1;
836}
837EXPORT_SYMBOL(splice_from_pipe_feed);
838
839
840
841
842
843
844
845
846
847
848
849int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
850{
851 while (!pipe->nrbufs) {
852 if (!pipe->writers)
853 return 0;
854
855 if (!pipe->waiting_writers && sd->num_spliced)
856 return 0;
857
858 if (sd->flags & SPLICE_F_NONBLOCK)
859 return -EAGAIN;
860
861 if (signal_pending(current))
862 return -ERESTARTSYS;
863
864 if (sd->need_wakeup) {
865 wakeup_pipe_writers(pipe);
866 sd->need_wakeup = false;
867 }
868
869 pipe_wait(pipe);
870 }
871
872 return 1;
873}
874EXPORT_SYMBOL(splice_from_pipe_next);
875
876
877
878
879
880
881
882
883
884
885void splice_from_pipe_begin(struct splice_desc *sd)
886{
887 sd->num_spliced = 0;
888 sd->need_wakeup = false;
889}
890EXPORT_SYMBOL(splice_from_pipe_begin);
891
892
893
894
895
896
897
898
899
900
901
902void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
903{
904 if (sd->need_wakeup)
905 wakeup_pipe_writers(pipe);
906}
907EXPORT_SYMBOL(splice_from_pipe_end);
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
923 splice_actor *actor)
924{
925 int ret;
926
927 splice_from_pipe_begin(sd);
928 do {
929 ret = splice_from_pipe_next(pipe, sd);
930 if (ret > 0)
931 ret = splice_from_pipe_feed(pipe, sd, actor);
932 } while (ret > 0);
933 splice_from_pipe_end(pipe, sd);
934
935 return sd->num_spliced ? sd->num_spliced : ret;
936}
937EXPORT_SYMBOL(__splice_from_pipe);
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
954 loff_t *ppos, size_t len, unsigned int flags,
955 splice_actor *actor)
956{
957 ssize_t ret;
958 struct splice_desc sd = {
959 .total_len = len,
960 .flags = flags,
961 .pos = *ppos,
962 .u.file = out,
963 };
964
965 pipe_lock(pipe);
966 ret = __splice_from_pipe(pipe, &sd, actor);
967 pipe_unlock(pipe);
968
969 return ret;
970}
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985ssize_t
986generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
987 loff_t *ppos, size_t len, unsigned int flags)
988{
989 struct address_space *mapping = out->f_mapping;
990 struct inode *inode = mapping->host;
991 struct splice_desc sd = {
992 .total_len = len,
993 .flags = flags,
994 .pos = *ppos,
995 .u.file = out,
996 };
997 ssize_t ret;
998
999 sb_start_write(inode->i_sb);
1000
1001 pipe_lock(pipe);
1002
1003 splice_from_pipe_begin(&sd);
1004 do {
1005 ret = splice_from_pipe_next(pipe, &sd);
1006 if (ret <= 0)
1007 break;
1008
1009 mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
1010 ret = file_remove_suid(out);
1011 if (!ret) {
1012 ret = file_update_time(out);
1013 if (!ret)
1014 ret = splice_from_pipe_feed(pipe, &sd,
1015 pipe_to_file);
1016 }
1017 mutex_unlock(&inode->i_mutex);
1018 } while (ret > 0);
1019 splice_from_pipe_end(pipe, &sd);
1020
1021 pipe_unlock(pipe);
1022
1023 if (sd.num_spliced)
1024 ret = sd.num_spliced;
1025
1026 if (ret > 0) {
1027 unsigned long nr_pages;
1028 int err;
1029
1030 nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1031
1032 err = generic_write_sync(out, *ppos, ret);
1033 if (err)
1034 ret = err;
1035 else
1036 *ppos += ret;
1037 balance_dirty_pages_ratelimited_nr(mapping, nr_pages);
1038 }
1039 sb_end_write(inode->i_sb);
1040
1041 return ret;
1042}
1043
1044EXPORT_SYMBOL(generic_file_splice_write);
1045
1046static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1047 struct splice_desc *sd)
1048{
1049 int ret;
1050 void *data;
1051
1052 data = buf->ops->map(pipe, buf, 0);
1053 ret = kernel_write(sd->u.file, data + buf->offset, sd->len, sd->pos);
1054 buf->ops->unmap(pipe, buf, data);
1055
1056 return ret;
1057}
1058
1059static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
1060 struct file *out, loff_t *ppos,
1061 size_t len, unsigned int flags)
1062{
1063 ssize_t ret;
1064
1065 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
1066 if (ret > 0)
1067 *ppos += ret;
1068
1069 return ret;
1070}
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1086 loff_t *ppos, size_t len, unsigned int flags)
1087{
1088 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1089}
1090
1091EXPORT_SYMBOL(generic_splice_sendpage);
1092
1093
1094
1095
1096static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1097 loff_t *ppos, size_t len, unsigned int flags)
1098{
1099 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
1100 loff_t *, size_t, unsigned int);
1101 int ret;
1102
1103 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1104 return -EBADF;
1105
1106 if (unlikely(out->f_flags & O_APPEND))
1107 return -EINVAL;
1108
1109 ret = rw_verify_area(WRITE, out, ppos, len);
1110 if (unlikely(ret < 0))
1111 return ret;
1112
1113 if (out->f_op && out->f_op->splice_write)
1114 splice_write = out->f_op->splice_write;
1115 else
1116 splice_write = default_file_splice_write;
1117
1118 return splice_write(pipe, out, ppos, len, flags);
1119}
1120
1121
1122
1123
1124static long do_splice_to(struct file *in, loff_t *ppos,
1125 struct pipe_inode_info *pipe, size_t len,
1126 unsigned int flags)
1127{
1128 ssize_t (*splice_read)(struct file *, loff_t *,
1129 struct pipe_inode_info *, size_t, unsigned int);
1130 int ret;
1131
1132 if (unlikely(!(in->f_mode & FMODE_READ)))
1133 return -EBADF;
1134
1135 ret = rw_verify_area(READ, in, ppos, len);
1136 if (unlikely(ret < 0))
1137 return ret;
1138
1139 if (in->f_op && in->f_op->splice_read)
1140 splice_read = in->f_op->splice_read;
1141 else
1142 splice_read = default_file_splice_read;
1143
1144 return splice_read(in, ppos, pipe, len, flags);
1145}
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1161 splice_direct_actor *actor)
1162{
1163 struct pipe_inode_info *pipe;
1164 long ret, bytes;
1165 umode_t i_mode;
1166 size_t len;
1167 int i, flags;
1168
1169
1170
1171
1172
1173
1174 i_mode = in->f_path.dentry->d_inode->i_mode;
1175 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1176 return -EINVAL;
1177
1178
1179
1180
1181
1182 pipe = current->splice_pipe;
1183 if (unlikely(!pipe)) {
1184 pipe = alloc_pipe_info(NULL);
1185 if (!pipe)
1186 return -ENOMEM;
1187
1188
1189
1190
1191
1192
1193 pipe->readers = 1;
1194
1195 current->splice_pipe = pipe;
1196 }
1197
1198
1199
1200
1201 ret = 0;
1202 bytes = 0;
1203 len = sd->total_len;
1204 flags = sd->flags;
1205
1206
1207
1208
1209 sd->flags &= ~SPLICE_F_NONBLOCK;
1210
1211 while (len) {
1212 size_t read_len;
1213 loff_t pos = sd->pos, prev_pos = pos;
1214
1215 ret = do_splice_to(in, &pos, pipe, len, flags);
1216 if (unlikely(ret <= 0))
1217 goto out_release;
1218
1219 read_len = ret;
1220 sd->total_len = read_len;
1221
1222
1223
1224
1225
1226
1227 ret = actor(pipe, sd);
1228 if (unlikely(ret <= 0)) {
1229 sd->pos = prev_pos;
1230 goto out_release;
1231 }
1232
1233 bytes += ret;
1234 len -= ret;
1235 sd->pos = pos;
1236
1237 if (ret < read_len) {
1238 sd->pos = prev_pos + ret;
1239 goto out_release;
1240 }
1241 }
1242
1243done:
1244 pipe->nrbufs = pipe->curbuf = 0;
1245 file_accessed(in);
1246 return bytes;
1247
1248out_release:
1249
1250
1251
1252
1253 for (i = 0; i < pipe->buffers; i++) {
1254 struct pipe_buffer *buf = pipe->bufs + i;
1255
1256 if (buf->ops) {
1257 buf->ops->release(pipe, buf);
1258 buf->ops = NULL;
1259 }
1260 }
1261
1262 if (!bytes)
1263 bytes = ret;
1264
1265 goto done;
1266}
1267EXPORT_SYMBOL(splice_direct_to_actor);
1268
1269static int direct_splice_actor(struct pipe_inode_info *pipe,
1270 struct splice_desc *sd)
1271{
1272 struct file *file = sd->u.file;
1273
1274 return do_splice_from(pipe, file, &file->f_pos, sd->total_len,
1275 sd->flags);
1276}
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1294 size_t len, unsigned int flags)
1295{
1296 struct splice_desc sd = {
1297 .len = len,
1298 .total_len = len,
1299 .flags = flags,
1300 .pos = *ppos,
1301 .u.file = out,
1302 };
1303 long ret;
1304
1305 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1306 if (ret > 0)
1307 *ppos = sd.pos;
1308
1309 return ret;
1310}
1311
1312static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1313 struct pipe_inode_info *opipe,
1314 size_t len, unsigned int flags);
1315
1316
1317
1318
1319static long do_splice(struct file *in, loff_t __user *off_in,
1320 struct file *out, loff_t __user *off_out,
1321 size_t len, unsigned int flags)
1322{
1323 struct pipe_inode_info *ipipe;
1324 struct pipe_inode_info *opipe;
1325 loff_t offset, *off;
1326 long ret;
1327
1328 ipipe = get_pipe_info(in);
1329 opipe = get_pipe_info(out);
1330
1331 if (ipipe && opipe) {
1332 if (off_in || off_out)
1333 return -ESPIPE;
1334
1335 if (!(in->f_mode & FMODE_READ))
1336 return -EBADF;
1337
1338 if (!(out->f_mode & FMODE_WRITE))
1339 return -EBADF;
1340
1341
1342 if (ipipe == opipe)
1343 return -EINVAL;
1344
1345 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1346 }
1347
1348 if (ipipe) {
1349 if (off_in)
1350 return -ESPIPE;
1351 if (off_out) {
1352 if (!(out->f_mode & FMODE_PWRITE))
1353 return -EINVAL;
1354 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1355 return -EFAULT;
1356 off = &offset;
1357 } else
1358 off = &out->f_pos;
1359
1360 ret = do_splice_from(ipipe, out, off, len, flags);
1361
1362 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1363 ret = -EFAULT;
1364
1365 return ret;
1366 }
1367
1368 if (opipe) {
1369 if (off_out)
1370 return -ESPIPE;
1371 if (off_in) {
1372 if (!(in->f_mode & FMODE_PREAD))
1373 return -EINVAL;
1374 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1375 return -EFAULT;
1376 off = &offset;
1377 } else
1378 off = &in->f_pos;
1379
1380 ret = do_splice_to(in, off, opipe, len, flags);
1381
1382 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1383 ret = -EFAULT;
1384
1385 return ret;
1386 }
1387
1388 return -EINVAL;
1389}
1390
1391
1392
1393
1394
1395
1396
1397
1398static int get_iovec_page_array(const struct iovec __user *iov,
1399 unsigned int nr_vecs, struct page **pages,
1400 struct partial_page *partial, bool aligned,
1401 unsigned int pipe_buffers)
1402{
1403 int buffers = 0, error = 0;
1404
1405 while (nr_vecs) {
1406 unsigned long off, npages;
1407 struct iovec entry;
1408 void __user *base;
1409 size_t len;
1410 int i;
1411
1412 error = -EFAULT;
1413 if (copy_from_user(&entry, iov, sizeof(entry)))
1414 break;
1415
1416 base = entry.iov_base;
1417 len = entry.iov_len;
1418
1419
1420
1421
1422 error = 0;
1423 if (unlikely(!len))
1424 break;
1425 error = -EFAULT;
1426 if (!access_ok(VERIFY_READ, base, len))
1427 break;
1428
1429
1430
1431
1432
1433 off = (unsigned long) base & ~PAGE_MASK;
1434
1435
1436
1437
1438
1439 error = -EINVAL;
1440 if (aligned && (off || len & ~PAGE_MASK))
1441 break;
1442
1443 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1444 if (npages > pipe_buffers - buffers)
1445 npages = pipe_buffers - buffers;
1446
1447 error = get_user_pages_fast((unsigned long)base, npages,
1448 0, &pages[buffers]);
1449
1450 if (unlikely(error <= 0))
1451 break;
1452
1453
1454
1455
1456 for (i = 0; i < error; i++) {
1457 const int plen = min_t(size_t, len, PAGE_SIZE - off);
1458
1459 partial[buffers].offset = off;
1460 partial[buffers].len = plen;
1461
1462 off = 0;
1463 len -= plen;
1464 buffers++;
1465 }
1466
1467
1468
1469
1470
1471
1472 if (len)
1473 break;
1474
1475
1476
1477
1478
1479
1480 if (error < npages || buffers == pipe_buffers)
1481 break;
1482
1483 nr_vecs--;
1484 iov++;
1485 }
1486
1487 if (buffers)
1488 return buffers;
1489
1490 return error;
1491}
1492
1493static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1494 struct splice_desc *sd)
1495{
1496 char *src;
1497 int ret;
1498
1499
1500
1501
1502
1503 if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) {
1504 src = buf->ops->map(pipe, buf, 1);
1505 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset,
1506 sd->len);
1507 buf->ops->unmap(pipe, buf, src);
1508 if (!ret) {
1509 ret = sd->len;
1510 goto out;
1511 }
1512 }
1513
1514
1515
1516
1517 src = buf->ops->map(pipe, buf, 0);
1518
1519 ret = sd->len;
1520 if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len))
1521 ret = -EFAULT;
1522
1523 buf->ops->unmap(pipe, buf, src);
1524out:
1525 if (ret > 0)
1526 sd->u.userptr += ret;
1527 return ret;
1528}
1529
1530
1531
1532
1533
1534static long vmsplice_to_user(struct file *file, const struct iovec __user *iov,
1535 unsigned long nr_segs, unsigned int flags)
1536{
1537 struct pipe_inode_info *pipe;
1538 struct splice_desc sd;
1539 ssize_t size;
1540 int error;
1541 long ret;
1542
1543 pipe = get_pipe_info(file);
1544 if (!pipe)
1545 return -EBADF;
1546
1547 pipe_lock(pipe);
1548
1549 error = ret = 0;
1550 while (nr_segs) {
1551 void __user *base;
1552 size_t len;
1553
1554
1555
1556
1557 error = get_user(base, &iov->iov_base);
1558 if (unlikely(error))
1559 break;
1560 error = get_user(len, &iov->iov_len);
1561 if (unlikely(error))
1562 break;
1563
1564
1565
1566
1567 if (unlikely(!len))
1568 break;
1569 if (unlikely(!base)) {
1570 error = -EFAULT;
1571 break;
1572 }
1573
1574 if (unlikely(!access_ok(VERIFY_WRITE, base, len))) {
1575 error = -EFAULT;
1576 break;
1577 }
1578
1579 sd.len = 0;
1580 sd.total_len = len;
1581 sd.flags = flags;
1582 sd.u.userptr = base;
1583 sd.pos = 0;
1584
1585 size = __splice_from_pipe(pipe, &sd, pipe_to_user);
1586 if (size < 0) {
1587 if (!ret)
1588 ret = size;
1589
1590 break;
1591 }
1592
1593 ret += size;
1594
1595 if (size < len)
1596 break;
1597
1598 nr_segs--;
1599 iov++;
1600 }
1601
1602 pipe_unlock(pipe);
1603
1604 if (!ret)
1605 ret = error;
1606
1607 return ret;
1608}
1609
1610
1611
1612
1613
1614
1615static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1616 unsigned long nr_segs, unsigned int flags)
1617{
1618 struct pipe_inode_info *pipe;
1619 struct page *pages[PIPE_DEF_BUFFERS];
1620 struct partial_page partial[PIPE_DEF_BUFFERS];
1621 struct splice_pipe_desc spd = {
1622 .pages = pages,
1623 .partial = partial,
1624 .nr_pages_max = PIPE_DEF_BUFFERS,
1625 .flags = flags,
1626 .ops = &user_page_pipe_buf_ops,
1627 .spd_release = spd_release_page,
1628 };
1629 long ret;
1630
1631 pipe = get_pipe_info(file);
1632 if (!pipe)
1633 return -EBADF;
1634
1635 if (splice_grow_spd(pipe, &spd))
1636 return -ENOMEM;
1637
1638 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1639 spd.partial, false,
1640 spd.nr_pages_max);
1641 if (spd.nr_pages <= 0)
1642 ret = spd.nr_pages;
1643 else
1644 ret = splice_to_pipe(pipe, &spd);
1645
1646 splice_shrink_spd(&spd);
1647 return ret;
1648}
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
1667 unsigned long, nr_segs, unsigned int, flags)
1668{
1669 struct file *file;
1670 long error;
1671 int fput;
1672
1673 if (unlikely(nr_segs > UIO_MAXIOV))
1674 return -EINVAL;
1675 else if (unlikely(!nr_segs))
1676 return 0;
1677
1678 error = -EBADF;
1679 file = fget_light(fd, &fput);
1680 if (file) {
1681 if (file->f_mode & FMODE_WRITE)
1682 error = vmsplice_to_pipe(file, iov, nr_segs, flags);
1683 else if (file->f_mode & FMODE_READ)
1684 error = vmsplice_to_user(file, iov, nr_segs, flags);
1685
1686 fput_light(file, fput);
1687 }
1688
1689 return error;
1690}
1691
1692SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1693 int, fd_out, loff_t __user *, off_out,
1694 size_t, len, unsigned int, flags)
1695{
1696 long error;
1697 struct file *in, *out;
1698 int fput_in, fput_out;
1699
1700 if (unlikely(!len))
1701 return 0;
1702
1703 error = -EBADF;
1704 in = fget_light(fd_in, &fput_in);
1705 if (in) {
1706 if (in->f_mode & FMODE_READ) {
1707 out = fget_light(fd_out, &fput_out);
1708 if (out) {
1709 if (out->f_mode & FMODE_WRITE)
1710 error = do_splice(in, off_in,
1711 out, off_out,
1712 len, flags);
1713 fput_light(out, fput_out);
1714 }
1715 }
1716
1717 fput_light(in, fput_in);
1718 }
1719
1720 return error;
1721}
1722
1723
1724
1725
1726
1727static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1728{
1729 int ret;
1730
1731
1732
1733
1734
1735 if (pipe->nrbufs)
1736 return 0;
1737
1738 ret = 0;
1739 pipe_lock(pipe);
1740
1741 while (!pipe->nrbufs) {
1742 if (signal_pending(current)) {
1743 ret = -ERESTARTSYS;
1744 break;
1745 }
1746 if (!pipe->writers)
1747 break;
1748 if (!pipe->waiting_writers) {
1749 if (flags & SPLICE_F_NONBLOCK) {
1750 ret = -EAGAIN;
1751 break;
1752 }
1753 }
1754 pipe_wait(pipe);
1755 }
1756
1757 pipe_unlock(pipe);
1758 return ret;
1759}
1760
1761
1762
1763
1764
1765static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1766{
1767 int ret;
1768
1769
1770
1771
1772
1773 if (pipe->nrbufs < pipe->buffers)
1774 return 0;
1775
1776 ret = 0;
1777 pipe_lock(pipe);
1778
1779 while (pipe->nrbufs >= pipe->buffers) {
1780 if (!pipe->readers) {
1781 send_sig(SIGPIPE, current, 0);
1782 ret = -EPIPE;
1783 break;
1784 }
1785 if (flags & SPLICE_F_NONBLOCK) {
1786 ret = -EAGAIN;
1787 break;
1788 }
1789 if (signal_pending(current)) {
1790 ret = -ERESTARTSYS;
1791 break;
1792 }
1793 pipe->waiting_writers++;
1794 pipe_wait(pipe);
1795 pipe->waiting_writers--;
1796 }
1797
1798 pipe_unlock(pipe);
1799 return ret;
1800}
1801
1802
1803
1804
1805static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1806 struct pipe_inode_info *opipe,
1807 size_t len, unsigned int flags)
1808{
1809 struct pipe_buffer *ibuf, *obuf;
1810 int ret = 0, nbuf;
1811 bool input_wakeup = false;
1812
1813
1814retry:
1815 ret = ipipe_prep(ipipe, flags);
1816 if (ret)
1817 return ret;
1818
1819 ret = opipe_prep(opipe, flags);
1820 if (ret)
1821 return ret;
1822
1823
1824
1825
1826
1827
1828 pipe_double_lock(ipipe, opipe);
1829
1830 do {
1831 if (!opipe->readers) {
1832 send_sig(SIGPIPE, current, 0);
1833 if (!ret)
1834 ret = -EPIPE;
1835 break;
1836 }
1837
1838 if (!ipipe->nrbufs && !ipipe->writers)
1839 break;
1840
1841
1842
1843
1844
1845 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1846
1847 if (ret)
1848 break;
1849
1850 if (flags & SPLICE_F_NONBLOCK) {
1851 ret = -EAGAIN;
1852 break;
1853 }
1854
1855
1856
1857
1858
1859
1860 pipe_unlock(ipipe);
1861 pipe_unlock(opipe);
1862 goto retry;
1863 }
1864
1865 ibuf = ipipe->bufs + ipipe->curbuf;
1866 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1867 obuf = opipe->bufs + nbuf;
1868
1869 if (len >= ibuf->len) {
1870
1871
1872
1873 *obuf = *ibuf;
1874 ibuf->ops = NULL;
1875 opipe->nrbufs++;
1876 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1877 ipipe->nrbufs--;
1878 input_wakeup = true;
1879 } else {
1880
1881
1882
1883
1884 ibuf->ops->get(ipipe, ibuf);
1885 *obuf = *ibuf;
1886
1887
1888
1889
1890
1891 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1892
1893 obuf->len = len;
1894 opipe->nrbufs++;
1895 ibuf->offset += obuf->len;
1896 ibuf->len -= obuf->len;
1897 }
1898 ret += obuf->len;
1899 len -= obuf->len;
1900 } while (len);
1901
1902 pipe_unlock(ipipe);
1903 pipe_unlock(opipe);
1904
1905
1906
1907
1908 if (ret > 0)
1909 wakeup_pipe_readers(opipe);
1910
1911 if (input_wakeup)
1912 wakeup_pipe_writers(ipipe);
1913
1914 return ret;
1915}
1916
1917
1918
1919
1920static int link_pipe(struct pipe_inode_info *ipipe,
1921 struct pipe_inode_info *opipe,
1922 size_t len, unsigned int flags)
1923{
1924 struct pipe_buffer *ibuf, *obuf;
1925 int ret = 0, i = 0, nbuf;
1926
1927
1928
1929
1930
1931
1932 pipe_double_lock(ipipe, opipe);
1933
1934 do {
1935 if (!opipe->readers) {
1936 send_sig(SIGPIPE, current, 0);
1937 if (!ret)
1938 ret = -EPIPE;
1939 break;
1940 }
1941
1942
1943
1944
1945
1946 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
1947 break;
1948
1949 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
1950 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1951
1952
1953
1954
1955
1956 ibuf->ops->get(ipipe, ibuf);
1957
1958 obuf = opipe->bufs + nbuf;
1959 *obuf = *ibuf;
1960
1961
1962
1963
1964
1965 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1966
1967 if (obuf->len > len)
1968 obuf->len = len;
1969
1970 opipe->nrbufs++;
1971 ret += obuf->len;
1972 len -= obuf->len;
1973 i++;
1974 } while (len);
1975
1976
1977
1978
1979
1980 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
1981 ret = -EAGAIN;
1982
1983 pipe_unlock(ipipe);
1984 pipe_unlock(opipe);
1985
1986
1987
1988
1989 if (ret > 0)
1990 wakeup_pipe_readers(opipe);
1991
1992 return ret;
1993}
1994
1995
1996
1997
1998
1999
2000
2001static long do_tee(struct file *in, struct file *out, size_t len,
2002 unsigned int flags)
2003{
2004 struct pipe_inode_info *ipipe = get_pipe_info(in);
2005 struct pipe_inode_info *opipe = get_pipe_info(out);
2006 int ret = -EINVAL;
2007
2008
2009
2010
2011
2012 if (ipipe && opipe && ipipe != opipe) {
2013
2014
2015
2016
2017 ret = ipipe_prep(ipipe, flags);
2018 if (!ret) {
2019 ret = opipe_prep(opipe, flags);
2020 if (!ret)
2021 ret = link_pipe(ipipe, opipe, len, flags);
2022 }
2023 }
2024
2025 return ret;
2026}
2027
2028SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2029{
2030 struct file *in;
2031 int error, fput_in;
2032
2033 if (unlikely(!len))
2034 return 0;
2035
2036 error = -EBADF;
2037 in = fget_light(fdin, &fput_in);
2038 if (in) {
2039 if (in->f_mode & FMODE_READ) {
2040 int fput_out;
2041 struct file *out = fget_light(fdout, &fput_out);
2042
2043 if (out) {
2044 if (out->f_mode & FMODE_WRITE)
2045 error = do_tee(in, out, len, flags);
2046 fput_light(out, fput_out);
2047 }
2048 }
2049 fput_light(in, fput_in);
2050 }
2051
2052 return error;
2053}
2054