1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/memcontrol.h>
58#include <linux/res_counter.h>
59#include <linux/static_key.h>
60#include <linux/aio.h>
61#include <linux/sched.h>
62
63#include <linux/filter.h>
64#include <linux/rculist_nulls.h>
65#include <linux/poll.h>
66
67#include <linux/atomic.h>
68#include <net/dst.h>
69#include <net/checksum.h>
70
71struct cgroup;
72struct cgroup_subsys;
73#ifdef CONFIG_NET
74int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
75void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
76#else
77static inline
78int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
79{
80 return 0;
81}
82static inline
83void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
84{
85}
86#endif
87
88
89
90
91
92
93
94#define SOCK_DEBUGGING
95#ifdef SOCK_DEBUGGING
96#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
97 printk(KERN_DEBUG msg); } while (0)
98#else
99
100static inline __printf(2, 3)
101void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
102{
103}
104#endif
105
106
107
108
109
110typedef struct {
111 spinlock_t slock;
112 int owned;
113 wait_queue_head_t wq;
114
115
116
117
118
119
120#ifdef CONFIG_DEBUG_LOCK_ALLOC
121 struct lockdep_map dep_map;
122#endif
123} socket_lock_t;
124
125struct sock;
126struct proto;
127struct net;
128
129typedef __u32 __bitwise __portpair;
130typedef __u64 __bitwise __addrpair;
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157struct sock_common {
158
159
160
161 union {
162 __addrpair skc_addrpair;
163 struct {
164 __be32 skc_daddr;
165 __be32 skc_rcv_saddr;
166 };
167 };
168 union {
169 unsigned int skc_hash;
170 __u16 skc_u16hashes[2];
171 };
172
173 union {
174 __portpair skc_portpair;
175 struct {
176 __be16 skc_dport;
177 __u16 skc_num;
178 };
179 };
180
181 unsigned short skc_family;
182 volatile unsigned char skc_state;
183 unsigned char skc_reuse:4;
184 unsigned char skc_reuseport:4;
185 int skc_bound_dev_if;
186 union {
187 struct hlist_node skc_bind_node;
188 struct hlist_nulls_node skc_portaddr_node;
189 };
190 struct proto *skc_prot;
191#ifdef CONFIG_NET_NS
192 struct net *skc_net;
193#endif
194
195
196
197
198
199 int skc_dontcopy_begin[0];
200
201 union {
202 struct hlist_node skc_node;
203 struct hlist_nulls_node skc_nulls_node;
204 };
205 int skc_tx_queue_mapping;
206 atomic_t skc_refcnt;
207
208 int skc_dontcopy_end[0];
209
210};
211
212struct cg_proto;
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285struct sock {
286
287
288
289
290 struct sock_common __sk_common;
291#define sk_node __sk_common.skc_node
292#define sk_nulls_node __sk_common.skc_nulls_node
293#define sk_refcnt __sk_common.skc_refcnt
294#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
295
296#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
297#define sk_dontcopy_end __sk_common.skc_dontcopy_end
298#define sk_hash __sk_common.skc_hash
299#define sk_family __sk_common.skc_family
300#define sk_state __sk_common.skc_state
301#define sk_reuse __sk_common.skc_reuse
302#define sk_reuseport __sk_common.skc_reuseport
303#define sk_bound_dev_if __sk_common.skc_bound_dev_if
304#define sk_bind_node __sk_common.skc_bind_node
305#define sk_prot __sk_common.skc_prot
306#define sk_net __sk_common.skc_net
307 socket_lock_t sk_lock;
308 struct sk_buff_head sk_receive_queue;
309
310
311
312
313
314
315
316
317 struct {
318 atomic_t rmem_alloc;
319 int len;
320 struct sk_buff *head;
321 struct sk_buff *tail;
322 } sk_backlog;
323#define sk_rmem_alloc sk_backlog.rmem_alloc
324 int sk_forward_alloc;
325#ifdef CONFIG_RPS
326 __u32 sk_rxhash;
327#endif
328 atomic_t sk_drops;
329 int sk_rcvbuf;
330
331 struct sk_filter __rcu *sk_filter;
332 struct socket_wq __rcu *sk_wq;
333
334#ifdef CONFIG_NET_DMA
335 struct sk_buff_head sk_async_wait_queue;
336#endif
337
338#ifdef CONFIG_XFRM
339 struct xfrm_policy *sk_policy[2];
340#endif
341 unsigned long sk_flags;
342 struct dst_entry *sk_rx_dst;
343 struct dst_entry __rcu *sk_dst_cache;
344 spinlock_t sk_dst_lock;
345 atomic_t sk_wmem_alloc;
346 atomic_t sk_omem_alloc;
347 int sk_sndbuf;
348 struct sk_buff_head sk_write_queue;
349 kmemcheck_bitfield_begin(flags);
350 unsigned int sk_shutdown : 2,
351 sk_no_check : 2,
352 sk_userlocks : 4,
353 sk_protocol : 8,
354 sk_type : 16;
355 kmemcheck_bitfield_end(flags);
356 int sk_wmem_queued;
357 gfp_t sk_allocation;
358 netdev_features_t sk_route_caps;
359 netdev_features_t sk_route_nocaps;
360 int sk_gso_type;
361 unsigned int sk_gso_max_size;
362 u16 sk_gso_max_segs;
363 int sk_rcvlowat;
364 unsigned long sk_lingertime;
365 struct sk_buff_head sk_error_queue;
366 struct proto *sk_prot_creator;
367 rwlock_t sk_callback_lock;
368 int sk_err,
369 sk_err_soft;
370 unsigned short sk_ack_backlog;
371 unsigned short sk_max_ack_backlog;
372 __u32 sk_priority;
373#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
374 __u32 sk_cgrp_prioidx;
375#endif
376 struct pid *sk_peer_pid;
377 const struct cred *sk_peer_cred;
378 long sk_rcvtimeo;
379 long sk_sndtimeo;
380 void *sk_protinfo;
381 struct timer_list sk_timer;
382 ktime_t sk_stamp;
383 struct socket *sk_socket;
384 void *sk_user_data;
385 struct page_frag sk_frag;
386 struct sk_buff *sk_send_head;
387 __s32 sk_peek_off;
388 int sk_write_pending;
389#ifdef CONFIG_SECURITY
390 void *sk_security;
391#endif
392 __u32 sk_mark;
393 u32 sk_classid;
394 struct cg_proto *sk_cgrp;
395 void (*sk_state_change)(struct sock *sk);
396 void (*sk_data_ready)(struct sock *sk, int bytes);
397 void (*sk_write_space)(struct sock *sk);
398 void (*sk_error_report)(struct sock *sk);
399 int (*sk_backlog_rcv)(struct sock *sk,
400 struct sk_buff *skb);
401 void (*sk_destruct)(struct sock *sk);
402};
403
404
405
406
407
408
409
410
411#define SK_NO_REUSE 0
412#define SK_CAN_REUSE 1
413#define SK_FORCE_REUSE 2
414
415static inline int sk_peek_offset(struct sock *sk, int flags)
416{
417 if ((flags & MSG_PEEK) && (sk->sk_peek_off >= 0))
418 return sk->sk_peek_off;
419 else
420 return 0;
421}
422
423static inline void sk_peek_offset_bwd(struct sock *sk, int val)
424{
425 if (sk->sk_peek_off >= 0) {
426 if (sk->sk_peek_off >= val)
427 sk->sk_peek_off -= val;
428 else
429 sk->sk_peek_off = 0;
430 }
431}
432
433static inline void sk_peek_offset_fwd(struct sock *sk, int val)
434{
435 if (sk->sk_peek_off >= 0)
436 sk->sk_peek_off += val;
437}
438
439
440
441
442static inline struct sock *sk_entry(const struct hlist_node *node)
443{
444 return hlist_entry(node, struct sock, sk_node);
445}
446
447static inline struct sock *__sk_head(const struct hlist_head *head)
448{
449 return hlist_entry(head->first, struct sock, sk_node);
450}
451
452static inline struct sock *sk_head(const struct hlist_head *head)
453{
454 return hlist_empty(head) ? NULL : __sk_head(head);
455}
456
457static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
458{
459 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
460}
461
462static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
463{
464 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
465}
466
467static inline struct sock *sk_next(const struct sock *sk)
468{
469 return sk->sk_node.next ?
470 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
471}
472
473static inline struct sock *sk_nulls_next(const struct sock *sk)
474{
475 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
476 hlist_nulls_entry(sk->sk_nulls_node.next,
477 struct sock, sk_nulls_node) :
478 NULL;
479}
480
481static inline bool sk_unhashed(const struct sock *sk)
482{
483 return hlist_unhashed(&sk->sk_node);
484}
485
486static inline bool sk_hashed(const struct sock *sk)
487{
488 return !sk_unhashed(sk);
489}
490
491static inline void sk_node_init(struct hlist_node *node)
492{
493 node->pprev = NULL;
494}
495
496static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
497{
498 node->pprev = NULL;
499}
500
501static inline void __sk_del_node(struct sock *sk)
502{
503 __hlist_del(&sk->sk_node);
504}
505
506
507static inline bool __sk_del_node_init(struct sock *sk)
508{
509 if (sk_hashed(sk)) {
510 __sk_del_node(sk);
511 sk_node_init(&sk->sk_node);
512 return true;
513 }
514 return false;
515}
516
517
518
519
520
521
522
523static inline void sock_hold(struct sock *sk)
524{
525 atomic_inc(&sk->sk_refcnt);
526}
527
528
529
530
531static inline void __sock_put(struct sock *sk)
532{
533 atomic_dec(&sk->sk_refcnt);
534}
535
536static inline bool sk_del_node_init(struct sock *sk)
537{
538 bool rc = __sk_del_node_init(sk);
539
540 if (rc) {
541
542 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
543 __sock_put(sk);
544 }
545 return rc;
546}
547#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
548
549static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
550{
551 if (sk_hashed(sk)) {
552 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
553 return true;
554 }
555 return false;
556}
557
558static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
559{
560 bool rc = __sk_nulls_del_node_init_rcu(sk);
561
562 if (rc) {
563
564 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
565 __sock_put(sk);
566 }
567 return rc;
568}
569
570static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
571{
572 hlist_add_head(&sk->sk_node, list);
573}
574
575static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
576{
577 sock_hold(sk);
578 __sk_add_node(sk, list);
579}
580
581static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
582{
583 sock_hold(sk);
584 hlist_add_head_rcu(&sk->sk_node, list);
585}
586
587static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
588{
589 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
590}
591
592static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
593{
594 sock_hold(sk);
595 __sk_nulls_add_node_rcu(sk, list);
596}
597
598static inline void __sk_del_bind_node(struct sock *sk)
599{
600 __hlist_del(&sk->sk_bind_node);
601}
602
603static inline void sk_add_bind_node(struct sock *sk,
604 struct hlist_head *list)
605{
606 hlist_add_head(&sk->sk_bind_node, list);
607}
608
609#define sk_for_each(__sk, list) \
610 hlist_for_each_entry(__sk, list, sk_node)
611#define sk_for_each_rcu(__sk, list) \
612 hlist_for_each_entry_rcu(__sk, list, sk_node)
613#define sk_nulls_for_each(__sk, node, list) \
614 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
615#define sk_nulls_for_each_rcu(__sk, node, list) \
616 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
617#define sk_for_each_from(__sk) \
618 hlist_for_each_entry_from(__sk, sk_node)
619#define sk_nulls_for_each_from(__sk, node) \
620 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
621 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
622#define sk_for_each_safe(__sk, tmp, list) \
623 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
624#define sk_for_each_bound(__sk, list) \
625 hlist_for_each_entry(__sk, list, sk_bind_node)
626
627static inline struct user_namespace *sk_user_ns(struct sock *sk)
628{
629
630
631
632
633 return sk->sk_socket->file->f_cred->user_ns;
634}
635
636
637enum sock_flags {
638 SOCK_DEAD,
639 SOCK_DONE,
640 SOCK_URGINLINE,
641 SOCK_KEEPOPEN,
642 SOCK_LINGER,
643 SOCK_DESTROY,
644 SOCK_BROADCAST,
645 SOCK_TIMESTAMP,
646 SOCK_ZAPPED,
647 SOCK_USE_WRITE_QUEUE,
648 SOCK_DBG,
649 SOCK_RCVTSTAMP,
650 SOCK_RCVTSTAMPNS,
651 SOCK_LOCALROUTE,
652 SOCK_QUEUE_SHRUNK,
653 SOCK_MEMALLOC,
654 SOCK_TIMESTAMPING_TX_HARDWARE,
655 SOCK_TIMESTAMPING_TX_SOFTWARE,
656 SOCK_TIMESTAMPING_RX_HARDWARE,
657 SOCK_TIMESTAMPING_RX_SOFTWARE,
658 SOCK_TIMESTAMPING_SOFTWARE,
659 SOCK_TIMESTAMPING_RAW_HARDWARE,
660 SOCK_TIMESTAMPING_SYS_HARDWARE,
661 SOCK_FASYNC,
662 SOCK_RXQ_OVFL,
663 SOCK_ZEROCOPY,
664 SOCK_WIFI_STATUS,
665 SOCK_NOFCS,
666
667
668
669 SOCK_FILTER_LOCKED,
670};
671
672static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
673{
674 nsk->sk_flags = osk->sk_flags;
675}
676
677static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
678{
679 __set_bit(flag, &sk->sk_flags);
680}
681
682static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
683{
684 __clear_bit(flag, &sk->sk_flags);
685}
686
687static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
688{
689 return test_bit(flag, &sk->sk_flags);
690}
691
692#ifdef CONFIG_NET
693extern struct static_key memalloc_socks;
694static inline int sk_memalloc_socks(void)
695{
696 return static_key_false(&memalloc_socks);
697}
698#else
699
700static inline int sk_memalloc_socks(void)
701{
702 return 0;
703}
704
705#endif
706
707static inline gfp_t sk_gfp_atomic(struct sock *sk, gfp_t gfp_mask)
708{
709 return GFP_ATOMIC | (sk->sk_allocation & __GFP_MEMALLOC);
710}
711
712static inline void sk_acceptq_removed(struct sock *sk)
713{
714 sk->sk_ack_backlog--;
715}
716
717static inline void sk_acceptq_added(struct sock *sk)
718{
719 sk->sk_ack_backlog++;
720}
721
722static inline bool sk_acceptq_is_full(const struct sock *sk)
723{
724 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
725}
726
727
728
729
730static inline int sk_stream_min_wspace(const struct sock *sk)
731{
732 return sk->sk_wmem_queued >> 1;
733}
734
735static inline int sk_stream_wspace(const struct sock *sk)
736{
737 return sk->sk_sndbuf - sk->sk_wmem_queued;
738}
739
740extern void sk_stream_write_space(struct sock *sk);
741
742static inline bool sk_stream_memory_free(const struct sock *sk)
743{
744 return sk->sk_wmem_queued < sk->sk_sndbuf;
745}
746
747
748static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
749{
750
751 skb_dst_force(skb);
752
753 if (!sk->sk_backlog.tail)
754 sk->sk_backlog.head = skb;
755 else
756 sk->sk_backlog.tail->next = skb;
757
758 sk->sk_backlog.tail = skb;
759 skb->next = NULL;
760}
761
762
763
764
765
766
767static inline bool sk_rcvqueues_full(const struct sock *sk, const struct sk_buff *skb,
768 unsigned int limit)
769{
770 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
771
772 return qsize > limit;
773}
774
775
776static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
777 unsigned int limit)
778{
779 if (sk_rcvqueues_full(sk, skb, limit))
780 return -ENOBUFS;
781
782 __sk_add_backlog(sk, skb);
783 sk->sk_backlog.len += skb->truesize;
784 return 0;
785}
786
787extern int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
788
789static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
790{
791 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
792 return __sk_backlog_rcv(sk, skb);
793
794 return sk->sk_backlog_rcv(sk, skb);
795}
796
797static inline void sock_rps_record_flow(const struct sock *sk)
798{
799#ifdef CONFIG_RPS
800 struct rps_sock_flow_table *sock_flow_table;
801
802 rcu_read_lock();
803 sock_flow_table = rcu_dereference(rps_sock_flow_table);
804 rps_record_sock_flow(sock_flow_table, sk->sk_rxhash);
805 rcu_read_unlock();
806#endif
807}
808
809static inline void sock_rps_reset_flow(const struct sock *sk)
810{
811#ifdef CONFIG_RPS
812 struct rps_sock_flow_table *sock_flow_table;
813
814 rcu_read_lock();
815 sock_flow_table = rcu_dereference(rps_sock_flow_table);
816 rps_reset_sock_flow(sock_flow_table, sk->sk_rxhash);
817 rcu_read_unlock();
818#endif
819}
820
821static inline void sock_rps_save_rxhash(struct sock *sk,
822 const struct sk_buff *skb)
823{
824#ifdef CONFIG_RPS
825 if (unlikely(sk->sk_rxhash != skb->rxhash)) {
826 sock_rps_reset_flow(sk);
827 sk->sk_rxhash = skb->rxhash;
828 }
829#endif
830}
831
832static inline void sock_rps_reset_rxhash(struct sock *sk)
833{
834#ifdef CONFIG_RPS
835 sock_rps_reset_flow(sk);
836 sk->sk_rxhash = 0;
837#endif
838}
839
840#define sk_wait_event(__sk, __timeo, __condition) \
841 ({ int __rc; \
842 release_sock(__sk); \
843 __rc = __condition; \
844 if (!__rc) { \
845 *(__timeo) = schedule_timeout(*(__timeo)); \
846 } \
847 lock_sock(__sk); \
848 __rc = __condition; \
849 __rc; \
850 })
851
852extern int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
853extern int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
854extern void sk_stream_wait_close(struct sock *sk, long timeo_p);
855extern int sk_stream_error(struct sock *sk, int flags, int err);
856extern void sk_stream_kill_queues(struct sock *sk);
857extern void sk_set_memalloc(struct sock *sk);
858extern void sk_clear_memalloc(struct sock *sk);
859
860extern int sk_wait_data(struct sock *sk, long *timeo);
861
862struct request_sock_ops;
863struct timewait_sock_ops;
864struct inet_hashinfo;
865struct raw_hashinfo;
866struct module;
867
868
869
870
871
872static inline void sk_prot_clear_nulls(struct sock *sk, int size)
873{
874 if (offsetof(struct sock, sk_node.next) != 0)
875 memset(sk, 0, offsetof(struct sock, sk_node.next));
876 memset(&sk->sk_node.pprev, 0,
877 size - offsetof(struct sock, sk_node.pprev));
878}
879
880
881
882
883
884struct proto {
885 void (*close)(struct sock *sk,
886 long timeout);
887 int (*connect)(struct sock *sk,
888 struct sockaddr *uaddr,
889 int addr_len);
890 int (*disconnect)(struct sock *sk, int flags);
891
892 struct sock * (*accept)(struct sock *sk, int flags, int *err);
893
894 int (*ioctl)(struct sock *sk, int cmd,
895 unsigned long arg);
896 int (*init)(struct sock *sk);
897 void (*destroy)(struct sock *sk);
898 void (*shutdown)(struct sock *sk, int how);
899 int (*setsockopt)(struct sock *sk, int level,
900 int optname, char __user *optval,
901 unsigned int optlen);
902 int (*getsockopt)(struct sock *sk, int level,
903 int optname, char __user *optval,
904 int __user *option);
905#ifdef CONFIG_COMPAT
906 int (*compat_setsockopt)(struct sock *sk,
907 int level,
908 int optname, char __user *optval,
909 unsigned int optlen);
910 int (*compat_getsockopt)(struct sock *sk,
911 int level,
912 int optname, char __user *optval,
913 int __user *option);
914 int (*compat_ioctl)(struct sock *sk,
915 unsigned int cmd, unsigned long arg);
916#endif
917 int (*sendmsg)(struct kiocb *iocb, struct sock *sk,
918 struct msghdr *msg, size_t len);
919 int (*recvmsg)(struct kiocb *iocb, struct sock *sk,
920 struct msghdr *msg,
921 size_t len, int noblock, int flags,
922 int *addr_len);
923 int (*sendpage)(struct sock *sk, struct page *page,
924 int offset, size_t size, int flags);
925 int (*bind)(struct sock *sk,
926 struct sockaddr *uaddr, int addr_len);
927
928 int (*backlog_rcv) (struct sock *sk,
929 struct sk_buff *skb);
930
931 void (*release_cb)(struct sock *sk);
932 void (*mtu_reduced)(struct sock *sk);
933
934
935 void (*hash)(struct sock *sk);
936 void (*unhash)(struct sock *sk);
937 void (*rehash)(struct sock *sk);
938 int (*get_port)(struct sock *sk, unsigned short snum);
939 void (*clear_sk)(struct sock *sk, int size);
940
941
942#ifdef CONFIG_PROC_FS
943 unsigned int inuse_idx;
944#endif
945
946
947 void (*enter_memory_pressure)(struct sock *sk);
948 atomic_long_t *memory_allocated;
949 struct percpu_counter *sockets_allocated;
950
951
952
953
954
955
956 int *memory_pressure;
957 long *sysctl_mem;
958 int *sysctl_wmem;
959 int *sysctl_rmem;
960 int max_header;
961 bool no_autobind;
962
963 struct kmem_cache *slab;
964 unsigned int obj_size;
965 int slab_flags;
966
967 struct percpu_counter *orphan_count;
968
969 struct request_sock_ops *rsk_prot;
970 struct timewait_sock_ops *twsk_prot;
971
972 union {
973 struct inet_hashinfo *hashinfo;
974 struct udp_table *udp_table;
975 struct raw_hashinfo *raw_hash;
976 } h;
977
978 struct module *owner;
979
980 char name[32];
981
982 struct list_head node;
983#ifdef SOCK_REFCNT_DEBUG
984 atomic_t socks;
985#endif
986#ifdef CONFIG_MEMCG_KMEM
987
988
989
990
991
992
993 int (*init_cgroup)(struct mem_cgroup *memcg,
994 struct cgroup_subsys *ss);
995 void (*destroy_cgroup)(struct mem_cgroup *memcg);
996 struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
997#endif
998};
999
1000
1001
1002
1003enum cg_proto_flags {
1004
1005 MEMCG_SOCK_ACTIVE,
1006
1007 MEMCG_SOCK_ACTIVATED,
1008};
1009
1010struct cg_proto {
1011 void (*enter_memory_pressure)(struct sock *sk);
1012 struct res_counter *memory_allocated;
1013 struct percpu_counter *sockets_allocated;
1014 int *memory_pressure;
1015 long *sysctl_mem;
1016 unsigned long flags;
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026 struct mem_cgroup *memcg;
1027};
1028
1029extern int proto_register(struct proto *prot, int alloc_slab);
1030extern void proto_unregister(struct proto *prot);
1031
1032static inline bool memcg_proto_active(struct cg_proto *cg_proto)
1033{
1034 return test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
1035}
1036
1037static inline bool memcg_proto_activated(struct cg_proto *cg_proto)
1038{
1039 return test_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags);
1040}
1041
1042#ifdef SOCK_REFCNT_DEBUG
1043static inline void sk_refcnt_debug_inc(struct sock *sk)
1044{
1045 atomic_inc(&sk->sk_prot->socks);
1046}
1047
1048static inline void sk_refcnt_debug_dec(struct sock *sk)
1049{
1050 atomic_dec(&sk->sk_prot->socks);
1051 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1052 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1053}
1054
1055static inline void sk_refcnt_debug_release(const struct sock *sk)
1056{
1057 if (atomic_read(&sk->sk_refcnt) != 1)
1058 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1059 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1060}
1061#else
1062#define sk_refcnt_debug_inc(sk) do { } while (0)
1063#define sk_refcnt_debug_dec(sk) do { } while (0)
1064#define sk_refcnt_debug_release(sk) do { } while (0)
1065#endif
1066
1067#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_NET)
1068extern struct static_key memcg_socket_limit_enabled;
1069static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1070 struct cg_proto *cg_proto)
1071{
1072 return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
1073}
1074#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
1075#else
1076#define mem_cgroup_sockets_enabled 0
1077static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1078 struct cg_proto *cg_proto)
1079{
1080 return NULL;
1081}
1082#endif
1083
1084
1085static inline bool sk_has_memory_pressure(const struct sock *sk)
1086{
1087 return sk->sk_prot->memory_pressure != NULL;
1088}
1089
1090static inline bool sk_under_memory_pressure(const struct sock *sk)
1091{
1092 if (!sk->sk_prot->memory_pressure)
1093 return false;
1094
1095 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1096 return !!*sk->sk_cgrp->memory_pressure;
1097
1098 return !!*sk->sk_prot->memory_pressure;
1099}
1100
1101static inline void sk_leave_memory_pressure(struct sock *sk)
1102{
1103 int *memory_pressure = sk->sk_prot->memory_pressure;
1104
1105 if (!memory_pressure)
1106 return;
1107
1108 if (*memory_pressure)
1109 *memory_pressure = 0;
1110
1111 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1112 struct cg_proto *cg_proto = sk->sk_cgrp;
1113 struct proto *prot = sk->sk_prot;
1114
1115 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1116 if (*cg_proto->memory_pressure)
1117 *cg_proto->memory_pressure = 0;
1118 }
1119
1120}
1121
1122static inline void sk_enter_memory_pressure(struct sock *sk)
1123{
1124 if (!sk->sk_prot->enter_memory_pressure)
1125 return;
1126
1127 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1128 struct cg_proto *cg_proto = sk->sk_cgrp;
1129 struct proto *prot = sk->sk_prot;
1130
1131 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1132 cg_proto->enter_memory_pressure(sk);
1133 }
1134
1135 sk->sk_prot->enter_memory_pressure(sk);
1136}
1137
1138static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1139{
1140 long *prot = sk->sk_prot->sysctl_mem;
1141 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1142 prot = sk->sk_cgrp->sysctl_mem;
1143 return prot[index];
1144}
1145
1146static inline void memcg_memory_allocated_add(struct cg_proto *prot,
1147 unsigned long amt,
1148 int *parent_status)
1149{
1150 struct res_counter *fail;
1151 int ret;
1152
1153 ret = res_counter_charge_nofail(prot->memory_allocated,
1154 amt << PAGE_SHIFT, &fail);
1155 if (ret < 0)
1156 *parent_status = OVER_LIMIT;
1157}
1158
1159static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
1160 unsigned long amt)
1161{
1162 res_counter_uncharge(prot->memory_allocated, amt << PAGE_SHIFT);
1163}
1164
1165static inline u64 memcg_memory_allocated_read(struct cg_proto *prot)
1166{
1167 u64 ret;
1168 ret = res_counter_read_u64(prot->memory_allocated, RES_USAGE);
1169 return ret >> PAGE_SHIFT;
1170}
1171
1172static inline long
1173sk_memory_allocated(const struct sock *sk)
1174{
1175 struct proto *prot = sk->sk_prot;
1176 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1177 return memcg_memory_allocated_read(sk->sk_cgrp);
1178
1179 return atomic_long_read(prot->memory_allocated);
1180}
1181
1182static inline long
1183sk_memory_allocated_add(struct sock *sk, int amt, int *parent_status)
1184{
1185 struct proto *prot = sk->sk_prot;
1186
1187 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1188 memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
1189
1190 atomic_long_add_return(amt, prot->memory_allocated);
1191 return memcg_memory_allocated_read(sk->sk_cgrp);
1192 }
1193
1194 return atomic_long_add_return(amt, prot->memory_allocated);
1195}
1196
1197static inline void
1198sk_memory_allocated_sub(struct sock *sk, int amt)
1199{
1200 struct proto *prot = sk->sk_prot;
1201
1202 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1203 memcg_memory_allocated_sub(sk->sk_cgrp, amt);
1204
1205 atomic_long_sub(amt, prot->memory_allocated);
1206}
1207
1208static inline void sk_sockets_allocated_dec(struct sock *sk)
1209{
1210 struct proto *prot = sk->sk_prot;
1211
1212 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1213 struct cg_proto *cg_proto = sk->sk_cgrp;
1214
1215 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1216 percpu_counter_dec(cg_proto->sockets_allocated);
1217 }
1218
1219 percpu_counter_dec(prot->sockets_allocated);
1220}
1221
1222static inline void sk_sockets_allocated_inc(struct sock *sk)
1223{
1224 struct proto *prot = sk->sk_prot;
1225
1226 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1227 struct cg_proto *cg_proto = sk->sk_cgrp;
1228
1229 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1230 percpu_counter_inc(cg_proto->sockets_allocated);
1231 }
1232
1233 percpu_counter_inc(prot->sockets_allocated);
1234}
1235
1236static inline int
1237sk_sockets_allocated_read_positive(struct sock *sk)
1238{
1239 struct proto *prot = sk->sk_prot;
1240
1241 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1242 return percpu_counter_read_positive(sk->sk_cgrp->sockets_allocated);
1243
1244 return percpu_counter_read_positive(prot->sockets_allocated);
1245}
1246
1247static inline int
1248proto_sockets_allocated_sum_positive(struct proto *prot)
1249{
1250 return percpu_counter_sum_positive(prot->sockets_allocated);
1251}
1252
1253static inline long
1254proto_memory_allocated(struct proto *prot)
1255{
1256 return atomic_long_read(prot->memory_allocated);
1257}
1258
1259static inline bool
1260proto_memory_pressure(struct proto *prot)
1261{
1262 if (!prot->memory_pressure)
1263 return false;
1264 return !!*prot->memory_pressure;
1265}
1266
1267
1268#ifdef CONFIG_PROC_FS
1269
1270extern void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1271extern int sock_prot_inuse_get(struct net *net, struct proto *proto);
1272#else
1273static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1274 int inc)
1275{
1276}
1277#endif
1278
1279
1280
1281
1282
1283static inline void __sk_prot_rehash(struct sock *sk)
1284{
1285 sk->sk_prot->unhash(sk);
1286 sk->sk_prot->hash(sk);
1287}
1288
1289void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1290
1291
1292#define SOCK_DESTROY_TIME (10*HZ)
1293
1294
1295#define PROT_SOCK 1024
1296
1297#define SHUTDOWN_MASK 3
1298#define RCV_SHUTDOWN 1
1299#define SEND_SHUTDOWN 2
1300
1301#define SOCK_SNDBUF_LOCK 1
1302#define SOCK_RCVBUF_LOCK 2
1303#define SOCK_BINDADDR_LOCK 4
1304#define SOCK_BINDPORT_LOCK 8
1305
1306
1307struct sock_iocb {
1308 struct list_head list;
1309
1310 int flags;
1311 int size;
1312 struct socket *sock;
1313 struct sock *sk;
1314 struct scm_cookie *scm;
1315 struct msghdr *msg, async_msg;
1316 struct kiocb *kiocb;
1317};
1318
1319static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
1320{
1321 return (struct sock_iocb *)iocb->private;
1322}
1323
1324static inline struct kiocb *siocb_to_kiocb(struct sock_iocb *si)
1325{
1326 return si->kiocb;
1327}
1328
1329struct socket_alloc {
1330 struct socket socket;
1331 struct inode vfs_inode;
1332};
1333
1334static inline struct socket *SOCKET_I(struct inode *inode)
1335{
1336 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1337}
1338
1339static inline struct inode *SOCK_INODE(struct socket *socket)
1340{
1341 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1342}
1343
1344
1345
1346
1347extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
1348extern void __sk_mem_reclaim(struct sock *sk);
1349
1350#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1351#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1352#define SK_MEM_SEND 0
1353#define SK_MEM_RECV 1
1354
1355static inline int sk_mem_pages(int amt)
1356{
1357 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1358}
1359
1360static inline bool sk_has_account(struct sock *sk)
1361{
1362
1363 return !!sk->sk_prot->memory_allocated;
1364}
1365
1366static inline bool sk_wmem_schedule(struct sock *sk, int size)
1367{
1368 if (!sk_has_account(sk))
1369 return true;
1370 return size <= sk->sk_forward_alloc ||
1371 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1372}
1373
1374static inline bool
1375sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1376{
1377 if (!sk_has_account(sk))
1378 return true;
1379 return size<= sk->sk_forward_alloc ||
1380 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1381 skb_pfmemalloc(skb);
1382}
1383
1384static inline void sk_mem_reclaim(struct sock *sk)
1385{
1386 if (!sk_has_account(sk))
1387 return;
1388 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1389 __sk_mem_reclaim(sk);
1390}
1391
1392static inline void sk_mem_reclaim_partial(struct sock *sk)
1393{
1394 if (!sk_has_account(sk))
1395 return;
1396 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1397 __sk_mem_reclaim(sk);
1398}
1399
1400static inline void sk_mem_charge(struct sock *sk, int size)
1401{
1402 if (!sk_has_account(sk))
1403 return;
1404 sk->sk_forward_alloc -= size;
1405}
1406
1407static inline void sk_mem_uncharge(struct sock *sk, int size)
1408{
1409 if (!sk_has_account(sk))
1410 return;
1411 sk->sk_forward_alloc += size;
1412}
1413
1414static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1415{
1416 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1417 sk->sk_wmem_queued -= skb->truesize;
1418 sk_mem_uncharge(sk, skb->truesize);
1419 __kfree_skb(skb);
1420}
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1436
1437
1438
1439
1440
1441
1442
1443
1444#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1445do { \
1446 sk->sk_lock.owned = 0; \
1447 init_waitqueue_head(&sk->sk_lock.wq); \
1448 spin_lock_init(&(sk)->sk_lock.slock); \
1449 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1450 sizeof((sk)->sk_lock)); \
1451 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1452 (skey), (sname)); \
1453 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1454} while (0)
1455
1456extern void lock_sock_nested(struct sock *sk, int subclass);
1457
1458static inline void lock_sock(struct sock *sk)
1459{
1460 lock_sock_nested(sk, 0);
1461}
1462
1463extern void release_sock(struct sock *sk);
1464
1465
1466#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1467#define bh_lock_sock_nested(__sk) \
1468 spin_lock_nested(&((__sk)->sk_lock.slock), \
1469 SINGLE_DEPTH_NESTING)
1470#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1471
1472extern bool lock_sock_fast(struct sock *sk);
1473
1474
1475
1476
1477
1478
1479
1480
1481static inline void unlock_sock_fast(struct sock *sk, bool slow)
1482{
1483 if (slow)
1484 release_sock(sk);
1485 else
1486 spin_unlock_bh(&sk->sk_lock.slock);
1487}
1488
1489
1490extern struct sock *sk_alloc(struct net *net, int family,
1491 gfp_t priority,
1492 struct proto *prot);
1493extern void sk_free(struct sock *sk);
1494extern void sk_release_kernel(struct sock *sk);
1495extern struct sock *sk_clone_lock(const struct sock *sk,
1496 const gfp_t priority);
1497
1498extern struct sk_buff *sock_wmalloc(struct sock *sk,
1499 unsigned long size, int force,
1500 gfp_t priority);
1501extern struct sk_buff *sock_rmalloc(struct sock *sk,
1502 unsigned long size, int force,
1503 gfp_t priority);
1504extern void sock_wfree(struct sk_buff *skb);
1505extern void sock_rfree(struct sk_buff *skb);
1506extern void sock_edemux(struct sk_buff *skb);
1507
1508extern int sock_setsockopt(struct socket *sock, int level,
1509 int op, char __user *optval,
1510 unsigned int optlen);
1511
1512extern int sock_getsockopt(struct socket *sock, int level,
1513 int op, char __user *optval,
1514 int __user *optlen);
1515extern struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1516 unsigned long size,
1517 int noblock,
1518 int *errcode);
1519extern struct sk_buff *sock_alloc_send_pskb(struct sock *sk,
1520 unsigned long header_len,
1521 unsigned long data_len,
1522 int noblock,
1523 int *errcode);
1524extern void *sock_kmalloc(struct sock *sk, int size,
1525 gfp_t priority);
1526extern void sock_kfree_s(struct sock *sk, void *mem, int size);
1527extern void sk_send_sigurg(struct sock *sk);
1528
1529
1530
1531
1532
1533extern int sock_no_bind(struct socket *,
1534 struct sockaddr *, int);
1535extern int sock_no_connect(struct socket *,
1536 struct sockaddr *, int, int);
1537extern int sock_no_socketpair(struct socket *,
1538 struct socket *);
1539extern int sock_no_accept(struct socket *,
1540 struct socket *, int);
1541extern int sock_no_getname(struct socket *,
1542 struct sockaddr *, int *, int);
1543extern unsigned int sock_no_poll(struct file *, struct socket *,
1544 struct poll_table_struct *);
1545extern int sock_no_ioctl(struct socket *, unsigned int,
1546 unsigned long);
1547extern int sock_no_listen(struct socket *, int);
1548extern int sock_no_shutdown(struct socket *, int);
1549extern int sock_no_getsockopt(struct socket *, int , int,
1550 char __user *, int __user *);
1551extern int sock_no_setsockopt(struct socket *, int, int,
1552 char __user *, unsigned int);
1553extern int sock_no_sendmsg(struct kiocb *, struct socket *,
1554 struct msghdr *, size_t);
1555extern int sock_no_recvmsg(struct kiocb *, struct socket *,
1556 struct msghdr *, size_t, int);
1557extern int sock_no_mmap(struct file *file,
1558 struct socket *sock,
1559 struct vm_area_struct *vma);
1560extern ssize_t sock_no_sendpage(struct socket *sock,
1561 struct page *page,
1562 int offset, size_t size,
1563 int flags);
1564
1565
1566
1567
1568
1569extern int sock_common_getsockopt(struct socket *sock, int level, int optname,
1570 char __user *optval, int __user *optlen);
1571extern int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
1572 struct msghdr *msg, size_t size, int flags);
1573extern int sock_common_setsockopt(struct socket *sock, int level, int optname,
1574 char __user *optval, unsigned int optlen);
1575extern int compat_sock_common_getsockopt(struct socket *sock, int level,
1576 int optname, char __user *optval, int __user *optlen);
1577extern int compat_sock_common_setsockopt(struct socket *sock, int level,
1578 int optname, char __user *optval, unsigned int optlen);
1579
1580extern void sk_common_release(struct sock *sk);
1581
1582
1583
1584
1585
1586
1587extern void sock_init_data(struct socket *sock, struct sock *sk);
1588
1589extern void sk_filter_release_rcu(struct rcu_head *rcu);
1590
1591
1592
1593
1594
1595
1596
1597
1598static inline void sk_filter_release(struct sk_filter *fp)
1599{
1600 if (atomic_dec_and_test(&fp->refcnt))
1601 call_rcu(&fp->rcu, sk_filter_release_rcu);
1602}
1603
1604static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1605{
1606 unsigned int size = sk_filter_len(fp);
1607
1608 atomic_sub(size, &sk->sk_omem_alloc);
1609 sk_filter_release(fp);
1610}
1611
1612static inline void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1613{
1614 atomic_inc(&fp->refcnt);
1615 atomic_add(sk_filter_len(fp), &sk->sk_omem_alloc);
1616}
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644static inline void sock_put(struct sock *sk)
1645{
1646 if (atomic_dec_and_test(&sk->sk_refcnt))
1647 sk_free(sk);
1648}
1649
1650extern int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1651 const int nested);
1652
1653static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1654{
1655 sk->sk_tx_queue_mapping = tx_queue;
1656}
1657
1658static inline void sk_tx_queue_clear(struct sock *sk)
1659{
1660 sk->sk_tx_queue_mapping = -1;
1661}
1662
1663static inline int sk_tx_queue_get(const struct sock *sk)
1664{
1665 return sk ? sk->sk_tx_queue_mapping : -1;
1666}
1667
1668static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1669{
1670 sk_tx_queue_clear(sk);
1671 sk->sk_socket = sock;
1672}
1673
1674static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1675{
1676 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1677 return &rcu_dereference_raw(sk->sk_wq)->wait;
1678}
1679
1680
1681
1682
1683
1684
1685
1686static inline void sock_orphan(struct sock *sk)
1687{
1688 write_lock_bh(&sk->sk_callback_lock);
1689 sock_set_flag(sk, SOCK_DEAD);
1690 sk_set_socket(sk, NULL);
1691 sk->sk_wq = NULL;
1692 write_unlock_bh(&sk->sk_callback_lock);
1693}
1694
1695static inline void sock_graft(struct sock *sk, struct socket *parent)
1696{
1697 write_lock_bh(&sk->sk_callback_lock);
1698 sk->sk_wq = parent->wq;
1699 parent->sk = sk;
1700 sk_set_socket(sk, parent);
1701 security_sock_graft(sk, parent);
1702 write_unlock_bh(&sk->sk_callback_lock);
1703}
1704
1705extern kuid_t sock_i_uid(struct sock *sk);
1706extern unsigned long sock_i_ino(struct sock *sk);
1707
1708static inline struct dst_entry *
1709__sk_dst_get(struct sock *sk)
1710{
1711 return rcu_dereference_check(sk->sk_dst_cache, sock_owned_by_user(sk) ||
1712 lockdep_is_held(&sk->sk_lock.slock));
1713}
1714
1715static inline struct dst_entry *
1716sk_dst_get(struct sock *sk)
1717{
1718 struct dst_entry *dst;
1719
1720 rcu_read_lock();
1721 dst = rcu_dereference(sk->sk_dst_cache);
1722 if (dst)
1723 dst_hold(dst);
1724 rcu_read_unlock();
1725 return dst;
1726}
1727
1728extern void sk_reset_txq(struct sock *sk);
1729
1730static inline void dst_negative_advice(struct sock *sk)
1731{
1732 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1733
1734 if (dst && dst->ops->negative_advice) {
1735 ndst = dst->ops->negative_advice(dst);
1736
1737 if (ndst != dst) {
1738 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1739 sk_reset_txq(sk);
1740 }
1741 }
1742}
1743
1744static inline void
1745__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1746{
1747 struct dst_entry *old_dst;
1748
1749 sk_tx_queue_clear(sk);
1750
1751
1752
1753
1754 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1755 rcu_assign_pointer(sk->sk_dst_cache, dst);
1756 dst_release(old_dst);
1757}
1758
1759static inline void
1760sk_dst_set(struct sock *sk, struct dst_entry *dst)
1761{
1762 spin_lock(&sk->sk_dst_lock);
1763 __sk_dst_set(sk, dst);
1764 spin_unlock(&sk->sk_dst_lock);
1765}
1766
1767static inline void
1768__sk_dst_reset(struct sock *sk)
1769{
1770 __sk_dst_set(sk, NULL);
1771}
1772
1773static inline void
1774sk_dst_reset(struct sock *sk)
1775{
1776 spin_lock(&sk->sk_dst_lock);
1777 __sk_dst_reset(sk);
1778 spin_unlock(&sk->sk_dst_lock);
1779}
1780
1781extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1782
1783extern struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1784
1785static inline bool sk_can_gso(const struct sock *sk)
1786{
1787 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1788}
1789
1790extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1791
1792static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1793{
1794 sk->sk_route_nocaps |= flags;
1795 sk->sk_route_caps &= ~flags;
1796}
1797
1798static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1799 char __user *from, char *to,
1800 int copy, int offset)
1801{
1802 if (skb->ip_summed == CHECKSUM_NONE) {
1803 int err = 0;
1804 __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
1805 if (err)
1806 return err;
1807 skb->csum = csum_block_add(skb->csum, csum, offset);
1808 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1809 if (!access_ok(VERIFY_READ, from, copy) ||
1810 __copy_from_user_nocache(to, from, copy))
1811 return -EFAULT;
1812 } else if (copy_from_user(to, from, copy))
1813 return -EFAULT;
1814
1815 return 0;
1816}
1817
1818static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1819 char __user *from, int copy)
1820{
1821 int err, offset = skb->len;
1822
1823 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1824 copy, offset);
1825 if (err)
1826 __skb_trim(skb, offset);
1827
1828 return err;
1829}
1830
1831static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
1832 struct sk_buff *skb,
1833 struct page *page,
1834 int off, int copy)
1835{
1836 int err;
1837
1838 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1839 copy, skb->len);
1840 if (err)
1841 return err;
1842
1843 skb->len += copy;
1844 skb->data_len += copy;
1845 skb->truesize += copy;
1846 sk->sk_wmem_queued += copy;
1847 sk_mem_charge(sk, copy);
1848 return 0;
1849}
1850
1851static inline int skb_copy_to_page(struct sock *sk, char __user *from,
1852 struct sk_buff *skb, struct page *page,
1853 int off, int copy)
1854{
1855 if (skb->ip_summed == CHECKSUM_NONE) {
1856 int err = 0;
1857 __wsum csum = csum_and_copy_from_user(from,
1858 page_address(page) + off,
1859 copy, 0, &err);
1860 if (err)
1861 return err;
1862 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1863 } else if (copy_from_user(page_address(page) + off, from, copy))
1864 return -EFAULT;
1865
1866 skb->len += copy;
1867 skb->data_len += copy;
1868 skb->truesize += copy;
1869 sk->sk_wmem_queued += copy;
1870 sk_mem_charge(sk, copy);
1871 return 0;
1872}
1873
1874
1875
1876
1877
1878
1879
1880static inline int sk_wmem_alloc_get(const struct sock *sk)
1881{
1882 return atomic_read(&sk->sk_wmem_alloc) - 1;
1883}
1884
1885
1886
1887
1888
1889
1890
1891static inline int sk_rmem_alloc_get(const struct sock *sk)
1892{
1893 return atomic_read(&sk->sk_rmem_alloc);
1894}
1895
1896
1897
1898
1899
1900
1901
1902static inline bool sk_has_allocations(const struct sock *sk)
1903{
1904 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1905}
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939static inline bool wq_has_sleeper(struct socket_wq *wq)
1940{
1941
1942
1943
1944
1945
1946 smp_mb();
1947 return wq && waitqueue_active(&wq->wait);
1948}
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958static inline void sock_poll_wait(struct file *filp,
1959 wait_queue_head_t *wait_address, poll_table *p)
1960{
1961 if (!poll_does_not_wait(p) && wait_address) {
1962 poll_wait(filp, wait_address, p);
1963
1964
1965
1966
1967
1968 smp_mb();
1969 }
1970}
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981static inline void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1982{
1983 skb_orphan(skb);
1984 skb->sk = sk;
1985 skb->destructor = sock_wfree;
1986
1987
1988
1989
1990
1991 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1992}
1993
1994static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
1995{
1996 skb_orphan(skb);
1997 skb->sk = sk;
1998 skb->destructor = sock_rfree;
1999 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2000 sk_mem_charge(sk, skb->truesize);
2001}
2002
2003extern void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2004 unsigned long expires);
2005
2006extern void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2007
2008extern int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2009
2010extern int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2011
2012
2013
2014
2015
2016static inline int sock_error(struct sock *sk)
2017{
2018 int err;
2019 if (likely(!sk->sk_err))
2020 return 0;
2021 err = xchg(&sk->sk_err, 0);
2022 return -err;
2023}
2024
2025static inline unsigned long sock_wspace(struct sock *sk)
2026{
2027 int amt = 0;
2028
2029 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2030 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2031 if (amt < 0)
2032 amt = 0;
2033 }
2034 return amt;
2035}
2036
2037static inline void sk_wake_async(struct sock *sk, int how, int band)
2038{
2039 if (sock_flag(sk, SOCK_FASYNC))
2040 sock_wake_async(sk->sk_socket, how, band);
2041}
2042
2043#define SOCK_MIN_SNDBUF 2048
2044
2045
2046
2047
2048#define SOCK_MIN_RCVBUF (2048 + sizeof(struct sk_buff))
2049
2050static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2051{
2052 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2053 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2054 sk->sk_sndbuf = max(sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2055 }
2056}
2057
2058struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
2059
2060
2061
2062
2063
2064
2065
2066
2067static inline struct page_frag *sk_page_frag(struct sock *sk)
2068{
2069 if (sk->sk_allocation & __GFP_WAIT)
2070 return ¤t->task_frag;
2071
2072 return &sk->sk_frag;
2073}
2074
2075extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2076
2077
2078
2079
2080static inline bool sock_writeable(const struct sock *sk)
2081{
2082 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2083}
2084
2085static inline gfp_t gfp_any(void)
2086{
2087 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2088}
2089
2090static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2091{
2092 return noblock ? 0 : sk->sk_rcvtimeo;
2093}
2094
2095static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2096{
2097 return noblock ? 0 : sk->sk_sndtimeo;
2098}
2099
2100static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2101{
2102 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2103}
2104
2105
2106
2107
2108static inline int sock_intr_errno(long timeo)
2109{
2110 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2111}
2112
2113extern void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2114 struct sk_buff *skb);
2115extern void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2116 struct sk_buff *skb);
2117
2118static inline void
2119sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2120{
2121 ktime_t kt = skb->tstamp;
2122 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2135 sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE) ||
2136 (kt.tv64 && sock_flag(sk, SOCK_TIMESTAMPING_SOFTWARE)) ||
2137 (hwtstamps->hwtstamp.tv64 &&
2138 sock_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE)) ||
2139 (hwtstamps->syststamp.tv64 &&
2140 sock_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE)))
2141 __sock_recv_timestamp(msg, sk, skb);
2142 else
2143 sk->sk_stamp = kt;
2144
2145 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2146 __sock_recv_wifi_status(msg, sk, skb);
2147}
2148
2149extern void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2150 struct sk_buff *skb);
2151
2152static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2153 struct sk_buff *skb)
2154{
2155#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2156 (1UL << SOCK_RCVTSTAMP) | \
2157 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
2158 (1UL << SOCK_TIMESTAMPING_SOFTWARE) | \
2159 (1UL << SOCK_TIMESTAMPING_RAW_HARDWARE) | \
2160 (1UL << SOCK_TIMESTAMPING_SYS_HARDWARE))
2161
2162 if (sk->sk_flags & FLAGS_TS_OR_DROPS)
2163 __sock_recv_ts_and_drops(msg, sk, skb);
2164 else
2165 sk->sk_stamp = skb->tstamp;
2166}
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176extern int sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187#ifdef CONFIG_NET_DMA
2188static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2189{
2190 __skb_unlink(skb, &sk->sk_receive_queue);
2191 if (!copied_early)
2192 __kfree_skb(skb);
2193 else
2194 __skb_queue_tail(&sk->sk_async_wait_queue, skb);
2195}
2196#else
2197static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2198{
2199 __skb_unlink(skb, &sk->sk_receive_queue);
2200 __kfree_skb(skb);
2201}
2202#endif
2203
2204static inline
2205struct net *sock_net(const struct sock *sk)
2206{
2207 return read_pnet(&sk->sk_net);
2208}
2209
2210static inline
2211void sock_net_set(struct sock *sk, struct net *net)
2212{
2213 write_pnet(&sk->sk_net, net);
2214}
2215
2216
2217
2218
2219
2220
2221
2222static inline void sk_change_net(struct sock *sk, struct net *net)
2223{
2224 put_net(sock_net(sk));
2225 sock_net_set(sk, hold_net(net));
2226}
2227
2228static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2229{
2230 if (skb->sk) {
2231 struct sock *sk = skb->sk;
2232
2233 skb->destructor = NULL;
2234 skb->sk = NULL;
2235 return sk;
2236 }
2237 return NULL;
2238}
2239
2240extern void sock_enable_timestamp(struct sock *sk, int flag);
2241extern int sock_get_timestamp(struct sock *, struct timeval __user *);
2242extern int sock_get_timestampns(struct sock *, struct timespec __user *);
2243
2244
2245
2246
2247extern int net_msg_warn;
2248#define NETDEBUG(fmt, args...) \
2249 do { if (net_msg_warn) printk(fmt,##args); } while (0)
2250
2251#define LIMIT_NETDEBUG(fmt, args...) \
2252 do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
2253
2254extern __u32 sysctl_wmem_max;
2255extern __u32 sysctl_rmem_max;
2256
2257extern int sysctl_optmem_max;
2258
2259extern __u32 sysctl_wmem_default;
2260extern __u32 sysctl_rmem_default;
2261
2262#endif
2263