Commit 7ab50771 authored by David S. Miller's avatar David S. Miller

Merge bk://kernel.bkbits.net/acme/net-2.6

into nuts.davemloft.net:/disk1/BK/net-2.6
parents 6ea12222 50bde7d0
...@@ -541,18 +541,19 @@ struct proto { ...@@ -541,18 +541,19 @@ struct proto {
/* Memory pressure */ /* Memory pressure */
void (*enter_memory_pressure)(void); void (*enter_memory_pressure)(void);
atomic_t memory_allocated; /* Current allocated memory. */ atomic_t *memory_allocated; /* Current allocated memory. */
atomic_t sockets_allocated; /* Current number of sockets. */ atomic_t *sockets_allocated; /* Current number of sockets. */
/* /*
* Pressure flag: try to collapse. * Pressure flag: try to collapse.
* Technical note: it is used by multiple contexts non atomically. * Technical note: it is used by multiple contexts non atomically.
* All the sk_stream_mem_schedule() is of this nature: accounting * All the sk_stream_mem_schedule() is of this nature: accounting
* is strict, actions are advisory and have some latency. * is strict, actions are advisory and have some latency.
*/ */
int memory_pressure; int *memory_pressure;
int sysctl_mem[3]; int *sysctl_mem;
int sysctl_wmem[3]; int *sysctl_wmem;
int sysctl_rmem[3]; int *sysctl_rmem;
int max_header;
char name[32]; char name[32];
...@@ -660,6 +661,21 @@ static inline void sk_stream_mem_reclaim(struct sock *sk) ...@@ -660,6 +661,21 @@ static inline void sk_stream_mem_reclaim(struct sock *sk)
__sk_stream_mem_reclaim(sk); __sk_stream_mem_reclaim(sk);
} }
static inline void sk_stream_writequeue_purge(struct sock *sk)
{
struct sk_buff *skb;
while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
sk_stream_free_skb(sk, skb);
sk_stream_mem_reclaim(sk);
}
static inline int sk_stream_rmem_schedule(struct sock *sk, struct sk_buff *skb)
{
return (int)skb->truesize <= sk->sk_forward_alloc ||
sk_stream_mem_schedule(sk, skb->truesize, 1);
}
/* Used by processes to "lock" a socket state, so that /* Used by processes to "lock" a socket state, so that
* interrupts and bottom half handlers won't change it * interrupts and bottom half handlers won't change it
* from under us. It essentially blocks any incoming * from under us. It essentially blocks any incoming
...@@ -1141,6 +1157,46 @@ static inline void sk_stream_moderate_sndbuf(struct sock *sk) ...@@ -1141,6 +1157,46 @@ static inline void sk_stream_moderate_sndbuf(struct sock *sk)
} }
} }
static inline struct sk_buff *sk_stream_alloc_pskb(struct sock *sk,
int size, int mem, int gfp)
{
struct sk_buff *skb = alloc_skb(size + sk->sk_prot->max_header, gfp);
if (skb) {
skb->truesize += mem;
if (sk->sk_forward_alloc >= (int)skb->truesize ||
sk_stream_mem_schedule(sk, skb->truesize, 0)) {
skb_reserve(skb, sk->sk_prot->max_header);
return skb;
}
__kfree_skb(skb);
} else {
sk->sk_prot->enter_memory_pressure();
sk_stream_moderate_sndbuf(sk);
}
return NULL;
}
static inline struct sk_buff *sk_stream_alloc_skb(struct sock *sk,
int size, int gfp)
{
return sk_stream_alloc_pskb(sk, size, 0, gfp);
}
static inline struct page *sk_stream_alloc_page(struct sock *sk)
{
struct page *page = NULL;
if (sk->sk_forward_alloc >= (int)PAGE_SIZE ||
sk_stream_mem_schedule(sk, PAGE_SIZE, 0))
page = alloc_pages(sk->sk_allocation, 0);
else {
sk->sk_prot->enter_memory_pressure();
sk_stream_moderate_sndbuf(sk);
}
return page;
}
#define sk_stream_for_retrans_queue(skb, sk) \ #define sk_stream_for_retrans_queue(skb, sk) \
for (skb = (sk)->sk_write_queue.next; \ for (skb = (sk)->sk_write_queue.next; \
(skb != (sk)->sk_send_head) && \ (skb != (sk)->sk_send_head) && \
......
...@@ -594,6 +594,9 @@ extern int sysctl_tcp_fack; ...@@ -594,6 +594,9 @@ extern int sysctl_tcp_fack;
extern int sysctl_tcp_reordering; extern int sysctl_tcp_reordering;
extern int sysctl_tcp_ecn; extern int sysctl_tcp_ecn;
extern int sysctl_tcp_dsack; extern int sysctl_tcp_dsack;
extern int sysctl_tcp_mem[3];
extern int sysctl_tcp_wmem[3];
extern int sysctl_tcp_rmem[3];
extern int sysctl_tcp_app_win; extern int sysctl_tcp_app_win;
extern int sysctl_tcp_adv_win_scale; extern int sysctl_tcp_adv_win_scale;
extern int sysctl_tcp_tw_reuse; extern int sysctl_tcp_tw_reuse;
...@@ -611,6 +614,10 @@ extern int sysctl_tcp_bic_low_window; ...@@ -611,6 +614,10 @@ extern int sysctl_tcp_bic_low_window;
extern int sysctl_tcp_default_win_scale; extern int sysctl_tcp_default_win_scale;
extern int sysctl_tcp_moderate_rcvbuf; extern int sysctl_tcp_moderate_rcvbuf;
extern atomic_t tcp_memory_allocated;
extern atomic_t tcp_sockets_allocated;
extern int tcp_memory_pressure;
struct open_request; struct open_request;
struct or_calltable { struct or_calltable {
...@@ -1862,52 +1869,6 @@ static __inline__ void tcp_openreq_init(struct open_request *req, ...@@ -1862,52 +1869,6 @@ static __inline__ void tcp_openreq_init(struct open_request *req,
extern void tcp_enter_memory_pressure(void); extern void tcp_enter_memory_pressure(void);
static inline struct sk_buff *tcp_alloc_pskb(struct sock *sk, int size, int mem, int gfp)
{
struct sk_buff *skb = alloc_skb(size+MAX_TCP_HEADER, gfp);
if (skb) {
skb->truesize += mem;
if (sk->sk_forward_alloc >= (int)skb->truesize ||
sk_stream_mem_schedule(sk, skb->truesize, 0)) {
skb_reserve(skb, MAX_TCP_HEADER);
return skb;
}
__kfree_skb(skb);
} else {
tcp_enter_memory_pressure();
sk_stream_moderate_sndbuf(sk);
}
return NULL;
}
static inline struct sk_buff *tcp_alloc_skb(struct sock *sk, int size, int gfp)
{
return tcp_alloc_pskb(sk, size, 0, gfp);
}
static inline struct page * tcp_alloc_page(struct sock *sk)
{
if (sk->sk_forward_alloc >= (int)PAGE_SIZE ||
sk_stream_mem_schedule(sk, PAGE_SIZE, 0)) {
struct page *page = alloc_pages(sk->sk_allocation, 0);
if (page)
return page;
}
tcp_enter_memory_pressure();
sk_stream_moderate_sndbuf(sk);
return NULL;
}
static inline void tcp_writequeue_purge(struct sock *sk)
{
struct sk_buff *skb;
while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
sk_stream_free_skb(sk, skb);
sk_stream_mem_reclaim(sk);
}
extern void tcp_listen_wlock(void); extern void tcp_listen_wlock(void);
/* - We may sleep inside this lock. /* - We may sleep inside this lock.
......
...@@ -193,12 +193,12 @@ void __sk_stream_mem_reclaim(struct sock *sk) ...@@ -193,12 +193,12 @@ void __sk_stream_mem_reclaim(struct sock *sk)
{ {
if (sk->sk_forward_alloc >= SK_STREAM_MEM_QUANTUM) { if (sk->sk_forward_alloc >= SK_STREAM_MEM_QUANTUM) {
atomic_sub(sk->sk_forward_alloc / SK_STREAM_MEM_QUANTUM, atomic_sub(sk->sk_forward_alloc / SK_STREAM_MEM_QUANTUM,
&sk->sk_prot->memory_allocated); sk->sk_prot->memory_allocated);
sk->sk_forward_alloc &= SK_STREAM_MEM_QUANTUM - 1; sk->sk_forward_alloc &= SK_STREAM_MEM_QUANTUM - 1;
if (sk->sk_prot->memory_pressure && if (*sk->sk_prot->memory_pressure &&
(atomic_read(&sk->sk_prot->memory_allocated) < (atomic_read(sk->sk_prot->memory_allocated) <
sk->sk_prot->sysctl_mem[0])) sk->sk_prot->sysctl_mem[0]))
sk->sk_prot->memory_pressure = 0; *sk->sk_prot->memory_pressure = 0;
} }
} }
...@@ -209,23 +209,23 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind) ...@@ -209,23 +209,23 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind)
int amt = sk_stream_pages(size); int amt = sk_stream_pages(size);
sk->sk_forward_alloc += amt * SK_STREAM_MEM_QUANTUM; sk->sk_forward_alloc += amt * SK_STREAM_MEM_QUANTUM;
atomic_add(amt, &sk->sk_prot->memory_allocated); atomic_add(amt, sk->sk_prot->memory_allocated);
/* Under limit. */ /* Under limit. */
if (atomic_read(&sk->sk_prot->memory_allocated) < sk->sk_prot->sysctl_mem[0]) { if (atomic_read(sk->sk_prot->memory_allocated) < sk->sk_prot->sysctl_mem[0]) {
if (sk->sk_prot->memory_pressure) if (*sk->sk_prot->memory_pressure)
sk->sk_prot->memory_pressure = 0; *sk->sk_prot->memory_pressure = 0;
return 1; return 1;
} }
/* Over hard limit. */ /* Over hard limit. */
if (atomic_read(&sk->sk_prot->memory_allocated) > sk->sk_prot->sysctl_mem[2]) { if (atomic_read(sk->sk_prot->memory_allocated) > sk->sk_prot->sysctl_mem[2]) {
sk->sk_prot->enter_memory_pressure(); sk->sk_prot->enter_memory_pressure();
goto suppress_allocation; goto suppress_allocation;
} }
/* Under pressure. */ /* Under pressure. */
if (atomic_read(&sk->sk_prot->memory_allocated) > sk->sk_prot->sysctl_mem[1]) if (atomic_read(sk->sk_prot->memory_allocated) > sk->sk_prot->sysctl_mem[1])
sk->sk_prot->enter_memory_pressure(); sk->sk_prot->enter_memory_pressure();
if (kind) { if (kind) {
...@@ -234,8 +234,8 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind) ...@@ -234,8 +234,8 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind)
} else if (sk->sk_wmem_queued < sk->sk_prot->sysctl_wmem[0]) } else if (sk->sk_wmem_queued < sk->sk_prot->sysctl_wmem[0])
return 1; return 1;
if (!sk->sk_prot->memory_pressure || if (!*sk->sk_prot->memory_pressure ||
sk->sk_prot->sysctl_mem[2] > atomic_read(&sk->sk_prot->sockets_allocated) * sk->sk_prot->sysctl_mem[2] > atomic_read(sk->sk_prot->sockets_allocated) *
sk_stream_pages(sk->sk_wmem_queued + sk_stream_pages(sk->sk_wmem_queued +
atomic_read(&sk->sk_rmem_alloc) + atomic_read(&sk->sk_rmem_alloc) +
sk->sk_forward_alloc)) sk->sk_forward_alloc))
...@@ -255,7 +255,7 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind) ...@@ -255,7 +255,7 @@ int sk_stream_mem_schedule(struct sock *sk, int size, int kind)
/* Alas. Undo changes. */ /* Alas. Undo changes. */
sk->sk_forward_alloc -= amt * SK_STREAM_MEM_QUANTUM; sk->sk_forward_alloc -= amt * SK_STREAM_MEM_QUANTUM;
atomic_sub(amt, &sk->sk_prot->memory_allocated); atomic_sub(amt, sk->sk_prot->memory_allocated);
return 0; return 0;
} }
......
...@@ -65,8 +65,8 @@ static int sockstat_seq_show(struct seq_file *seq, void *v) ...@@ -65,8 +65,8 @@ static int sockstat_seq_show(struct seq_file *seq, void *v)
socket_seq_show(seq); socket_seq_show(seq);
seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %d\n", seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %d\n",
fold_prot_inuse(&tcp_prot), atomic_read(&tcp_orphan_count), fold_prot_inuse(&tcp_prot), atomic_read(&tcp_orphan_count),
tcp_tw_count, atomic_read(&tcp_prot.sockets_allocated), tcp_tw_count, atomic_read(&tcp_sockets_allocated),
atomic_read(&tcp_prot.memory_allocated)); atomic_read(&tcp_memory_allocated));
seq_printf(seq, "UDP: inuse %d\n", fold_prot_inuse(&udp_prot)); seq_printf(seq, "UDP: inuse %d\n", fold_prot_inuse(&udp_prot));
seq_printf(seq, "RAW: inuse %d\n", fold_prot_inuse(&raw_prot)); seq_printf(seq, "RAW: inuse %d\n", fold_prot_inuse(&raw_prot));
seq_printf(seq, "FRAG: inuse %d memory %d\n", ip_frag_nqueues, seq_printf(seq, "FRAG: inuse %d memory %d\n", ip_frag_nqueues,
......
...@@ -508,24 +508,24 @@ ctl_table ipv4_table[] = { ...@@ -508,24 +508,24 @@ ctl_table ipv4_table[] = {
{ {
.ctl_name = NET_TCP_MEM, .ctl_name = NET_TCP_MEM,
.procname = "tcp_mem", .procname = "tcp_mem",
.data = &tcp_prot.sysctl_mem, .data = &sysctl_tcp_mem,
.maxlen = sizeof(tcp_prot.sysctl_mem), .maxlen = sizeof(sysctl_tcp_mem),
.mode = 0644, .mode = 0644,
.proc_handler = &proc_dointvec .proc_handler = &proc_dointvec
}, },
{ {
.ctl_name = NET_TCP_WMEM, .ctl_name = NET_TCP_WMEM,
.procname = "tcp_wmem", .procname = "tcp_wmem",
.data = &tcp_prot.sysctl_wmem, .data = &sysctl_tcp_wmem,
.maxlen = sizeof(tcp_prot.sysctl_wmem), .maxlen = sizeof(sysctl_tcp_wmem),
.mode = 0644, .mode = 0644,
.proc_handler = &proc_dointvec .proc_handler = &proc_dointvec
}, },
{ {
.ctl_name = NET_TCP_RMEM, .ctl_name = NET_TCP_RMEM,
.procname = "tcp_rmem", .procname = "tcp_rmem",
.data = &tcp_prot.sysctl_rmem, .data = &sysctl_tcp_rmem,
.maxlen = sizeof(tcp_prot.sysctl_rmem), .maxlen = sizeof(sysctl_tcp_rmem),
.mode = 0644, .mode = 0644,
.proc_handler = &proc_dointvec .proc_handler = &proc_dointvec
}, },
......
...@@ -278,14 +278,40 @@ atomic_t tcp_orphan_count = ATOMIC_INIT(0); ...@@ -278,14 +278,40 @@ atomic_t tcp_orphan_count = ATOMIC_INIT(0);
int sysctl_tcp_default_win_scale = 7; int sysctl_tcp_default_win_scale = 7;
int sysctl_tcp_mem[3];
int sysctl_tcp_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 };
int sysctl_tcp_rmem[3] = { 4 * 1024, 87380, 87380 * 2 };
EXPORT_SYMBOL(sysctl_tcp_mem);
EXPORT_SYMBOL(sysctl_tcp_rmem);
EXPORT_SYMBOL(sysctl_tcp_wmem);
atomic_t tcp_memory_allocated; /* Current allocated memory. */
atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */
EXPORT_SYMBOL(tcp_memory_allocated);
EXPORT_SYMBOL(tcp_sockets_allocated);
/*
* Pressure flag: try to collapse.
* Technical note: it is used by multiple contexts non atomically.
* All the sk_stream_mem_schedule() is of this nature: accounting
* is strict, actions are advisory and have some latency.
*/
int tcp_memory_pressure;
EXPORT_SYMBOL(tcp_memory_pressure);
void tcp_enter_memory_pressure(void) void tcp_enter_memory_pressure(void)
{ {
if (!tcp_prot.memory_pressure) { if (!tcp_memory_pressure) {
NET_INC_STATS(TCPMemoryPressures); NET_INC_STATS(TCPMemoryPressures);
tcp_prot.memory_pressure = 1; tcp_memory_pressure = 1;
} }
} }
EXPORT_SYMBOL(tcp_enter_memory_pressure);
/* /*
* LISTEN is a special case for poll.. * LISTEN is a special case for poll..
*/ */
...@@ -639,7 +665,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse ...@@ -639,7 +665,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
if (!sk_stream_memory_free(sk)) if (!sk_stream_memory_free(sk))
goto wait_for_sndbuf; goto wait_for_sndbuf;
skb = tcp_alloc_pskb(sk, 0, tp->mss_cache, skb = sk_stream_alloc_pskb(sk, 0, tp->mss_cache,
sk->sk_allocation); sk->sk_allocation);
if (!skb) if (!skb)
goto wait_for_memory; goto wait_for_memory;
...@@ -806,7 +832,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -806,7 +832,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
if (!sk_stream_memory_free(sk)) if (!sk_stream_memory_free(sk))
goto wait_for_sndbuf; goto wait_for_sndbuf;
skb = tcp_alloc_pskb(sk, select_size(sk, tp), skb = sk_stream_alloc_pskb(sk, select_size(sk, tp),
0, sk->sk_allocation); 0, sk->sk_allocation);
if (!skb) if (!skb)
goto wait_for_memory; goto wait_for_memory;
...@@ -868,7 +894,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -868,7 +894,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
if (!page) { if (!page) {
/* Allocate new cache page. */ /* Allocate new cache page. */
if (!(page = tcp_alloc_page(sk))) if (!(page = sk_stream_alloc_page(sk)))
goto wait_for_memory; goto wait_for_memory;
off = 0; off = 0;
} }
...@@ -1722,7 +1748,7 @@ void tcp_close(struct sock *sk, long timeout) ...@@ -1722,7 +1748,7 @@ void tcp_close(struct sock *sk, long timeout)
sk_stream_mem_reclaim(sk); sk_stream_mem_reclaim(sk);
if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans || if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans ||
(sk->sk_wmem_queued > SOCK_MIN_SNDBUF && (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
atomic_read(&tcp_prot.memory_allocated) > tcp_prot.sysctl_mem[2])) { atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) {
if (net_ratelimit()) if (net_ratelimit())
printk(KERN_INFO "TCP: too many of orphaned " printk(KERN_INFO "TCP: too many of orphaned "
"sockets\n"); "sockets\n");
...@@ -1778,7 +1804,7 @@ int tcp_disconnect(struct sock *sk, int flags) ...@@ -1778,7 +1804,7 @@ int tcp_disconnect(struct sock *sk, int flags)
tcp_clear_xmit_timers(sk); tcp_clear_xmit_timers(sk);
__skb_queue_purge(&sk->sk_receive_queue); __skb_queue_purge(&sk->sk_receive_queue);
tcp_writequeue_purge(sk); sk_stream_writequeue_purge(sk);
__skb_queue_purge(&tp->out_of_order_queue); __skb_queue_purge(&tp->out_of_order_queue);
inet->dport = 0; inet->dport = 0;
...@@ -2269,15 +2295,15 @@ void __init tcp_init(void) ...@@ -2269,15 +2295,15 @@ void __init tcp_init(void)
} }
tcp_port_rover = sysctl_local_port_range[0] - 1; tcp_port_rover = sysctl_local_port_range[0] - 1;
tcp_prot.sysctl_mem[0] = 768 << order; sysctl_tcp_mem[0] = 768 << order;
tcp_prot.sysctl_mem[1] = 1024 << order; sysctl_tcp_mem[1] = 1024 << order;
tcp_prot.sysctl_mem[2] = 1536 << order; sysctl_tcp_mem[2] = 1536 << order;
if (order < 3) { if (order < 3) {
tcp_prot.sysctl_wmem[2] = 64 * 1024; sysctl_tcp_wmem[2] = 64 * 1024;
tcp_prot.sysctl_rmem[0] = PAGE_SIZE; sysctl_tcp_rmem[0] = PAGE_SIZE;
tcp_prot.sysctl_rmem[1] = 43689; sysctl_tcp_rmem[1] = 43689;
tcp_prot.sysctl_rmem[2] = 2 * 43689; sysctl_tcp_rmem[2] = 2 * 43689;
} }
printk(KERN_INFO "TCP: Hash tables configured " printk(KERN_INFO "TCP: Hash tables configured "
......
...@@ -207,7 +207,7 @@ static void tcp_fixup_sndbuf(struct sock *sk) ...@@ -207,7 +207,7 @@ static void tcp_fixup_sndbuf(struct sock *sk)
sizeof(struct sk_buff); sizeof(struct sk_buff);
if (sk->sk_sndbuf < 3 * sndmem) if (sk->sk_sndbuf < 3 * sndmem)
sk->sk_sndbuf = min(3 * sndmem, tcp_prot.sysctl_wmem[2]); sk->sk_sndbuf = min(3 * sndmem, sysctl_tcp_wmem[2]);
} }
/* 2. Tuning advertised window (window_clamp, rcv_ssthresh) /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
...@@ -291,7 +291,7 @@ static void tcp_fixup_rcvbuf(struct sock *sk) ...@@ -291,7 +291,7 @@ static void tcp_fixup_rcvbuf(struct sock *sk)
while (tcp_win_from_space(rcvmem) < tp->advmss) while (tcp_win_from_space(rcvmem) < tp->advmss)
rcvmem += 128; rcvmem += 128;
if (sk->sk_rcvbuf < 4 * rcvmem) if (sk->sk_rcvbuf < 4 * rcvmem)
sk->sk_rcvbuf = min(4 * rcvmem, tcp_prot.sysctl_rmem[2]); sk->sk_rcvbuf = min(4 * rcvmem, sysctl_tcp_rmem[2]);
} }
/* 4. Try to fixup all. It is made iimediately after connection enters /* 4. Try to fixup all. It is made iimediately after connection enters
...@@ -347,12 +347,12 @@ static void tcp_clamp_window(struct sock *sk, struct tcp_opt *tp) ...@@ -347,12 +347,12 @@ static void tcp_clamp_window(struct sock *sk, struct tcp_opt *tp)
* do not clamp window. Try to expand rcvbuf instead. * do not clamp window. Try to expand rcvbuf instead.
*/ */
if (ofo_win) { if (ofo_win) {
if (sk->sk_rcvbuf < tcp_prot.sysctl_rmem[2] && if (sk->sk_rcvbuf < sysctl_tcp_rmem[2] &&
!(sk->sk_userlocks & SOCK_RCVBUF_LOCK) && !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
!tcp_prot.memory_pressure && !tcp_prot.memory_pressure &&
atomic_read(&tcp_prot.memory_allocated) < tcp_prot.sysctl_mem[0]) atomic_read(&tcp_memory_allocated) < sysctl_tcp_mem[0])
sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc), sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc),
tcp_prot.sysctl_rmem[2]); sysctl_tcp_rmem[2]);
} }
if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf) { if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf) {
app_win += ofo_win; app_win += ofo_win;
...@@ -477,7 +477,7 @@ void tcp_rcv_space_adjust(struct sock *sk) ...@@ -477,7 +477,7 @@ void tcp_rcv_space_adjust(struct sock *sk)
while (tcp_win_from_space(rcvmem) < tp->advmss) while (tcp_win_from_space(rcvmem) < tp->advmss)
rcvmem += 128; rcvmem += 128;
space *= rcvmem; space *= rcvmem;
space = min(space, tcp_prot.sysctl_rmem[2]); space = min(space, sysctl_tcp_rmem[2]);
if (space > sk->sk_rcvbuf) { if (space > sk->sk_rcvbuf) {
sk->sk_rcvbuf = space; sk->sk_rcvbuf = space;
...@@ -3398,12 +3398,6 @@ static void tcp_ofo_queue(struct sock *sk) ...@@ -3398,12 +3398,6 @@ static void tcp_ofo_queue(struct sock *sk)
} }
} }
static inline int tcp_rmem_schedule(struct sock *sk, struct sk_buff *skb)
{
return (int)skb->truesize <= sk->sk_forward_alloc ||
sk_stream_mem_schedule(sk, skb->truesize, 1);
}
static int tcp_prune_queue(struct sock *sk); static int tcp_prune_queue(struct sock *sk);
static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
...@@ -3457,8 +3451,9 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) ...@@ -3457,8 +3451,9 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
queue_and_out: queue_and_out:
if (eaten < 0 && if (eaten < 0 &&
(atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
!tcp_rmem_schedule(sk, skb))) { !sk_stream_rmem_schedule(sk, skb))) {
if (tcp_prune_queue(sk) < 0 || !tcp_rmem_schedule(sk, skb)) if (tcp_prune_queue(sk) < 0 ||
!sk_stream_rmem_schedule(sk, skb))
goto drop; goto drop;
} }
sk_stream_set_owner_r(skb, sk); sk_stream_set_owner_r(skb, sk);
...@@ -3530,8 +3525,9 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) ...@@ -3530,8 +3525,9 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
TCP_ECN_check_ce(tp, skb); TCP_ECN_check_ce(tp, skb);
if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
!tcp_rmem_schedule(sk, skb)) { !sk_stream_rmem_schedule(sk, skb)) {
if (tcp_prune_queue(sk) < 0 || !tcp_rmem_schedule(sk, skb)) if (tcp_prune_queue(sk) < 0 ||
!sk_stream_rmem_schedule(sk, skb))
goto drop; goto drop;
} }
...@@ -3849,14 +3845,14 @@ static void tcp_new_space(struct sock *sk) ...@@ -3849,14 +3845,14 @@ static void tcp_new_space(struct sock *sk)
if (tp->packets_out < tp->snd_cwnd && if (tp->packets_out < tp->snd_cwnd &&
!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) && !(sk->sk_userlocks & SOCK_SNDBUF_LOCK) &&
!tcp_prot.memory_pressure && !tcp_prot.memory_pressure &&
atomic_read(&tcp_prot.memory_allocated) < tcp_prot.sysctl_mem[0]) { atomic_read(&tcp_memory_allocated) < sysctl_tcp_mem[0]) {
int sndmem = max_t(u32, tp->mss_clamp, tp->mss_cache) + int sndmem = max_t(u32, tp->mss_clamp, tp->mss_cache) +
MAX_TCP_HEADER + 16 + sizeof(struct sk_buff), MAX_TCP_HEADER + 16 + sizeof(struct sk_buff),
demanded = max_t(unsigned int, tp->snd_cwnd, demanded = max_t(unsigned int, tp->snd_cwnd,
tp->reordering + 1); tp->reordering + 1);
sndmem *= 2*demanded; sndmem *= 2*demanded;
if (sndmem > sk->sk_sndbuf) if (sndmem > sk->sk_sndbuf)
sk->sk_sndbuf = min(sndmem, tcp_prot.sysctl_wmem[2]); sk->sk_sndbuf = min(sndmem, sysctl_tcp_wmem[2]);
tp->snd_cwnd_stamp = tcp_time_stamp; tp->snd_cwnd_stamp = tcp_time_stamp;
} }
......
...@@ -2086,10 +2086,10 @@ static int tcp_v4_init_sock(struct sock *sk) ...@@ -2086,10 +2086,10 @@ static int tcp_v4_init_sock(struct sock *sk)
tp->af_specific = &ipv4_specific; tp->af_specific = &ipv4_specific;
sk->sk_sndbuf = tcp_prot.sysctl_wmem[1]; sk->sk_sndbuf = sysctl_tcp_wmem[1];
sk->sk_rcvbuf = tcp_prot.sysctl_rmem[1]; sk->sk_rcvbuf = sysctl_tcp_rmem[1];
atomic_inc(&tcp_prot.sockets_allocated); atomic_inc(&tcp_sockets_allocated);
return 0; return 0;
} }
...@@ -2101,7 +2101,7 @@ static int tcp_v4_destroy_sock(struct sock *sk) ...@@ -2101,7 +2101,7 @@ static int tcp_v4_destroy_sock(struct sock *sk)
tcp_clear_xmit_timers(sk); tcp_clear_xmit_timers(sk);
/* Cleanup up the write buffer. */ /* Cleanup up the write buffer. */
tcp_writequeue_purge(sk); sk_stream_writequeue_purge(sk);
/* Cleans up our, hopefully empty, out_of_order_queue. */ /* Cleans up our, hopefully empty, out_of_order_queue. */
__skb_queue_purge(&tp->out_of_order_queue); __skb_queue_purge(&tp->out_of_order_queue);
...@@ -2113,7 +2113,7 @@ static int tcp_v4_destroy_sock(struct sock *sk) ...@@ -2113,7 +2113,7 @@ static int tcp_v4_destroy_sock(struct sock *sk)
if (tp->bind_hash) if (tp->bind_hash)
tcp_put_port(sk); tcp_put_port(sk);
atomic_dec(&tcp_prot.sockets_allocated); atomic_dec(&tcp_sockets_allocated);
return 0; return 0;
} }
...@@ -2600,8 +2600,13 @@ struct proto tcp_prot = { ...@@ -2600,8 +2600,13 @@ struct proto tcp_prot = {
.unhash = tcp_unhash, .unhash = tcp_unhash,
.get_port = tcp_v4_get_port, .get_port = tcp_v4_get_port,
.enter_memory_pressure = tcp_enter_memory_pressure, .enter_memory_pressure = tcp_enter_memory_pressure,
.sysctl_wmem = { 4 * 1024, 16 * 1024, 128 * 1024 }, .sockets_allocated = &tcp_sockets_allocated,
.sysctl_rmem = { 4 * 1024, 87380, 87380 * 2 }, .memory_allocated = &tcp_memory_allocated,
.memory_pressure = &tcp_memory_pressure,
.sysctl_mem = sysctl_tcp_mem,
.sysctl_wmem = sysctl_tcp_wmem,
.sysctl_rmem = sysctl_tcp_rmem,
.max_header = MAX_TCP_HEADER,
}; };
......
...@@ -801,7 +801,7 @@ struct sock *tcp_create_openreq_child(struct sock *sk, struct open_request *req, ...@@ -801,7 +801,7 @@ struct sock *tcp_create_openreq_child(struct sock *sk, struct open_request *req,
#ifdef INET_REFCNT_DEBUG #ifdef INET_REFCNT_DEBUG
atomic_inc(&inet_sock_nr); atomic_inc(&inet_sock_nr);
#endif #endif
atomic_inc(&tcp_prot.sockets_allocated); atomic_inc(&tcp_sockets_allocated);
if (sock_flag(newsk, SOCK_KEEPOPEN)) if (sock_flag(newsk, SOCK_KEEPOPEN))
tcp_reset_keepalive_timer(newsk, tcp_reset_keepalive_timer(newsk,
......
...@@ -372,7 +372,7 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len) ...@@ -372,7 +372,7 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len)
return -ENOMEM; return -ENOMEM;
/* Get a new skb... force flag on. */ /* Get a new skb... force flag on. */
buff = tcp_alloc_skb(sk, nsize, GFP_ATOMIC); buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
if (buff == NULL) if (buff == NULL)
return -ENOMEM; /* We'll just try again later. */ return -ENOMEM; /* We'll just try again later. */
sk_charge_skb(sk, buff); sk_charge_skb(sk, buff);
......
...@@ -113,7 +113,7 @@ static int tcp_out_of_resources(struct sock *sk, int do_reset) ...@@ -113,7 +113,7 @@ static int tcp_out_of_resources(struct sock *sk, int do_reset)
if (orphans >= sysctl_tcp_max_orphans || if (orphans >= sysctl_tcp_max_orphans ||
(sk->sk_wmem_queued > SOCK_MIN_SNDBUF && (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
atomic_read(&tcp_prot.memory_allocated) > tcp_prot.sysctl_mem[2])) { atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) {
if (net_ratelimit()) if (net_ratelimit())
printk(KERN_INFO "Out of socket memory\n"); printk(KERN_INFO "Out of socket memory\n");
......
...@@ -1882,10 +1882,10 @@ static int tcp_v6_init_sock(struct sock *sk) ...@@ -1882,10 +1882,10 @@ static int tcp_v6_init_sock(struct sock *sk)
sk->sk_write_space = sk_stream_write_space; sk->sk_write_space = sk_stream_write_space;
sk->sk_use_write_queue = 1; sk->sk_use_write_queue = 1;
sk->sk_sndbuf = tcp_prot.sysctl_wmem[1]; sk->sk_sndbuf = sysctl_tcp_wmem[1];
sk->sk_rcvbuf = tcp_prot.sysctl_rmem[1]; sk->sk_rcvbuf = sysctl_tcp_rmem[1];
atomic_inc(&tcp_prot.sockets_allocated); atomic_inc(&tcp_sockets_allocated);
return 0; return 0;
} }
...@@ -1897,7 +1897,7 @@ static int tcp_v6_destroy_sock(struct sock *sk) ...@@ -1897,7 +1897,7 @@ static int tcp_v6_destroy_sock(struct sock *sk)
tcp_clear_xmit_timers(sk); tcp_clear_xmit_timers(sk);
/* Cleanup up the write buffer. */ /* Cleanup up the write buffer. */
tcp_writequeue_purge(sk); sk_stream_writequeue_purge(sk);
/* Cleans up our, hopefully empty, out_of_order_queue. */ /* Cleans up our, hopefully empty, out_of_order_queue. */
__skb_queue_purge(&tp->out_of_order_queue); __skb_queue_purge(&tp->out_of_order_queue);
...@@ -1909,7 +1909,7 @@ static int tcp_v6_destroy_sock(struct sock *sk) ...@@ -1909,7 +1909,7 @@ static int tcp_v6_destroy_sock(struct sock *sk)
if (tcp_sk(sk)->bind_hash) if (tcp_sk(sk)->bind_hash)
tcp_put_port(sk); tcp_put_port(sk);
atomic_dec(&tcp_prot.sockets_allocated); atomic_dec(&tcp_sockets_allocated);
return inet6_destroy_sock(sk); return inet6_destroy_sock(sk);
} }
...@@ -2095,6 +2095,14 @@ struct proto tcpv6_prot = { ...@@ -2095,6 +2095,14 @@ struct proto tcpv6_prot = {
.hash = tcp_v6_hash, .hash = tcp_v6_hash,
.unhash = tcp_unhash, .unhash = tcp_unhash,
.get_port = tcp_v6_get_port, .get_port = tcp_v6_get_port,
.enter_memory_pressure = tcp_enter_memory_pressure,
.sockets_allocated = &tcp_sockets_allocated,
.memory_allocated = &tcp_memory_allocated,
.memory_pressure = &tcp_memory_pressure,
.sysctl_mem = sysctl_tcp_mem,
.sysctl_wmem = sysctl_tcp_wmem,
.sysctl_rmem = sysctl_tcp_rmem,
.max_header = MAX_TCP_HEADER,
}; };
static struct inet6_protocol tcpv6_protocol = { static struct inet6_protocol tcpv6_protocol = {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment