Commit 60cd4bcd authored by Shakeel Butt's avatar Shakeel Butt Committed by Linus Torvalds

memcg: localize memcg_kmem_enabled() check

Move the memcg_kmem_enabled() checks into memcg kmem charge/uncharge
functions, so, the users don't have to explicitly check that condition.

This is purely code cleanup patch without any functional change.  Only
the order of checks in memcg_charge_slab() can potentially be changed
but the functionally it will be same.  This should not matter as
memcg_charge_slab() is not in the hot path.

Link: http://lkml.kernel.org/r/20190103161203.162375-1-shakeelb@google.comSigned-off-by: default avatarShakeel Butt <shakeelb@google.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Roman Gushchin <guro@fb.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9234bae9
...@@ -140,8 +140,7 @@ static int anon_pipe_buf_steal(struct pipe_inode_info *pipe, ...@@ -140,8 +140,7 @@ static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
struct page *page = buf->page; struct page *page = buf->page;
if (page_count(page) == 1) { if (page_count(page) == 1) {
if (memcg_kmem_enabled()) memcg_kmem_uncharge(page, 0);
memcg_kmem_uncharge(page, 0);
__SetPageLocked(page); __SetPageLocked(page);
return 0; return 0;
} }
......
...@@ -1273,12 +1273,12 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) ...@@ -1273,12 +1273,12 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep); struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
void memcg_kmem_put_cache(struct kmem_cache *cachep); void memcg_kmem_put_cache(struct kmem_cache *cachep);
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
struct mem_cgroup *memcg);
#ifdef CONFIG_MEMCG_KMEM #ifdef CONFIG_MEMCG_KMEM
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order); int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
void memcg_kmem_uncharge(struct page *page, int order); void __memcg_kmem_uncharge(struct page *page, int order);
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
struct mem_cgroup *memcg);
extern struct static_key_false memcg_kmem_enabled_key; extern struct static_key_false memcg_kmem_enabled_key;
extern struct workqueue_struct *memcg_kmem_cache_wq; extern struct workqueue_struct *memcg_kmem_cache_wq;
...@@ -1300,6 +1300,26 @@ static inline bool memcg_kmem_enabled(void) ...@@ -1300,6 +1300,26 @@ static inline bool memcg_kmem_enabled(void)
return static_branch_unlikely(&memcg_kmem_enabled_key); return static_branch_unlikely(&memcg_kmem_enabled_key);
} }
static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{
if (memcg_kmem_enabled())
return __memcg_kmem_charge(page, gfp, order);
return 0;
}
static inline void memcg_kmem_uncharge(struct page *page, int order)
{
if (memcg_kmem_enabled())
__memcg_kmem_uncharge(page, order);
}
static inline int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp,
int order, struct mem_cgroup *memcg)
{
if (memcg_kmem_enabled())
return __memcg_kmem_charge_memcg(page, gfp, order, memcg);
return 0;
}
/* /*
* helper for accessing a memcg's index. It will be used as an index in the * helper for accessing a memcg's index. It will be used as an index in the
* child cache array in kmem_cache, and also to derive its name. This function * child cache array in kmem_cache, and also to derive its name. This function
...@@ -1325,6 +1345,15 @@ static inline void memcg_kmem_uncharge(struct page *page, int order) ...@@ -1325,6 +1345,15 @@ static inline void memcg_kmem_uncharge(struct page *page, int order)
{ {
} }
static inline int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{
return 0;
}
static inline void __memcg_kmem_uncharge(struct page *page, int order)
{
}
#define for_each_memcg_cache_index(_idx) \ #define for_each_memcg_cache_index(_idx) \
for (; NULL; ) for (; NULL; )
......
...@@ -2573,7 +2573,7 @@ void memcg_kmem_put_cache(struct kmem_cache *cachep) ...@@ -2573,7 +2573,7 @@ void memcg_kmem_put_cache(struct kmem_cache *cachep)
} }
/** /**
* memcg_kmem_charge_memcg: charge a kmem page * __memcg_kmem_charge_memcg: charge a kmem page
* @page: page to charge * @page: page to charge
* @gfp: reclaim mode * @gfp: reclaim mode
* @order: allocation order * @order: allocation order
...@@ -2581,7 +2581,7 @@ void memcg_kmem_put_cache(struct kmem_cache *cachep) ...@@ -2581,7 +2581,7 @@ void memcg_kmem_put_cache(struct kmem_cache *cachep)
* *
* Returns 0 on success, an error code on failure. * Returns 0 on success, an error code on failure.
*/ */
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
struct mem_cgroup *memcg) struct mem_cgroup *memcg)
{ {
unsigned int nr_pages = 1 << order; unsigned int nr_pages = 1 << order;
...@@ -2604,24 +2604,24 @@ int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, ...@@ -2604,24 +2604,24 @@ int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
} }
/** /**
* memcg_kmem_charge: charge a kmem page to the current memory cgroup * __memcg_kmem_charge: charge a kmem page to the current memory cgroup
* @page: page to charge * @page: page to charge
* @gfp: reclaim mode * @gfp: reclaim mode
* @order: allocation order * @order: allocation order
* *
* Returns 0 on success, an error code on failure. * Returns 0 on success, an error code on failure.
*/ */
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{ {
struct mem_cgroup *memcg; struct mem_cgroup *memcg;
int ret = 0; int ret = 0;
if (mem_cgroup_disabled() || memcg_kmem_bypass()) if (memcg_kmem_bypass())
return 0; return 0;
memcg = get_mem_cgroup_from_current(); memcg = get_mem_cgroup_from_current();
if (!mem_cgroup_is_root(memcg)) { if (!mem_cgroup_is_root(memcg)) {
ret = memcg_kmem_charge_memcg(page, gfp, order, memcg); ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
if (!ret) if (!ret)
__SetPageKmemcg(page); __SetPageKmemcg(page);
} }
...@@ -2629,11 +2629,11 @@ int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) ...@@ -2629,11 +2629,11 @@ int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
return ret; return ret;
} }
/** /**
* memcg_kmem_uncharge: uncharge a kmem page * __memcg_kmem_uncharge: uncharge a kmem page
* @page: page to uncharge * @page: page to uncharge
* @order: allocation order * @order: allocation order
*/ */
void memcg_kmem_uncharge(struct page *page, int order) void __memcg_kmem_uncharge(struct page *page, int order)
{ {
struct mem_cgroup *memcg = page->mem_cgroup; struct mem_cgroup *memcg = page->mem_cgroup;
unsigned int nr_pages = 1 << order; unsigned int nr_pages = 1 << order;
......
...@@ -1056,7 +1056,7 @@ static __always_inline bool free_pages_prepare(struct page *page, ...@@ -1056,7 +1056,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
if (PageMappingFlags(page)) if (PageMappingFlags(page))
page->mapping = NULL; page->mapping = NULL;
if (memcg_kmem_enabled() && PageKmemcg(page)) if (memcg_kmem_enabled() && PageKmemcg(page))
memcg_kmem_uncharge(page, order); __memcg_kmem_uncharge(page, order);
if (check_free) if (check_free)
bad += free_pages_check(page); bad += free_pages_check(page);
if (bad) if (bad)
...@@ -4568,7 +4568,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, ...@@ -4568,7 +4568,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
out: out:
if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) { unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) {
__free_pages(page, order); __free_pages(page, order);
page = NULL; page = NULL;
} }
......
...@@ -276,8 +276,6 @@ static __always_inline int memcg_charge_slab(struct page *page, ...@@ -276,8 +276,6 @@ static __always_inline int memcg_charge_slab(struct page *page,
gfp_t gfp, int order, gfp_t gfp, int order,
struct kmem_cache *s) struct kmem_cache *s)
{ {
if (!memcg_kmem_enabled())
return 0;
if (is_root_cache(s)) if (is_root_cache(s))
return 0; return 0;
return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg); return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);
...@@ -286,8 +284,6 @@ static __always_inline int memcg_charge_slab(struct page *page, ...@@ -286,8 +284,6 @@ static __always_inline int memcg_charge_slab(struct page *page,
static __always_inline void memcg_uncharge_slab(struct page *page, int order, static __always_inline void memcg_uncharge_slab(struct page *page, int order,
struct kmem_cache *s) struct kmem_cache *s)
{ {
if (!memcg_kmem_enabled())
return;
memcg_kmem_uncharge(page, order); memcg_kmem_uncharge(page, order);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment