Skip to content

L
linux
Commit 9826a516 authored by Michel Lespinasse's avatar Michel Lespinasse Committed by Linus Torvalds
Browse files
Options

mm: interval tree updates 

Update the generic interval tree code that was introduced in "mm: replace
vma prio_tree with an interval tree".

Changes:

- fixed 'endpoing' typo noticed by Andrew Morton

- replaced include/linux/interval_tree_tmpl.h, which was used as a
  template (including it automatically defined the interval tree
  functions) with include/linux/interval_tree_generic.h, which only
  defines a preprocessor macro INTERVAL_TREE_DEFINE(), which itself
  defines the interval tree functions when invoked. Now that is a very
  long macro which is unfortunate, but it does make the usage sites
  (lib/interval_tree.c and mm/interval_tree.c) a bit nicer than previously.

- make use of RB_DECLARE_CALLBACKS() in the INTERVAL_TREE_DEFINE() macro,
  instead of duplicating that code in the interval tree template.

- replaced vma_interval_tree_add(), which was actually handling the
  nonlinear and interval tree cases, with vma_interval_tree_insert_after()
  which handles only the interval tree case and has an API that is more
  consistent with the other interval tree handling functions.
  The nonlinear case is now handled explicitly in kernel/fork.c dup_mmap().
Signed-off-by: default avatarMichel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9c079add
Show whitespace changes
Inline Side-by-side
Showing with 235 additions and 263 deletions
include/linux/interval_tree_generic.h 0 → 100644
View file @ 9826a516
/*
Interval Trees
(C) 2012 Michel Lespinasse <walken@google.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
include/linux/interval_tree_generic.h
*/
#include <linux/rbtree_augmented.h>
/*
* Template for implementing interval trees
*
* ITSTRUCT: struct type of the interval tree nodes
* ITRB: name of struct rb_node field within ITSTRUCT
* ITTYPE: type of the interval endpoints
* ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree
* ITSTART(n): start endpoint of ITSTRUCT node n
* ITLAST(n): last endpoint of ITSTRUCT node n
* ITSTATIC: 'static' or empty
* ITPREFIX: prefix to use for the inline tree definitions
*
* Note - before using this, please consider if non-generic version
* (interval_tree.h) would work for you...
*/
#define INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \
ITSTART, ITLAST, ITSTATIC, ITPREFIX) \
\
/* Callbacks for augmented rbtree insert and remove */ \
\
static inline ITTYPE ITPREFIX ## _compute_subtree_last(ITSTRUCT *node) \
{ \
ITTYPE max = ITLAST(node), subtree_last; \
if (node->ITRB.rb_left) { \
subtree_last = rb_entry(node->ITRB.rb_left, \
ITSTRUCT, ITRB)->ITSUBTREE; \
if (max < subtree_last) \
max = subtree_last; \
} \
if (node->ITRB.rb_right) { \
subtree_last = rb_entry(node->ITRB.rb_right, \
ITSTRUCT, ITRB)->ITSUBTREE; \
if (max < subtree_last) \
max = subtree_last; \
} \
return max; \
} \
\
RB_DECLARE_CALLBACKS(static, ITPREFIX ## _augment, ITSTRUCT, ITRB, \
ITTYPE, ITSUBTREE, ITPREFIX ## _compute_subtree_last) \
\
/* Insert / remove interval nodes from the tree */ \
\
ITSTATIC void ITPREFIX ## _insert(ITSTRUCT *node, struct rb_root *root) \
{ \
struct rb_node **link = &root->rb_node, *rb_parent = NULL; \
ITTYPE start = ITSTART(node), last = ITLAST(node); \
ITSTRUCT *parent; \
\
while (*link) { \
rb_parent = *link; \
parent = rb_entry(rb_parent, ITSTRUCT, ITRB); \
if (parent->ITSUBTREE < last) \
parent->ITSUBTREE = last; \
if (start < ITSTART(parent)) \
link = &parent->ITRB.rb_left; \
else \
link = &parent->ITRB.rb_right; \
} \
\
node->ITSUBTREE = last; \
rb_link_node(&node->ITRB, rb_parent, link); \
rb_insert_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \
} \
\
ITSTATIC void ITPREFIX ## _remove(ITSTRUCT *node, struct rb_root *root) \
{ \
rb_erase_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \
} \
\
/* \
* Iterate over intervals intersecting [start;last] \
* \
* Note that a node's interval intersects [start;last] iff: \
* Cond1: ITSTART(node) <= last \
* and \
* Cond2: start <= ITLAST(node) \
*/ \
\
static ITSTRUCT * \
ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
{ \
while (true) { \
/* \
* Loop invariant: start <= node->ITSUBTREE \
* (Cond2 is satisfied by one of the subtree nodes) \
*/ \
if (node->ITRB.rb_left) { \
ITSTRUCT *left = rb_entry(node->ITRB.rb_left, \
ITSTRUCT, ITRB); \
if (start <= left->ITSUBTREE) { \
/* \
* Some nodes in left subtree satisfy Cond2. \
* Iterate to find the leftmost such node N. \
* If it also satisfies Cond1, that's the \
* match we are looking for. Otherwise, there \
* is no matching interval as nodes to the \
* right of N can't satisfy Cond1 either. \
*/ \
node = left; \
continue; \
} \
} \
if (ITSTART(node) <= last) { /* Cond1 */ \
if (start <= ITLAST(node)) /* Cond2 */ \
return node; /* node is leftmost match */ \
if (node->ITRB.rb_right) { \
node = rb_entry(node->ITRB.rb_right, \
ITSTRUCT, ITRB); \
if (start <= node->ITSUBTREE) \
continue; \
} \
} \
return NULL; /* No match */ \
} \
} \
\
ITSTATIC ITSTRUCT * \
ITPREFIX ## _iter_first(struct rb_root *root, ITTYPE start, ITTYPE last) \
{ \
ITSTRUCT *node; \
\
if (!root->rb_node) \
return NULL; \
node = rb_entry(root->rb_node, ITSTRUCT, ITRB); \
if (node->ITSUBTREE < start) \
return NULL; \
return ITPREFIX ## _subtree_search(node, start, last); \
} \
\
ITSTATIC ITSTRUCT * \
ITPREFIX ## _iter_next(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
{ \
struct rb_node *rb = node->ITRB.rb_right, *prev; \
\
while (true) { \
/* \
* Loop invariants: \
* Cond1: ITSTART(node) <= last \
* rb == node->ITRB.rb_right \
* \
* First, search right subtree if suitable \
*/ \
if (rb) { \
ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB); \
if (start <= right->ITSUBTREE) \
return ITPREFIX ## _subtree_search(right, \
start, last); \
} \
\
/* Move up the tree until we come from a node's left child */ \
do { \
rb = rb_parent(&node->ITRB); \
if (!rb) \
return NULL; \
prev = &node->ITRB; \
node = rb_entry(rb, ITSTRUCT, ITRB); \
rb = node->ITRB.rb_right; \
} while (prev == rb); \
\
/* Check if the node intersects [start;last] */ \
if (last < ITSTART(node)) /* !Cond1 */ \
return NULL; \
else if (start <= ITLAST(node)) /* Cond2 */ \
return node; \
} \
}
include/linux/interval_tree_tmpl.h deleted 100644 → 0
View file @ 9c079add
/*
Interval Trees
(C) 2012 Michel Lespinasse <walken@google.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
include/linux/interval_tree_tmpl.h
*/
#include <linux/rbtree_augmented.h>
/*
* Template for implementing interval trees
*
* ITSTRUCT: struct type of the interval tree nodes
* ITRB: name of struct rb_node field within ITSTRUCT
* ITTYPE: type of the interval endpoints
* ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree
* ITSTART(n): start endpoint of ITSTRUCT node n
* ITLAST(n): last endpoing of ITSTRUCT node n
* ITSTATIC: 'static' or empty
* ITPREFIX: prefix to use for the inline tree definitions
*/
/* IT(name) -> ITPREFIX_name */
#define _ITNAME(prefix, name) prefix ## _ ## name
#define ITNAME(prefix, name) _ITNAME(prefix, name)
#define IT(name) ITNAME(ITPREFIX, name)
/* Callbacks for augmented rbtree insert and remove */
static inline ITTYPE IT(compute_subtree_last)(ITSTRUCT *node)
{
ITTYPE max = ITLAST(node), subtree_last;
if (node->ITRB.rb_left) {
subtree_last = rb_entry(node->ITRB.rb_left,
ITSTRUCT, ITRB)->ITSUBTREE;
if (max < subtree_last)
max = subtree_last;
}
if (node->ITRB.rb_right) {
subtree_last = rb_entry(node->ITRB.rb_right,
ITSTRUCT, ITRB)->ITSUBTREE;
if (max < subtree_last)
max = subtree_last;
}
return max;
}
static inline void
IT(augment_propagate)(struct rb_node *rb, struct rb_node *stop)
{
while (rb != stop) {
ITSTRUCT *node = rb_entry(rb, ITSTRUCT, ITRB);
ITTYPE subtree_last = IT(compute_subtree_last)(node);
if (node->ITSUBTREE == subtree_last)
break;
node->ITSUBTREE = subtree_last;
rb = rb_parent(&node->ITRB);
}
}
static inline void
IT(augment_copy)(struct rb_node *rb_old, struct rb_node *rb_new)
{
ITSTRUCT *old = rb_entry(rb_old, ITSTRUCT, ITRB);
ITSTRUCT *new = rb_entry(rb_new, ITSTRUCT, ITRB);
new->ITSUBTREE = old->ITSUBTREE;
}
static void IT(augment_rotate)(struct rb_node *rb_old, struct rb_node *rb_new)
{
ITSTRUCT *old = rb_entry(rb_old, ITSTRUCT, ITRB);
ITSTRUCT *new = rb_entry(rb_new, ITSTRUCT, ITRB);
new->ITSUBTREE = old->ITSUBTREE;
old->ITSUBTREE = IT(compute_subtree_last)(old);
}
static const struct rb_augment_callbacks IT(augment_callbacks) = {
IT(augment_propagate), IT(augment_copy), IT(augment_rotate)
};
/* Insert / remove interval nodes from the tree */
ITSTATIC void IT(insert)(ITSTRUCT *node, struct rb_root *root)
{
struct rb_node **link = &root->rb_node, *rb_parent = NULL;
ITTYPE start = ITSTART(node), last = ITLAST(node);
ITSTRUCT *parent;
while (*link) {
rb_parent = *link;
parent = rb_entry(rb_parent, ITSTRUCT, ITRB);
if (parent->ITSUBTREE < last)
parent->ITSUBTREE = last;
if (start < ITSTART(parent))
link = &parent->ITRB.rb_left;
else
link = &parent->ITRB.rb_right;
}
node->ITSUBTREE = last;
rb_link_node(&node->ITRB, rb_parent, link);
rb_insert_augmented(&node->ITRB, root, &IT(augment_callbacks));
}
ITSTATIC void IT(remove)(ITSTRUCT *node, struct rb_root *root)
{
rb_erase_augmented(&node->ITRB, root, &IT(augment_callbacks));
}
/*
* Iterate over intervals intersecting [start;last]
*
* Note that a node's interval intersects [start;last] iff:
* Cond1: ITSTART(node) <= last
* and
* Cond2: start <= ITLAST(node)
*/
static ITSTRUCT *IT(subtree_search)(ITSTRUCT *node, ITTYPE start, ITTYPE last)
{
while (true) {
/*
* Loop invariant: start <= node->ITSUBTREE
* (Cond2 is satisfied by one of the subtree nodes)
*/
if (node->ITRB.rb_left) {
ITSTRUCT *left = rb_entry(node->ITRB.rb_left,
ITSTRUCT, ITRB);
if (start <= left->ITSUBTREE) {
/*
* Some nodes in left subtree satisfy Cond2.
* Iterate to find the leftmost such node N.
* If it also satisfies Cond1, that's the match
* we are looking for. Otherwise, there is no
* matching interval as nodes to the right of N
* can't satisfy Cond1 either.
*/
node = left;
continue;
}
}
if (ITSTART(node) <= last) { /* Cond1 */
if (start <= ITLAST(node)) /* Cond2 */
return node; /* node is leftmost match */
if (node->ITRB.rb_right) {
node = rb_entry(node->ITRB.rb_right,
ITSTRUCT, ITRB);
if (start <= node->ITSUBTREE)
continue;
}
}
return NULL; /* No match */
}
}
ITSTATIC ITSTRUCT *IT(iter_first)(struct rb_root *root,
ITTYPE start, ITTYPE last)
{
ITSTRUCT *node;
if (!root->rb_node)
return NULL;
node = rb_entry(root->rb_node, ITSTRUCT, ITRB);
if (node->ITSUBTREE < start)
return NULL;
return IT(subtree_search)(node, start, last);
}
ITSTATIC ITSTRUCT *IT(iter_next)(ITSTRUCT *node, ITTYPE start, ITTYPE last)
{
struct rb_node *rb = node->ITRB.rb_right, *prev;
while (true) {
/*
* Loop invariants:
* Cond1: ITSTART(node) <= last
* rb == node->ITRB.rb_right
*
* First, search right subtree if suitable
*/
if (rb) {
ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB);
if (start <= right->ITSUBTREE)
return IT(subtree_search)(right, start, last);
}
/* Move up the tree until we come from a node's left child */
do {
rb = rb_parent(&node->ITRB);
if (!rb)
return NULL;
prev = &node->ITRB;
node = rb_entry(rb, ITSTRUCT, ITRB);
rb = node->ITRB.rb_right;
} while (prev == rb);
/* Check if the node intersects [start;last] */
if (last < ITSTART(node)) /* !Cond1 */
return NULL;
else if (start <= ITLAST(node)) /* Cond2 */
return node;
}
}
include/linux/mm.h
View file @ 9826a516
...@@ -1355,11 +1355,11 @@ extern atomic_long_t mmap_pages_allocated; ...@@ -1355,11 +1355,11 @@ extern atomic_long_t mmap_pages_allocated;
extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
/* interval_tree.c */ /* interval_tree.c */
void vma_interval_tree_add(struct vm_area_struct *vma,
struct vm_area_struct *old,
struct address_space *mapping);
void vma_interval_tree_insert(struct vm_area_struct *node, void vma_interval_tree_insert(struct vm_area_struct *node,
struct rb_root *root); struct rb_root *root);
void vma_interval_tree_insert_after(struct vm_area_struct *node,
struct vm_area_struct *prev,
struct rb_root *root);
void vma_interval_tree_remove(struct vm_area_struct *node, void vma_interval_tree_remove(struct vm_area_struct *node,
struct rb_root *root); struct rb_root *root);
struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root, struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
......
kernel/fork.c
View file @ 9826a516
...@@ -423,7 +423,12 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) ...@@ -423,7 +423,12 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
mapping->i_mmap_writable++; mapping->i_mmap_writable++;
flush_dcache_mmap_lock(mapping); flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */ /* insert tmp into the share list, just after mpnt */
vma_interval_tree_add(tmp, mpnt, mapping); if (unlikely(tmp->vm_flags & VM_NONLINEAR))
vma_nonlinear_insert(tmp,
&mapping->i_mmap_nonlinear);
else
vma_interval_tree_insert_after(tmp, mpnt,
&mapping->i_mmap);
flush_dcache_mmap_unlock(mapping); flush_dcache_mmap_unlock(mapping);
mutex_unlock(&mapping->i_mmap_mutex); mutex_unlock(&mapping->i_mmap_mutex);
} }
......
lib/interval_tree.c
View file @ 9826a516
#include <linux/init.h> #include <linux/init.h>
#include <linux/interval_tree.h> #include <linux/interval_tree.h>
#include <linux/interval_tree_generic.h>
#define ITSTRUCT struct interval_tree_node #define START(node) ((node)->start)
#define ITRB rb #define LAST(node) ((node)->last)
#define ITTYPE unsigned long
#define ITSUBTREE __subtree_last
#define ITSTART(n) ((n)->start)
#define ITLAST(n) ((n)->last)
#define ITSTATIC
#define ITPREFIX interval_tree
#include <linux/interval_tree_tmpl.h> INTERVAL_TREE_DEFINE(struct interval_tree_node, rb,
unsigned long, __subtree_last,
START, LAST,, interval_tree)
mm/interval_tree.c
View file @ 9826a516
...@@ -8,40 +8,38 @@ ...@@ -8,40 +8,38 @@
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/interval_tree_generic.h>
#define ITSTRUCT struct vm_area_struct static inline unsigned long vma_start_pgoff(struct vm_area_struct *v)
#define ITRB shared.linear.rb {
#define ITTYPE unsigned long return v->vm_pgoff;
#define ITSUBTREE shared.linear.rb_subtree_last }
#define ITSTART(n) ((n)->vm_pgoff)
#define ITLAST(n) ((n)->vm_pgoff + \ static inline unsigned long vma_last_pgoff(struct vm_area_struct *v)
(((n)->vm_end - (n)->vm_start) >> PAGE_SHIFT) - 1) {
#define ITSTATIC return v->vm_pgoff + ((v->vm_end - v->vm_start) >> PAGE_SHIFT) - 1;
#define ITPREFIX vma_interval_tree }
#include <linux/interval_tree_tmpl.h> INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.linear.rb,
unsigned long, shared.linear.rb_subtree_last,
/* Insert old immediately after vma in the interval tree */ vma_start_pgoff, vma_last_pgoff,, vma_interval_tree)
void vma_interval_tree_add(struct vm_area_struct *vma,
struct vm_area_struct *old, /* Insert node immediately after prev in the interval tree */
struct address_space *mapping) void vma_interval_tree_insert_after(struct vm_area_struct *node,
struct vm_area_struct *prev,
struct rb_root *root)
{ {
struct rb_node **link; struct rb_node **link;
struct vm_area_struct *parent; struct vm_area_struct *parent;
unsigned long last; unsigned long last = vma_last_pgoff(node);
if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
list_add(&vma->shared.nonlinear, &old->shared.nonlinear);
return;
}
last = ITLAST(vma); VM_BUG_ON(vma_start_pgoff(node) != vma_start_pgoff(prev));
if (!old->shared.linear.rb.rb_right) { if (!prev->shared.linear.rb.rb_right) {
parent = old; parent = prev;
link = &old->shared.linear.rb.rb_right; link = &prev->shared.linear.rb.rb_right;
} else { } else {
parent = rb_entry(old->shared.linear.rb.rb_right, parent = rb_entry(prev->shared.linear.rb.rb_right,
struct vm_area_struct, shared.linear.rb); struct vm_area_struct, shared.linear.rb);
if (parent->shared.linear.rb_subtree_last < last) if (parent->shared.linear.rb_subtree_last < last)
parent->shared.linear.rb_subtree_last = last; parent->shared.linear.rb_subtree_last = last;
...@@ -54,8 +52,8 @@ void vma_interval_tree_add(struct vm_area_struct *vma, ...@@ -54,8 +52,8 @@ void vma_interval_tree_add(struct vm_area_struct *vma,
link = &parent->shared.linear.rb.rb_left; link = &parent->shared.linear.rb.rb_left;
} }
vma->shared.linear.rb_subtree_last = last; node->shared.linear.rb_subtree_last = last;
rb_link_node(&vma->shared.linear.rb, &parent->shared.linear.rb, link); rb_link_node(&node->shared.linear.rb, &parent->shared.linear.rb, link);
rb_insert_augmented(&vma->shared.linear.rb, &mapping->i_mmap, rb_insert_augmented(&node->shared.linear.rb, root,
&vma_interval_tree_augment_callbacks); &vma_interval_tree_augment);
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7