Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
L
linux
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
linux
Commits
98797a24
Commit
98797a24
authored
Jun 14, 2009
by
Russell King
Committed by
Russell King
Jun 14, 2009
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'copy_user' of
git://git.marvell.com/orion
into devel
parents
ca8cbc83
c626e3f5
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
253 additions
and
2 deletions
+253
-2
arch/arm/Kconfig
arch/arm/Kconfig
+16
-0
arch/arm/include/asm/uaccess.h
arch/arm/include/asm/uaccess.h
+2
-0
arch/arm/lib/Makefile
arch/arm/lib/Makefile
+3
-0
arch/arm/lib/clear_user.S
arch/arm/lib/clear_user.S
+2
-1
arch/arm/lib/copy_to_user.S
arch/arm/lib/copy_to_user.S
+2
-1
arch/arm/lib/uaccess_with_memcpy.c
arch/arm/lib/uaccess_with_memcpy.c
+228
-0
No files found.
arch/arm/Kconfig
View file @
98797a24
...
@@ -1091,6 +1091,22 @@ config ALIGNMENT_TRAP
...
@@ -1091,6 +1091,22 @@ config ALIGNMENT_TRAP
correct operation of some network protocols. With an IP-only
correct operation of some network protocols. With an IP-only
configuration it is safe to say N, otherwise say Y.
configuration it is safe to say N, otherwise say Y.
config UACCESS_WITH_MEMCPY
bool "Use kernel mem{cpy,set}() for {copy_to,clear}_user() (EXPERIMENTAL)"
depends on MMU && EXPERIMENTAL
default y if CPU_FEROCEON
help
Implement faster copy_to_user and clear_user methods for CPU
cores where a 8-word STM instruction give significantly higher
memory write throughput than a sequence of individual 32bit stores.
A possible side effect is a slight increase in scheduling latency
between threads sharing the same address space if they invoke
such copy operations with large buffers.
However, if the CPU data cache is using a write-allocate mode,
this option is unlikely to provide any performance gain.
endmenu
endmenu
menu "Boot options"
menu "Boot options"
...
...
arch/arm/include/asm/uaccess.h
View file @
98797a24
...
@@ -386,7 +386,9 @@ do { \
...
@@ -386,7 +386,9 @@ do { \
#ifdef CONFIG_MMU
#ifdef CONFIG_MMU
extern
unsigned
long
__must_check
__copy_from_user
(
void
*
to
,
const
void
__user
*
from
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__copy_from_user
(
void
*
to
,
const
void
__user
*
from
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__copy_to_user
(
void
__user
*
to
,
const
void
*
from
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__copy_to_user
(
void
__user
*
to
,
const
void
*
from
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__copy_to_user_std
(
void
__user
*
to
,
const
void
*
from
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__clear_user
(
void
__user
*
addr
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__clear_user
(
void
__user
*
addr
,
unsigned
long
n
);
extern
unsigned
long
__must_check
__clear_user_std
(
void
__user
*
addr
,
unsigned
long
n
);
#else
#else
#define __copy_from_user(to,from,n) (memcpy(to, (void __force *)from, n), 0)
#define __copy_from_user(to,from,n) (memcpy(to, (void __force *)from, n), 0)
#define __copy_to_user(to,from,n) (memcpy((void __force *)to, from, n), 0)
#define __copy_to_user(to,from,n) (memcpy((void __force *)to, from, n), 0)
...
...
arch/arm/lib/Makefile
View file @
98797a24
...
@@ -29,6 +29,9 @@ else
...
@@ -29,6 +29,9 @@ else
endif
endif
endif
endif
# using lib_ here won't override already available weak symbols
obj-$(CONFIG_UACCESS_WITH_MEMCPY)
+=
uaccess_with_memcpy.o
lib-$(CONFIG_MMU)
+=
$
(
mmu-y
)
lib-$(CONFIG_MMU)
+=
$
(
mmu-y
)
ifeq
($(CONFIG_CPU_32v3),y)
ifeq
($(CONFIG_CPU_32v3),y)
...
...
arch/arm/lib/clear_user.S
View file @
98797a24
...
@@ -18,7 +18,8 @@
...
@@ -18,7 +18,8 @@
*
:
sz
-
number
of
bytes
to
clear
*
:
sz
-
number
of
bytes
to
clear
*
Returns
:
number
of
bytes
NOT
cleared
*
Returns
:
number
of
bytes
NOT
cleared
*/
*/
ENTRY
(
__clear_user
)
ENTRY
(
__clear_user_std
)
WEAK
(
__clear_user
)
stmfd
sp
!,
{
r1
,
lr
}
stmfd
sp
!,
{
r1
,
lr
}
mov
r2
,
#
0
mov
r2
,
#
0
cmp
r1
,
#
4
cmp
r1
,
#
4
...
...
arch/arm/lib/copy_to_user.S
View file @
98797a24
...
@@ -86,7 +86,8 @@
...
@@ -86,7 +86,8 @@
.
text
.
text
ENTRY
(
__copy_to_user
)
ENTRY
(
__copy_to_user_std
)
WEAK
(
__copy_to_user
)
#include "copy_template.S"
#include "copy_template.S"
...
...
arch/arm/lib/uaccess_with_memcpy.c
0 → 100644
View file @
98797a24
/*
* linux/arch/arm/lib/uaccess_with_memcpy.c
*
* Written by: Lennert Buytenhek and Nicolas Pitre
* Copyright (C) 2009 Marvell Semiconductor
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/kernel.h>
#include <linux/ctype.h>
#include <linux/uaccess.h>
#include <linux/rwsem.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/hardirq.h>
/* for in_atomic() */
#include <asm/current.h>
#include <asm/page.h>
static
int
pin_page_for_write
(
const
void
__user
*
_addr
,
pte_t
**
ptep
,
spinlock_t
**
ptlp
)
{
unsigned
long
addr
=
(
unsigned
long
)
_addr
;
pgd_t
*
pgd
;
pmd_t
*
pmd
;
pte_t
*
pte
;
spinlock_t
*
ptl
;
pgd
=
pgd_offset
(
current
->
mm
,
addr
);
if
(
unlikely
(
pgd_none
(
*
pgd
)
||
pgd_bad
(
*
pgd
)))
return
0
;
pmd
=
pmd_offset
(
pgd
,
addr
);
if
(
unlikely
(
pmd_none
(
*
pmd
)
||
pmd_bad
(
*
pmd
)))
return
0
;
pte
=
pte_offset_map_lock
(
current
->
mm
,
pmd
,
addr
,
&
ptl
);
if
(
unlikely
(
!
pte_present
(
*
pte
)
||
!
pte_young
(
*
pte
)
||
!
pte_write
(
*
pte
)
||
!
pte_dirty
(
*
pte
)))
{
pte_unmap_unlock
(
pte
,
ptl
);
return
0
;
}
*
ptep
=
pte
;
*
ptlp
=
ptl
;
return
1
;
}
static
unsigned
long
noinline
__copy_to_user_memcpy
(
void
__user
*
to
,
const
void
*
from
,
unsigned
long
n
)
{
int
atomic
;
if
(
unlikely
(
segment_eq
(
get_fs
(),
KERNEL_DS
)))
{
memcpy
((
void
*
)
to
,
from
,
n
);
return
0
;
}
/* the mmap semaphore is taken only if not in an atomic context */
atomic
=
in_atomic
();
if
(
!
atomic
)
down_read
(
&
current
->
mm
->
mmap_sem
);
while
(
n
)
{
pte_t
*
pte
;
spinlock_t
*
ptl
;
int
tocopy
;
while
(
!
pin_page_for_write
(
to
,
&
pte
,
&
ptl
))
{
if
(
!
atomic
)
up_read
(
&
current
->
mm
->
mmap_sem
);
if
(
__put_user
(
0
,
(
char
__user
*
)
to
))
goto
out
;
if
(
!
atomic
)
down_read
(
&
current
->
mm
->
mmap_sem
);
}
tocopy
=
(
~
(
unsigned
long
)
to
&
~
PAGE_MASK
)
+
1
;
if
(
tocopy
>
n
)
tocopy
=
n
;
memcpy
((
void
*
)
to
,
from
,
tocopy
);
to
+=
tocopy
;
from
+=
tocopy
;
n
-=
tocopy
;
pte_unmap_unlock
(
pte
,
ptl
);
}
if
(
!
atomic
)
up_read
(
&
current
->
mm
->
mmap_sem
);
out:
return
n
;
}
unsigned
long
__copy_to_user
(
void
__user
*
to
,
const
void
*
from
,
unsigned
long
n
)
{
/*
* This test is stubbed out of the main function above to keep
* the overhead for small copies low by avoiding a large
* register dump on the stack just to reload them right away.
* With frame pointer disabled, tail call optimization kicks in
* as well making this test almost invisible.
*/
if
(
n
<
64
)
return
__copy_to_user_std
(
to
,
from
,
n
);
return
__copy_to_user_memcpy
(
to
,
from
,
n
);
}
static
unsigned
long
noinline
__clear_user_memset
(
void
__user
*
addr
,
unsigned
long
n
)
{
if
(
unlikely
(
segment_eq
(
get_fs
(),
KERNEL_DS
)))
{
memset
((
void
*
)
addr
,
0
,
n
);
return
0
;
}
down_read
(
&
current
->
mm
->
mmap_sem
);
while
(
n
)
{
pte_t
*
pte
;
spinlock_t
*
ptl
;
int
tocopy
;
while
(
!
pin_page_for_write
(
addr
,
&
pte
,
&
ptl
))
{
up_read
(
&
current
->
mm
->
mmap_sem
);
if
(
__put_user
(
0
,
(
char
__user
*
)
addr
))
goto
out
;
down_read
(
&
current
->
mm
->
mmap_sem
);
}
tocopy
=
(
~
(
unsigned
long
)
addr
&
~
PAGE_MASK
)
+
1
;
if
(
tocopy
>
n
)
tocopy
=
n
;
memset
((
void
*
)
addr
,
0
,
tocopy
);
addr
+=
tocopy
;
n
-=
tocopy
;
pte_unmap_unlock
(
pte
,
ptl
);
}
up_read
(
&
current
->
mm
->
mmap_sem
);
out:
return
n
;
}
unsigned
long
__clear_user
(
void
__user
*
addr
,
unsigned
long
n
)
{
/* See rational for this in __copy_to_user() above. */
if
(
n
<
64
)
return
__clear_user_std
(
addr
,
n
);
return
__clear_user_memset
(
addr
,
n
);
}
#if 0
/*
* This code is disabled by default, but kept around in case the chosen
* thresholds need to be revalidated. Some overhead (small but still)
* would be implied by a runtime determined variable threshold, and
* so far the measurement on concerned targets didn't show a worthwhile
* variation.
*
* Note that a fairly precise sched_clock() implementation is needed
* for results to make some sense.
*/
#include <linux/vmalloc.h>
static int __init test_size_treshold(void)
{
struct page *src_page, *dst_page;
void *user_ptr, *kernel_ptr;
unsigned long long t0, t1, t2;
int size, ret;
ret = -ENOMEM;
src_page = alloc_page(GFP_KERNEL);
if (!src_page)
goto no_src;
dst_page = alloc_page(GFP_KERNEL);
if (!dst_page)
goto no_dst;
kernel_ptr = page_address(src_page);
user_ptr = vmap(&dst_page, 1, VM_IOREMAP, __pgprot(__P010));
if (!user_ptr)
goto no_vmap;
/* warm up the src page dcache */
ret = __copy_to_user_memcpy(user_ptr, kernel_ptr, PAGE_SIZE);
for (size = PAGE_SIZE; size >= 4; size /= 2) {
t0 = sched_clock();
ret |= __copy_to_user_memcpy(user_ptr, kernel_ptr, size);
t1 = sched_clock();
ret |= __copy_to_user_std(user_ptr, kernel_ptr, size);
t2 = sched_clock();
printk("copy_to_user: %d %llu %llu\n", size, t1 - t0, t2 - t1);
}
for (size = PAGE_SIZE; size >= 4; size /= 2) {
t0 = sched_clock();
ret |= __clear_user_memset(user_ptr, size);
t1 = sched_clock();
ret |= __clear_user_std(user_ptr, size);
t2 = sched_clock();
printk("clear_user: %d %llu %llu\n", size, t1 - t0, t2 - t1);
}
if (ret)
ret = -EFAULT;
vunmap(user_ptr);
no_vmap:
put_page(dst_page);
no_dst:
put_page(src_page);
no_src:
return ret;
}
subsys_initcall(test_size_treshold);
#endif
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment