Skip to content

C
cpython
Commit aac4d034 authored by Pablo Galindo's avatar Pablo Galindo Committed by GitHub
Browse files
Options

bpo-26826: Expose copy_file_range in the os module (GH-7255)

parent 545a3b88
Hide whitespace changes
Inline Side-by-side
Showing with 363 additions and 19 deletions
Doc/library/os.rst
View file @ aac4d034
......@@ -707,6 +707,28 @@ as internal buffering of data.
pass
.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None)
Copy *count* bytes from file descriptor *src*, starting from offset
*offset_src*, to file descriptor *dst*, starting from offset *offset_dst*.
If *offset_src* is None, then *src* is read from the current position;
respectively for *offset_dst*. The files pointed by *src* and *dst*
must reside in the same filesystem, otherwise an :exc:`OSError` is
raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`.
This copy is done without the additional cost of transferring data
from the kernel to user space and then back into the kernel. Additionally,
some filesystems could implement extra optimizations. The copy is done as if
both files are opened as binary.
The return value is the amount of bytes copied. This could be less than the
amount requested.
.. availability:: Linux kernel >= 4.5 or glibc >= 2.27.
.. versionadded:: 3.8
.. function:: device_encoding(fd)
Return a string describing the encoding of the device associated with *fd*
......
Lib/test/test_os.py
View file @ aac4d034
......@@ -231,6 +231,89 @@ class FileTests(unittest.TestCase):
except (NotImplementedError, OSError):
pass # No OS support or unprivileged user
@unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
def test_copy_file_range_invalid_values(self):
with self.assertRaises(ValueError):
os.copy_file_range(0, 1, -10)
@unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
def test_copy_file_range(self):
TESTFN2 = support.TESTFN + ".3"
data = b'0123456789'
create_file(support.TESTFN, data)
self.addCleanup(support.unlink, support.TESTFN)
in_file = open(support.TESTFN, 'rb')
self.addCleanup(in_file.close)
in_fd = in_file.fileno()
out_file = open(TESTFN2, 'w+b')
self.addCleanup(support.unlink, TESTFN2)
self.addCleanup(out_file.close)
out_fd = out_file.fileno()
try:
i = os.copy_file_range(in_fd, out_fd, 5)
except OSError as e:
# Handle the case in which Python was compiled
# in a system with the syscall but without support
# in the kernel.
if e.errno != errno.ENOSYS:
raise
self.skipTest(e)
else:
# The number of copied bytes can be less than
# the number of bytes originally requested.
self.assertIn(i, range(0, 6));
with open(TESTFN2, 'rb') as in_file:
self.assertEqual(in_file.read(), data[:i])
@unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
def test_copy_file_range_offset(self):
TESTFN4 = support.TESTFN + ".4"
data = b'0123456789'
bytes_to_copy = 6
in_skip = 3
out_seek = 5
create_file(support.TESTFN, data)
self.addCleanup(support.unlink, support.TESTFN)
in_file = open(support.TESTFN, 'rb')
self.addCleanup(in_file.close)
in_fd = in_file.fileno()
out_file = open(TESTFN4, 'w+b')
self.addCleanup(support.unlink, TESTFN4)
self.addCleanup(out_file.close)
out_fd = out_file.fileno()
try:
i = os.copy_file_range(in_fd, out_fd, bytes_to_copy,
offset_src=in_skip,
offset_dst=out_seek)
except OSError as e:
# Handle the case in which Python was compiled
# in a system with the syscall but without support
# in the kernel.
if e.errno != errno.ENOSYS:
raise
self.skipTest(e)
else:
# The number of copied bytes can be less than
# the number of bytes originally requested.
self.assertIn(i, range(0, bytes_to_copy+1));
with open(TESTFN4, 'rb') as in_file:
read = in_file.read()
# seeked bytes (5) are zero'ed
self.assertEqual(read[:out_seek], b'\x00'*out_seek)
# 012 are skipped (in_skip)
# 345678 are copied in the file (in_skip + bytes_to_copy)
self.assertEqual(read[out_seek:],
data[in_skip:in_skip+i])
# Test attributes on return values from os.*stat* family.
class StatAttributeTests(unittest.TestCase):
......
Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst 0 → 100644
View file @ aac4d034
Expose :func:`copy_file_range` as a low level API in the :mod:`os` module.
Modules/clinic/posixmodule.c.h
View file @ aac4d034
......@@ -5395,6 +5395,108 @@ exit:
#endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */
#if defined(HAVE_COPY_FILE_RANGE)
PyDoc_STRVAR(os_copy_file_range__doc__,
"copy_file_range($module, /, src, dst, count, offset_src=None,\n"
" offset_dst=None)\n"
"--\n"
"\n"
"Copy count bytes from one file descriptor to another.\n"
"\n"
" src\n"
" Source file descriptor.\n"
" dst\n"
" Destination file descriptor.\n"
" count\n"
" Number of bytes to copy.\n"
" offset_src\n"
" Starting offset in src.\n"
" offset_dst\n"
" Starting offset in dst.\n"
"\n"
"If offset_src is None, then src is read from the current position;\n"
"respectively for offset_dst.");
#define OS_COPY_FILE_RANGE_METHODDEF \
{"copy_file_range", (PyCFunction)(void(*)(void))os_copy_file_range, METH_FASTCALL|METH_KEYWORDS, os_copy_file_range__doc__},
static PyObject *
os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count,
PyObject *offset_src, PyObject *offset_dst);
static PyObject *
os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
static const char * const _keywords[] = {"src", "dst", "count", "offset_src", "offset_dst", NULL};
static _PyArg_Parser _parser = {NULL, _keywords, "copy_file_range", 0};
PyObject *argsbuf[5];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3;
int src;
int dst;
Py_ssize_t count;
PyObject *offset_src = Py_None;
PyObject *offset_dst = Py_None;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 5, 0, argsbuf);
if (!args) {
goto exit;
}
if (PyFloat_Check(args[0])) {
PyErr_SetString(PyExc_TypeError,
"integer argument expected, got float" );
goto exit;
}
src = _PyLong_AsInt(args[0]);
if (src == -1 && PyErr_Occurred()) {
goto exit;
}
if (PyFloat_Check(args[1])) {
PyErr_SetString(PyExc_TypeError,
"integer argument expected, got float" );
goto exit;
}
dst = _PyLong_AsInt(args[1]);
if (dst == -1 && PyErr_Occurred()) {
goto exit;
}
if (PyFloat_Check(args[2])) {
PyErr_SetString(PyExc_TypeError,
"integer argument expected, got float" );
goto exit;
}
{
Py_ssize_t ival = -1;
PyObject *iobj = PyNumber_Index(args[2]);
if (iobj != NULL) {
ival = PyLong_AsSsize_t(iobj);
Py_DECREF(iobj);
}
if (ival == -1 && PyErr_Occurred()) {
goto exit;
}
count = ival;
}
if (!noptargs) {
goto skip_optional_pos;
}
if (args[3]) {
offset_src = args[3];
if (!--noptargs) {
goto skip_optional_pos;
}
}
offset_dst = args[4];
skip_optional_pos:
return_value = os_copy_file_range_impl(module, src, dst, count, offset_src, offset_dst);
exit:
return return_value;
}
#endif /* defined(HAVE_COPY_FILE_RANGE) */
#if defined(HAVE_MKFIFO)
PyDoc_STRVAR(os_mkfifo__doc__,
......@@ -8460,6 +8562,10 @@ exit:
#define OS_PWRITEV_METHODDEF
#endif /* !defined(OS_PWRITEV_METHODDEF) */
#ifndef OS_COPY_FILE_RANGE_METHODDEF
#define OS_COPY_FILE_RANGE_METHODDEF
#endif /* !defined(OS_COPY_FILE_RANGE_METHODDEF) */
#ifndef OS_MKFIFO_METHODDEF
#define OS_MKFIFO_METHODDEF
#endif /* !defined(OS_MKFIFO_METHODDEF) */
......@@ -8635,4 +8741,4 @@ exit:
#ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
#define OS__REMOVE_DLL_DIRECTORY_METHODDEF
#endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
/*[clinic end generated code: output=855b81aafd05beed input=a9049054013a1b77]*/
/*[clinic end generated code: output=b3ae8afd275ea5cd input=a9049054013a1b77]*/
Modules/posixmodule.c
View file @ aac4d034
......@@ -117,6 +117,10 @@ corresponding Unix manual entries for more information on calls.");
#include <sched.h>
#endif
#ifdef HAVE_COPY_FILE_RANGE
#include <unistd.h>
#endif
#if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY)
#undef HAVE_SCHED_SETAFFINITY
#endif
......@@ -9455,8 +9459,74 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset,
}
#endif /* HAVE_PWRITEV */
#ifdef HAVE_COPY_FILE_RANGE
/*[clinic input]
os.copy_file_range
src: int
Source file descriptor.
dst: int
Destination file descriptor.
count: Py_ssize_t
Number of bytes to copy.
offset_src: object = None
Starting offset in src.
offset_dst: object = None
Starting offset in dst.
Copy count bytes from one file descriptor to another.
If offset_src is None, then src is read from the current position;
respectively for offset_dst.
[clinic start generated code]*/
static PyObject *
os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count,
PyObject *offset_src, PyObject *offset_dst)
/*[clinic end generated code: output=1a91713a1d99fc7a input=42fdce72681b25a9]*/
{
off_t offset_src_val, offset_dst_val;
off_t *p_offset_src = NULL;
off_t *p_offset_dst = NULL;
Py_ssize_t ret;
int async_err = 0;
/* The flags argument is provided to allow
* for future extensions and currently must be to 0. */
int flags = 0;
if (count < 0) {
PyErr_SetString(PyExc_ValueError, "negative value for 'count' not allowed");
return NULL;
}
if (offset_src != Py_None) {
if (!Py_off_t_converter(offset_src, &offset_src_val)) {
return NULL;
}
p_offset_src = &offset_src_val;
}
if (offset_dst != Py_None) {
if (!Py_off_t_converter(offset_dst, &offset_dst_val)) {
return NULL;
}
p_offset_dst = &offset_dst_val;
}
do {
Py_BEGIN_ALLOW_THREADS
ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags);
Py_END_ALLOW_THREADS
} while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
if (ret < 0) {
return (!async_err) ? posix_error() : NULL;
}
return PyLong_FromSsize_t(ret);
}
#endif /* HAVE_COPY_FILE_RANGE*/
#ifdef HAVE_MKFIFO
/*[clinic input]
......@@ -13432,6 +13502,7 @@ static PyMethodDef posix_methods[] = {
OS_POSIX_SPAWN_METHODDEF
OS_POSIX_SPAWNP_METHODDEF
OS_READLINK_METHODDEF
OS_COPY_FILE_RANGE_METHODDEF
OS_RENAME_METHODDEF
OS_REPLACE_METHODDEF
OS_RMDIR_METHODDEF
......
aclocal.m4
View file @ aac4d034
......@@ -12,9 +12,9 @@
# PARTICULAR PURPOSE.
m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])])
dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
dnl serial 11 (pkg-config-0.29.1)
dnl
# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
# serial 11 (pkg-config-0.29.1)
dnl Copyright © 2004 Scott James Remnant <scott@netsplit.com>.
dnl Copyright © 2012-2015 Dan Nicholson <dbn.lists@gmail.com>
dnl
......@@ -288,5 +288,73 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
AS_VAR_IF([$1], [""], [$5], [$4])dnl
])dnl PKG_CHECK_VAR
dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
dnl [DESCRIPTION], [DEFAULT])
dnl ------------------------------------------
dnl
dnl Prepare a "--with-" configure option using the lowercase
dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
dnl PKG_CHECK_MODULES in a single macro.
AC_DEFUN([PKG_WITH_MODULES],
[
m4_pushdef([with_arg], m4_tolower([$1]))
m4_pushdef([description],
[m4_default([$5], [build with ]with_arg[ support])])
m4_pushdef([def_arg], [m4_default([$6], [auto])])
m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
m4_case(def_arg,
[yes],[m4_pushdef([with_without], [--without-]with_arg)],
[m4_pushdef([with_without],[--with-]with_arg)])
AC_ARG_WITH(with_arg,
AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
[AS_TR_SH([with_]with_arg)=def_arg])
AS_CASE([$AS_TR_SH([with_]with_arg)],
[yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
[auto],[PKG_CHECK_MODULES([$1],[$2],
[m4_n([def_action_if_found]) $3],
[m4_n([def_action_if_not_found]) $4])])
m4_popdef([with_arg])
m4_popdef([description])
m4_popdef([def_arg])
])dnl PKG_WITH_MODULES
dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
dnl [DESCRIPTION], [DEFAULT])
dnl -----------------------------------------------
dnl
dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
dnl check._[VARIABLE-PREFIX] is exported as make variable.
AC_DEFUN([PKG_HAVE_WITH_MODULES],
[
PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
AM_CONDITIONAL([HAVE_][$1],
[test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
])dnl PKG_HAVE_WITH_MODULES
dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
dnl [DESCRIPTION], [DEFAULT])
dnl ------------------------------------------------------
dnl
dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
dnl and preprocessor variable.
AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
[
PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
[AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
])dnl PKG_HAVE_DEFINE_WITH_MODULES
m4_include([m4/ax_c_float_words_bigendian.m4])
m4_include([m4/ax_check_openssl.m4])
configure
View file @ aac4d034
......@@ -785,7 +785,6 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
......@@ -898,7 +897,6 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
......@@ -1151,15 +1149,6 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
......@@ -1297,7 +1286,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir runstatedir
libdir localedir mandir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
......@@ -1450,7 +1439,6 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
......@@ -11476,7 +11464,8 @@ fi
# checks for library functions
for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \
clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
faccessat fchmod fchmodat fchown fchownat \
fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
futimens futimes gai_strerror getentropy \
getgrgid_r getgrnam_r \
......
configure.ac
View file @ aac4d034
......@@ -3520,7 +3520,8 @@ fi
# checks for library functions
AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \
clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
faccessat fchmod fchmodat fchown fchownat \
fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
futimens futimes gai_strerror getentropy \
getgrgid_r getgrnam_r \
......
pyconfig.h.in
View file @ aac4d034
......@@ -148,6 +148,9 @@
/* Define to 1 if you have the `copysign' function. */
#undef HAVE_COPYSIGN
/* Define to 1 if you have the `copy_file_range' function. */
#undef HAVE_COPY_FILE_RANGE
/* Define to 1 if you have the <crypt.h> header file. */
#undef HAVE_CRYPT_H
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7