Commit 8b232c01 authored by Jérome Perrin's avatar Jérome Perrin

librecipe/execute: fallback to polling when inotify fails

When user ran out of inotify watch, starting wrappers using
wait-for-files is not possible:

    AssertionError: Traceback (most recent call last):
      File "/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wrapper", line 13, in <module>
        sys.exit(slapos.recipe.librecipe.execute.generic_exec(['/bin/echo', 'done'], wait_list=['/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wait']))
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 100, in generic_exec
        _wait_files_creation(wait_list)
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in _wait_files_creation
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in <dictcomp>
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 110, in add_watch
        return _libc_call(_libc.inotify_add_watch, self.fileno(), fsencode(path), mask)
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 39, in _libc_call
        raise OSError(errno, os.strerror(errno))
    OSError: [Errno 28] No space left on device

This catches inotify errors and fallback to simple polling in that case.
parent 8363516d
Pipeline #26278 failed with stage
in 0 seconds
...@@ -5,6 +5,7 @@ import sys ...@@ -5,6 +5,7 @@ import sys
import os import os
import signal import signal
import subprocess import subprocess
import time
from collections import defaultdict from collections import defaultdict
from inotify_simple import INotify, flags from inotify_simple import INotify, flags
...@@ -14,24 +15,34 @@ def _wait_files_creation(file_list): ...@@ -14,24 +15,34 @@ def _wait_files_creation(file_list):
# Establish a list of directory and subfiles. # Establish a list of directory and subfiles.
# and test existence before watching, so that we don't miss an event. # and test existence before watching, so that we don't miss an event.
directories = defaultdict(dict) directories = defaultdict(dict)
for f in file_list: def check_if_files_exists():
dirname, filename = os.path.split(f) for f in file_list:
directories[dirname][filename] = os.path.lexists(f) dirname, filename = os.path.split(f)
directories[dirname][filename] = os.path.lexists(f)
check_if_files_exists()
def all_files_exists(): def all_files_exists():
return all(all(six.itervalues(files)) for files in six.itervalues(directories)) return all(all(six.itervalues(files)) for files in six.itervalues(directories))
with INotify() as inotify: with INotify() as inotify:
watchdescriptors = {inotify.add_watch(dirname, try:
flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM watchdescriptors = {inotify.add_watch(dirname,
): dirname flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
for dirname in directories} ): dirname
for dirname in directories}
while not all_files_exists(): except OSError as e:
for event in inotify.read(): if e.errno not in (errno.ENOSPC, errno.EMFILE):
directory = directories[watchdescriptors[event.wd]] raise
if event.name in directory: print('Error using inotify, falling back to polling')
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO) while not all_files_exists():
time.sleep(0.1)
check_if_files_exists()
else:
while not all_files_exists():
for event in inotify.read():
directory = directories[watchdescriptors[event.wd]]
if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def _libc(): def _libc():
from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
......
...@@ -175,6 +175,12 @@ class TestPidFile(WrapperTestCase): ...@@ -175,6 +175,12 @@ class TestPidFile(WrapperTestCase):
class TestWaitForFiles(WrapperTestCase): class TestWaitForFiles(WrapperTestCase):
env = None
if sys.platform.startswith("linux"):
expected_output = 'done\n'
else:
expected_output = 'Error using inotify, falling back to polling\ndone\n'
def getOptions(self): def getOptions(self):
self.waitfile = self.getTempPath('wait') self.waitfile = self.getTempPath('wait')
return { return {
...@@ -192,6 +198,7 @@ class TestWaitForFiles(WrapperTestCase): ...@@ -192,6 +198,7 @@ class TestWaitForFiles(WrapperTestCase):
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
universal_newlines=True, universal_newlines=True,
env=self.env,
) )
self.addCleanup(self.terminate_process, process) self.addCleanup(self.terminate_process, process)
if process.poll(): if process.poll():
...@@ -207,13 +214,57 @@ class TestWaitForFiles(WrapperTestCase): ...@@ -207,13 +214,57 @@ class TestWaitForFiles(WrapperTestCase):
for _ in range(20): for _ in range(20):
time.sleep(0.1) time.sleep(0.1)
if process.poll() is not None: if process.poll() is not None:
self.assertEqual(process.stdout.read(), 'done\n') self.assertEqual(process.stdout.read(), self.expected_output)
self.assertEqual(process.returncode, 0) self.assertEqual(process.returncode, 0)
break break
else: else:
self.fail('process did not start after file was created') self.fail('process did not start after file was created')
@unittest.skipUnless(sys.platform.startswith("linux"), "Inotify is linux only")
class TestWaitForFilesInotifyError(TestWaitForFiles):
def setUp(self):
super(TestWaitForFilesInotifyError, self).setUp()
# use LD_PRELOAD to inject errors into inotify_add_watch calls
inotify_mock_c = self.getTempPath('inotify_mock.c')
inotify_mock_o = self.getTempPath('inotify_mock.o')
inotify_mock_so = self.getTempPath('inotify_mock.so')
with open(inotify_mock_c, 'w') as f:
f.write('''
#include <sys/inotify.h>
#include <string.h>
#include <errno.h>
int inotify_add_watch(int fd, const char *pathname, uint32_t mask) {
errno = ENOSPC;
return -1;
}
/* This is a bit tricky because inotify_simple calls
inotify_add_watch with ctypes.CDLL("libc.so"), which uses
dlopen("libc.so") and dlsym("inotify_add_watch"), so we first
override dlsym to return our own inotify_add_watch.
https://github.com/chrisjbillington/inotify_simple/blob/55737898/inotify_simple.py#L110
*/
extern void *__libc_dlsym (void *, const char *);
void *dlsym(void *handle, const char *symbol) {
if (strcmp(symbol, "inotify_add_watch") == 0) {
return (void *)inotify_add_watch;
}
return (void *)__libc_dlsym(handle, symbol);
}
''')
subprocess.check_call(['gcc', '-c', '-fPIC', '-o', inotify_mock_o, inotify_mock_c])
subprocess.check_call(['gcc', '-shared', '-o', inotify_mock_so, inotify_mock_o])
self.env = dict(
os.environ,
PYTHONUNBUFFERED='1',
LD_PRELOAD=inotify_mock_so)
expected_output = 'Error using inotify, falling back to polling\ndone\n'
class TestPrivateTmpFS(WrapperTestCase): class TestPrivateTmpFS(WrapperTestCase):
def getOptions(self): def getOptions(self):
self.tmpdir = self.getTempPath('tmpdir') self.tmpdir = self.getTempPath('tmpdir')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment