limit number of openable files (#131)

Unlimited (e.g. 2**30) ulimit "nofile" will cause slow performance with programs that iterate all possible file descriptor, like fakeroot. Let extract.py spawn the container with a sane default ulimit, overriding host's limit, and let the container itself warn if it was started with unsane limits.

extract.py

@@ -82,7 +82,7 @@ def call_docker(input_file, container, target, report_file, memory_limit, tmpdir
 
         shutil.copy(input_file, str(Path(tmpdir.name, 'input', Path(input_file).name)))
 
-        command = f'docker run --rm -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}'
+        command = f'docker run --rm --ulimit nofile=20000:50000 -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}'
         subprocess.run(command, shell=True)
 
         with suppress(shutil.Error):

fact_extractor/docker_extraction.py

@@ -22,7 +22,7 @@
 
 from helperFunctions.config import get_config_dir
 from helperFunctions.file_system import change_owner_of_output_files
-from helperFunctions.program_setup import load_config, setup_logging
+from helperFunctions.program_setup import check_ulimits, load_config, setup_logging
 from unpacker.unpack import unpack
 
 
@@ -43,6 +43,7 @@ def _parse_args():
 def main(args):
     config = load_config(f'{get_config_dir()}/main.cfg')
     setup_logging(debug=False)
+    check_ulimits()
 
     input_dir = Path(config.get('unpack', 'data_folder'), 'input')
     input_file = list(input_dir.iterdir())[0]

fact_extractor/helperFunctions/program_setup.py

@@ -1,6 +1,7 @@
 import argparse
 import configparser
 import logging
+import resource
 
 from common_helper_files import create_dir_for_file
 
@@ -39,6 +40,16 @@ def setup_logging(debug, log_file=None, log_level=None):
     logger.addHandler(console_log)
 
 
+def check_ulimits():
+    # Get number of openable files
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if soft < 1024:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (min(1024, hard), hard))
+        logging.info(f'The number of openable files has been raised from {soft} to {min(1024, hard)}.')
+    elif soft == resource.RLIM_INFINITY or soft > 100000:
+        logging.warning('Warning: A very high (or no) nofile limit will slow down fakeroot and cause other problems.')
+
+
 def load_config(config_file):
     config = configparser.ConfigParser()
     config.read(config_file)