Merge branch 'master' of https://github.com/mercure-imaging/mercure

addons/vagrant/docker/Vagrantfile

@@ -4,17 +4,27 @@ require 'getoptlong'
 
 orthanc=''
 branch='--branch latest-stable'
+use_default_disk=false
 
 opts = GetoptLong.new(
   [ '--orthanc', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ]  
+  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ],
+  [ '--smaller-disk', GetoptLong::OPTIONAL_ARGUMENT ]
 )
 opts.each do |opt, arg|
   case opt
     when '--orthanc'
       orthanc='yes'
     when '--dev'
-      branch=''      
+      branch=''
+    when '--smaller-disk'
+      use_default_disk=true
+  end
+end
+if !use_default_disk then
+  # Install vagrant-disksize to allow resizing the vagrant box disk.
+  unless Vagrant.has_plugin?("vagrant-disksize")
+    raise  Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Either use --smaller-disk, or install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'"
   end
 end
 
@@ -43,6 +53,9 @@ Vagrant.configure(2) do |config|
   config.vm.network "forwarded_port", guest: 8000, host: 8000, auto_correct: true, host_ip: "127.0.0.1"
   config.vm.network "forwarded_port", guest: 11112, host: 11112, auto_correct: true, host_ip: "127.0.0.1"
 
+  if use_default_disk == false then
+    config.disksize.size = "70GB"
+  end
   if orthanc == "yes" then
     config.vm.network "forwarded_port", guest: 8008, host: 8008, auto_correct: true, host_ip: "127.0.0.1"
     config.vm.network "forwarded_port", guest: 8042, host: 8042, auto_correct: true, host_ip: "127.0.0.1"

addons/vagrant/nomad/Vagrantfile

@@ -4,17 +4,27 @@ require 'getoptlong'
 
 orthanc=''
 branch='--branch latest-stable'
+use_default_disk=false
 
 opts = GetoptLong.new(
   [ '--orthanc', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ]
+  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ],
+  [ '--smaller-disk', GetoptLong::OPTIONAL_ARGUMENT ]
 )
 opts.each do |opt, arg|
   case opt
     when '--orthanc'
       orthanc='yes'
     when '--dev'
       branch=''
+    when '--smaller-disk'
+      use_default_disk=true
+  end
+end
+if !use_default_disk then
+  # Install vagrant-disksize to allow resizing the vagrant box disk.
+  unless Vagrant.has_plugin?("vagrant-disksize")
+    raise  Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Either use --smaller-disk, or install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'"
   end
 end
 
@@ -45,6 +55,9 @@ Vagrant.configure(2) do |config|
   config.vm.network "forwarded_port", guest: 11112, host: 11112, auto_correct: true, host_ip: "127.0.0.1"
   config.vm.network "forwarded_port", guest: 4646, host: 4646, auto_correct: true, host_ip: "127.0.0.1"
 
+  if use_default_disk == false then
+    config.disksize.size = "70GB"
+  end
   if orthanc == "yes" then
     config.vm.network "forwarded_port", guest: 8008, host: 8008, auto_correct: true, host_ip: "127.0.0.1"
     config.vm.network "forwarded_port", guest: 8042, host: 8042, auto_correct: true, host_ip: "127.0.0.1"

addons/vagrant/systemd/Vagrantfile

@@ -4,17 +4,27 @@ require 'getoptlong'
 
 orthanc=''
 branch='--branch latest-stable'
+use_default_disk=false
 
 opts = GetoptLong.new(
   [ '--orthanc', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ]
+  [ '--dev', GetoptLong::OPTIONAL_ARGUMENT ],
+  [ '--smaller-disk', GetoptLong::OPTIONAL_ARGUMENT ]
 )
 opts.each do |opt, arg|
   case opt
     when '--orthanc'
       orthanc='yes'
     when '--dev'
       branch=''
+    when '--smaller-disk'
+      use_default_disk=true
+  end
+end
+if !use_default_disk then
+  # Install vagrant-disksize to allow resizing the vagrant box disk.
+  unless Vagrant.has_plugin?("vagrant-disksize")
+    raise  Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Either use --smaller-disk, or install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'"
   end
 end
 
@@ -44,6 +54,9 @@ Vagrant.configure(2) do |config|
   config.vm.network "forwarded_port", guest: 8000, host: 8000, auto_correct: true, host_ip: "127.0.0.1"
   config.vm.network "forwarded_port", guest: 11112, host: 11112, auto_correct: true, host_ip: "127.0.0.1"
 
+  if use_default_disk == false then
+    config.disksize.size = "70GB"
+  end
   if orthanc == "yes" then
     config.vm.network "forwarded_port", guest: 8008, host: 8008, auto_correct: true, host_ip: "127.0.0.1"
     config.vm.network "forwarded_port", guest: 8042, host: 8042, auto_correct: true, host_ip: "127.0.0.1"
@@ -69,4 +82,4 @@ Vagrant.configure(2) do |config|
       v.vmx["memsize"] = "4096"
     end
   end
-end
+end

common/types.py

@@ -176,6 +176,7 @@ class Config(BaseModel, Compat):
     rules: Dict[str, Rule]
     modules: Dict[str, Module]
     process_runner: Literal["docker", "nomad", ""] = ""
+    processing_runtime: Optional[str] = None
     bookkeeper_api_key: Optional[str]
     features: Dict[str, bool]
     processing_logs: ProcessingLogsConfig = ProcessingLogsConfig()

install.sh

@@ -346,7 +346,7 @@ install_app_files() {
 install_packages() {
   echo "## Installing Linux packages..."
   sudo apt-get update
-  sudo apt-get install -y build-essential wget git dcmtk jq inetutils-ping sshpass postgresql postgresql-contrib libpq-dev git-lfs python3-wheel python3-dev python3 python3-venv
+  sudo apt-get install -y build-essential wget git dcmtk jq inetutils-ping sshpass postgresql postgresql-contrib libpq-dev git-lfs python3-wheel python3-dev python3 python3-venv sendmail
 }
 
 

installation/mercure_receiver.service

@@ -11,6 +11,7 @@ WorkingDirectory=/opt/mercure/app
 ExecStart=/opt/mercure/app/receiver.sh
 Restart=on-failure
 RestartSec=3
+User=mercure
 
 [Install]
 WantedBy=multi-user.target

process/process_series.py

@@ -139,6 +139,10 @@ def decode_task_json(json_string: Optional[str]) -> Any:
         logger.error("No docker tag supplied")
         return False
 
+    runtime = {}
+    if config.mercure.processing_runtime:
+        runtime = dict(runtime=config.mercure.processing_runtime)
+
     additional_volumes: Dict[str, Dict[str, str]] = decode_task_json(module.additional_volumes)
     module_environment = decode_task_json(module.environment)
     mercure_environment = dict(MERCURE_IN_DIR=container_in_dir, MERCURE_OUT_DIR=container_out_dir)
@@ -232,14 +236,18 @@ def decode_task_json(json_string: Optional[str]) -> Any:
 
         # We might be operating in a user-remapped namespace. This makes sure that the user inside the container can read and write the files.
         ( real_folder / "in" ).chmod(0o777)
-        for k in ( real_folder / "in" ).glob("**/*"):
-            k.chmod(0o666)
+        try:
+            for k in ( real_folder / "in" ).glob("**/*"):
+                k.chmod(0o666)
+        except PermissionError:
+            raise Exception("Unable to prepare input files for processor. The receiver may be running as root, which is no longer supported. ")
         ( real_folder / "out" ).chmod(0o777)
 
         container  = docker_client.containers.run(
             docker_tag,
             volumes=merged_volumes,
             environment=environment,
+            **runtime,
             **set_command,
             **arguments,
             **user_info,

requirements.in

@@ -11,7 +11,7 @@ wheel>=0.38.1
 # logging / monitoring
 graphyte
 daiquiri
-aiohttp
+aiohttp>=3.8.5
 types-requests
 
 # documentation
@@ -50,10 +50,10 @@ docker
 python-nomad
 
 # other
-pydicom
-pydantic
+pydicom ~= 2.4.1
+pydantic >= 1.10.9, <2.0.0
 pillow >= 9.2.0
 dicomweb-client
 boto3
 freezegun
-certifi>=2022.12.07
+certifi>=2023.07.22

requirements.txt

@@ -6,7 +6,7 @@
 #
 aiofiles==23.1.0
     # via -r requirements.in
-aiohttp==3.8.4
+aiohttp==3.8.5
     # via -r requirements.in
 aiopg==1.4.0
     # via databases
@@ -36,7 +36,7 @@ botocore==1.29.159
     # via
     #   boto3
     #   s3transfer
-certifi==2023.5.7
+certifi==2023.7.22
     # via
     #   -r requirements.in
     #   requests
@@ -217,6 +217,7 @@ starlette==0.28.0
     # via
     #   -r requirements.in
     #   starlette-auth-toolkit
+    #   starlette-decorouter
 starlette-auth-toolkit==0.5.0
     # via -r requirements.in
 starlette-decorouter==1.0.0

tests/data/test_config.json

@@ -21,6 +21,7 @@
     "retention": 259200,
     "offpeak_start": "22:00",
     "offpeak_end": "06:00",
+    "processing_runtime": "runc",
     "targets": {
         "test_target": {
             "target_type": "dicom",

tests/test_processor.py

@@ -251,6 +251,7 @@ async def test_process_series(fs, mercure_config: Callable[[Dict], Config], mock
         user=uid_string,
         group_add=[os.getegid()],
         volumes=unittest.mock.ANY,
+        runtime="runc",
         detach=True),
         call('busybox:stable-musl', volumes=unittest.mock.ANY, userns_mode='host', command='chown -R 1000:1000 /tmp/output', detach=True)
         ]
@@ -330,6 +331,7 @@ async def test_multi_process_series(fs, mercure_config: Callable[[Dict], Config]
                 environment={"MERCURE_IN_DIR": "/tmp/data", "MERCURE_OUT_DIR": "/tmp/output",  'MONAI_INPUTPATH': '/tmp/data', 'MONAI_OUTPUTPATH': '/tmp/output'},
                 user=uid_string,
                 group_add=[os.getegid()],
+                runtime="runc",
                 volumes={
                     str(processor_path / "in"): {"bind": "/tmp/data", "mode": "rw"},
                     str(processor_path / "out"): {"bind": "/tmp/output", "mode": "rw"},