diff --git a/src/c3/create_gridwrapper.py b/src/c3/create_gridwrapper.py index 5cbcbd9..158fd8f 100644 --- a/src/c3/create_gridwrapper.py +++ b/src/c3/create_gridwrapper.py @@ -7,7 +7,7 @@ from c3.pythonscript import Pythonscript from c3.utils import convert_notebook from c3.create_operator import create_operator -from c3.templates import grid_wrapper_template, cos_grid_wrapper_template, gw_component_setup_code +from c3.templates import grid_wrapper_template, cos_grid_wrapper_template, component_setup_code_wo_logging def wrap_component(component_path, @@ -79,7 +79,7 @@ def get_component_elements(file_path): # Adding code -def edit_component_code(file_path): +def edit_component_code(file_path, component_process): file_name = os.path.basename(file_path) if file_path.endswith('.ipynb'): logging.info('Convert notebook to python script') @@ -88,14 +88,16 @@ def edit_component_code(file_path): file_name = os.path.basename(file_path) else: # write edited code to different file - target_file = os.path.join(os.path.dirname(file_path), 'component_' + file_name) + target_file = os.path.join(os.path.dirname(file_path), 'component_' + file_name.replace('-', '_')) target_file_name = os.path.basename(target_file) with open(file_path, 'r') as f: script = f.read() + assert component_process in script, (f'Did not find the grid process {component_process} in the script. ' + f'Please provide the grid process in the arguments `-p `.') # Add code for logging and cli parameters to the beginning of the script - script = gw_component_setup_code + script + script = component_setup_code_wo_logging + script # replace old filename with new file name script = script.replace(file_name, target_file_name) with open(target_file, 'w') as f: @@ -114,7 +116,7 @@ def apply_grid_wrapper(file_path, component_process, cos): assert file_path.endswith('.py') or file_path.endswith('.ipynb'), \ "Please provide a component file path to a python script or notebook." - file_path = edit_component_code(file_path) + file_path = edit_component_code(file_path, component_process) description, interface, inputs, dependencies = get_component_elements(file_path) @@ -142,7 +144,7 @@ def main(): help='Path to python script or notebook') parser.add_argument('ADDITIONAL_FILES', type=str, nargs='*', help='List of paths to additional files to include in the container image') - parser.add_argument('-p', '--component_process', type=str, required=True, + parser.add_argument('-p', '--component_process', type=str, default='grid_process', help='Name of the component sub process that is executed for each batch.') parser.add_argument('--cos', action=argparse.BooleanOptionalAction, default=False, help='Creates a grid wrapper for processing COS files') @@ -156,8 +158,14 @@ def main(): parser.add_argument('-l', '--log_level', type=str, default='INFO') parser.add_argument('--dockerfile_template_path', type=str, default='', help='Path to custom dockerfile template') - parser.add_argument('--test_mode', action='store_true') - parser.add_argument('--no-cache', action='store_true') + parser.add_argument('--local_mode', action='store_true', + help='Continue processing after docker errors.') + parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.') + parser.add_argument('--skip-logging', action='store_true', + help='Exclude logging code from component setup code') + parser.add_argument('--keep-generated-files', action='store_true', + help='Do not delete temporary generated files.') + args = parser.parse_args() # Init logging @@ -169,13 +177,14 @@ def main(): handler.setLevel(args.log_level) root.addHandler(handler) - grid_wrapper_file_path, component_path = apply_grid_wrapper( - file_path=args.FILE_PATH, - component_process=args.component_process, - cos=args.cos, - ) + grid_wrapper_file_path = component_path = '' + try: + grid_wrapper_file_path, component_path = apply_grid_wrapper( + file_path=args.FILE_PATH, + component_process=args.component_process, + cos=args.cos, + ) - if args.repository is not None: logging.info('Generate CLAIMED operator for grid wrapper') # Add component path and init file path to additional_files @@ -196,14 +205,24 @@ def main(): custom_dockerfile_template=custom_dockerfile_template, additional_files=args.ADDITIONAL_FILES, log_level=args.log_level, - test_mode=args.test_mode, + local_mode=args.local_mode, no_cache=args.no_cache, overwrite_files=args.overwrite, rename_files=args.rename, + skip_logging=args.skip_logging, + keep_generated_files=args.keep_generated_files, ) - - logging.info('Remove local component file') - os.remove(component_path) + except Exception as err: + logging.error('Error while generating CLAIMED grid wrapper. ' + 'Consider using `--log_level DEBUG` and `--keep-generated-files` for debugging.') + raise err + finally: + if not args.keep_generated_files: + logging.info('Remove local component file and grid wrapper code.') + if os.path.isfile(grid_wrapper_file_path): + os.remove(grid_wrapper_file_path) + if os.path.isfile(component_path): + os.remove(component_path) if __name__ == '__main__': diff --git a/src/c3/create_operator.py b/src/c3/create_operator.py index 6660d2c..18df7fd 100644 --- a/src/c3/create_operator.py +++ b/src/c3/create_operator.py @@ -7,13 +7,15 @@ import subprocess import glob import re +import json from pathlib import Path from string import Template from typing import Optional from c3.pythonscript import Pythonscript +from c3.notebook import Notebook from c3.rscript import Rscript from c3.utils import convert_notebook, get_image_version -from c3.templates import (python_component_setup_code, r_component_setup_code, +from c3.templates import (python_component_setup_code, component_setup_code_wo_logging, r_component_setup_code, python_dockerfile_template, r_dockerfile_template, kfp_component_template, kubernetes_job_template, cwl_component_template) @@ -129,12 +131,16 @@ def create_kubernetes_job(name, repository, version, target_code, target_dir, co def create_cwl_component(name, repository, version, file_path, inputs, outputs): + type_dict = {'String': 'string', 'Integer': 'int', 'Float': 'float', 'Boolean': 'bool'} # get environment entries i = 1 input_envs = str() for input, options in inputs.items(): i += 1 - input_envs += (f" {input}:\n type: string\n default: {options['default']}\n " + # Convert string default value to CWL types + default_value = options['default'] if options['type'] == 'String' and options['default'] != '"None"' \ + else options['default'].strip('"\'') + input_envs += (f" {input}:\n type: {type_dict[options['type']]}\n default: {default_value}\n " f"inputBinding:\n position: {i}\n prefix: --{input}\n") if len(outputs) == 0: @@ -224,54 +230,63 @@ def create_operator(file_path: str, custom_dockerfile_template: Optional[Template], additional_files: str = None, log_level='INFO', - test_mode=False, + local_mode=False, no_cache=False, rename_files=None, overwrite_files=False, + skip_logging=False, + keep_generated_files=False, ): logging.info('Parameters: ') logging.info('file_path: ' + file_path) - logging.info('repository: ' + repository) + logging.info('repository: ' + str(repository)) logging.info('version: ' + str(version)) - logging.info('additional_files: ' + str(additional_files)) + logging.info('additional_files: ' + '; '.join(additional_files)) - if file_path.endswith('.ipynb'): - logging.info('Convert notebook to python script') - target_code = convert_notebook(file_path) - command = '/opt/app-root/bin/ipython' - working_dir = '/opt/app-root/src/' - - elif file_path.endswith('.py'): + if file_path.endswith('.py'): # use temp file for processing target_code = 'claimed_' + os.path.basename(file_path) # Copy file to current working directory shutil.copy(file_path, target_code) + # Add code for logging and cli parameters to the beginning of the script + with open(target_code, 'r') as f: + script = f.read() + if skip_logging: + script = component_setup_code_wo_logging + script + else: + script = python_component_setup_code + script + with open(target_code, 'w') as f: + f.write(script) + # getting parameter from the script + script_data = Pythonscript(target_code) + dockerfile_template = custom_dockerfile_template or python_dockerfile_template command = '/opt/app-root/bin/python' working_dir = '/opt/app-root/src/' - elif file_path.lower().endswith('.r'): + elif file_path.endswith('.ipynb'): # use temp file for processing target_code = 'claimed_' + os.path.basename(file_path) # Copy file to current working directory shutil.copy(file_path, target_code) - command = 'Rscript' - working_dir = '/home/docker/' - else: - raise NotImplementedError('Please provide a file_path to a jupyter notebook, python script, or R script.') - - if target_code.endswith('.py'): - # Add code for logging and cli parameters to the beginning of the script - with open(target_code, 'r') as f: - script = f.read() - script = python_component_setup_code + script - with open(target_code, 'w') as f: - f.write(script) - + with open(target_code, 'r') as json_file: + notebook = json.load(json_file) + # Add code for logging and cli parameters to the beginning of the notebook + notebook['cells'].insert(0, { + 'cell_type': 'code', 'execution_count': None, 'metadata': {}, 'outputs': [], + 'source': component_setup_code_wo_logging if skip_logging else python_component_setup_code}) + with open(target_code, 'w') as json_file: + json.dump(notebook, json_file) # getting parameter from the script - script_data = Pythonscript(target_code) + script_data = Notebook(target_code) dockerfile_template = custom_dockerfile_template or python_dockerfile_template + command = '/opt/app-root/bin/ipython' + working_dir = '/opt/app-root/src/' - elif target_code.lower().endswith('.r'): + elif file_path.lower().endswith('.r'): + # use temp file for processing + target_code = 'claimed_' + os.path.basename(file_path) + # Copy file to current working directory + shutil.copy(file_path, target_code) # Add code for logging and cli parameters to the beginning of the script with open(target_code, 'r') as f: script = f.read() @@ -281,8 +296,10 @@ def create_operator(file_path: str, # getting parameter from the script script_data = Rscript(target_code) dockerfile_template = custom_dockerfile_template or r_dockerfile_template + command = 'Rscript' + working_dir = '/home/docker/' else: - raise NotImplementedError('C3 currently only supports jupyter notebooks, python scripts, and R scripts.') + raise NotImplementedError('Please provide a file_path to a jupyter notebook, python script, or R script.') name = script_data.get_name() # convert description into a string with a single line @@ -303,10 +320,10 @@ def create_operator(file_path: str, target_dir += '/' logging.info('Operator name: ' + name) - logging.info('Description:: ' + description) - logging.info('Inputs: ' + str(inputs)) - logging.info('Outputs: ' + str(outputs)) - logging.info('Requirements: ' + str(requirements)) + logging.info('Description: ' + description) + logging.info('Inputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in inputs.items()]))) + logging.info('Outputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in outputs.items()]))) + logging.info('Requirements: ' + '; '.join(requirements)) logging.debug(f'Target code: {target_code}') logging.debug(f'Target directory: {target_dir}') @@ -334,6 +351,12 @@ def create_operator(file_path: str, # auto increase version based on registered images version = get_image_version(repository, name) + if repository is None: + if not local_mode: + logging.warning('No repository provided. The container image is only saved locally. Add `-r ` ' + 'to push the image to a container registry or run `--local_mode` to suppress this warning.') + local_mode = True + logging.info(f'Building container image claimed-{name}:{version}') try: # Run docker build @@ -341,50 +364,52 @@ def create_operator(file_path: str, f"docker build --platform linux/amd64 -t claimed-{name}:{version} . {'--no-cache' if no_cache else ''}", stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True ) - - # Run docker tag - logging.debug(f'Tagging images with "latest" and "{version}"') - subprocess.run( - f"docker tag claimed-{name}:{version} {repository}/claimed-{name}:{version}", - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, - ) - subprocess.run( - f"docker tag claimed-{name}:{version} {repository}/claimed-{name}:latest", - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, - ) + if repository is not None: + # Run docker tag + logging.debug(f'Tagging images with "latest" and "{version}"') + subprocess.run( + f"docker tag claimed-{name}:{version} {repository}/claimed-{name}:{version}", + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, + ) + subprocess.run( + f"docker tag claimed-{name}:{version} {repository}/claimed-{name}:latest", + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, + ) except Exception as err: - remove_temporary_files(file_path, target_code) logging.error('Docker build failed. Consider running C3 with `--log_level DEBUG` to see the docker build logs.') + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err - logging.info('Successfully built image') + logging.info(f'Successfully built image claimed-{name}:{version}') - logging.info(f'Pushing images to registry {repository}') - try: - # Run docker push - subprocess.run( - f"docker push {repository}/claimed-{name}:latest", - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, - ) - subprocess.run( - f"docker push {repository}/claimed-{name}:{version}", - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, - ) - logging.info('Successfully pushed image to registry') - except Exception as err: - logging.error(f'Could not push images to namespace {repository}. ' - f'Please check if docker is logged in or select a namespace with access.') - if test_mode: - logging.info('Continue processing (test mode).') - pass - else: - remove_temporary_files(file_path, target_code) + if local_mode: + logging.info(f'No repository provided, skip docker push.') + else: + logging.info(f'Pushing images to registry {repository}') + try: + # Run docker push + subprocess.run( + f"docker push {repository}/claimed-{name}:latest", + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, + ) + subprocess.run( + f"docker push {repository}/claimed-{name}:{version}", + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, + ) + logging.info('Successfully pushed image to registry') + except Exception as err: + logging.error(f'Could not push images to namespace {repository}. ' + f'Please check if docker is logged in or select a namespace with access.') + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err # Check for existing files and optionally modify them before overwriting try: check_existing_files(file_path, rename_files, overwrite_files) except Exception as err: - remove_temporary_files(file_path, target_code) + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err # Create application scripts @@ -398,7 +423,8 @@ def create_operator(file_path: str, print_claimed_command(name, repository, version, inputs) # Remove temp files - remove_temporary_files(file_path, target_code) + if not keep_generated_files: + remove_temporary_files(file_path, target_code) def main(): @@ -407,7 +433,7 @@ def main(): help='Path to python script or notebook') parser.add_argument('ADDITIONAL_FILES', type=str, nargs='*', help='Paths to additional files to include in the container image') - parser.add_argument('-r', '--repository', type=str, required=True, + parser.add_argument('-r', '--repository', type=str, default=None, help='Container registry address, e.g. docker.io/') parser.add_argument('-v', '--version', type=str, default=None, help='Container image version. Auto-increases the version number if not provided (default 0.1)') @@ -417,8 +443,13 @@ def main(): parser.add_argument('-l', '--log_level', type=str, default='INFO') parser.add_argument('--dockerfile_template_path', type=str, default='', help='Path to custom dockerfile template') - parser.add_argument('--test_mode', action='store_true') - parser.add_argument('--no-cache', action='store_true') + parser.add_argument('--local_mode', action='store_true', + help='Continue processing after docker errors.') + parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.') + parser.add_argument('--skip-logging', action='store_true', + help='Exclude logging code from component setup code') + parser.add_argument('--keep-generated-files', action='store_true', + help='Do not delete temporary generated files.') args = parser.parse_args() # Init logging @@ -445,10 +476,12 @@ def main(): custom_dockerfile_template=custom_dockerfile_template, additional_files=args.ADDITIONAL_FILES, log_level=args.log_level, - test_mode=args.test_mode, + local_mode=args.local_mode, no_cache=args.no_cache, overwrite_files=args.overwrite, rename_files=args.rename, + skip_logging=args.skip_logging, + keep_generated_files=args.keep_generated_files, ) diff --git a/src/c3/notebook.py b/src/c3/notebook.py new file mode 100644 index 0000000..73c1602 --- /dev/null +++ b/src/c3/notebook.py @@ -0,0 +1,98 @@ +import json +import re +import os +import logging +from c3.parser import ContentParser, NotebookReader + + +class Notebook(): + def __init__(self, path): + self.path = path + with open(path) as json_file: + self.notebook = json.load(json_file) + + self.name = os.path.basename(path)[:-6].replace('_', '-').lower() + + if self.notebook['cells'][1]['cell_type'] == self.notebook['cells'][2]['cell_type'] == 'markdown': + # backwards compatibility (v0.1 description was included in second cell, merge first two markdown cells) + logging.info('Merge first two markdown cells for description. ' + 'The file name is used as the operator name, not the first markdown cell.') + self.description = self.notebook['cells'][1]['source'][0] + '\n' + self.notebook['cells'][2]['source'][0] + else: + # Using second cell because first cell was added for setup code + self.description = self.notebook['cells'][1]['source'][0] + + self.inputs = self._get_input_vars() + self.outputs = self._get_output_vars() + + def _get_input_vars(self): + cp = ContentParser() + env_names = cp.parse(self.path)['inputs'] + return_value = dict() + notebook_code_lines = list(NotebookReader(self.path).read_next_code_line()) + for env_name, default in env_names.items(): + comment_line = str() + for line in notebook_code_lines: + if re.search("[\"']" + env_name + "[\"']", line): + if not comment_line.strip().startswith('#'): + # previous line was no description, reset comment_line. + comment_line = '' + if comment_line == '': + logging.info(f'Interface: No description for variable {env_name} provided.') + if re.search(r'=\s*int\(\s*os', line): + type = 'Integer' + elif re.search(r'=\s*float\(\s*os', line): + type = 'Float' + elif re.search(r'=\s*bool\(\s*os', line): + type = 'Boolean' + else: + type = 'String' + return_value[env_name] = { + 'description': comment_line.replace('#', '').replace("\"", "\'").strip(), + 'type': type, + 'default': default + } + break + comment_line = line + return return_value + + def _get_output_vars(self): + cp = ContentParser() + output_names = cp.parse(self.path)['outputs'] + # TODO: Does not check for description code + return_value = {name: { + 'description': f'Output path for {name}', + 'type': 'String', + } for name in output_names} + return return_value + + def get_requirements(self): + requirements = [] + notebook_code_lines = list(NotebookReader(self.path).read_next_code_line()) + # Add dnf install + for line in notebook_code_lines: + if re.search(r'[\s#]*dnf\s*.[^#]*', line): + if '-y' not in line: + # Adding default repo + line += ' -y' + requirements.append(line.replace('#', '').strip()) + + # Add pip install + pattern = r"^[# ]*(pip[ ]*install)[ ]*(.[^#]*)" + for line in notebook_code_lines: + result = re.findall(pattern, line) + if len(result) == 1: + requirements.append((result[0][0] + ' ' + result[0][1].strip())) + return requirements + + def get_name(self): + return self.name + + def get_description(self): + return self.description + + def get_inputs(self): + return self.inputs + + def get_outputs(self): + return self.outputs diff --git a/src/c3/parser.py b/src/c3/parser.py index 524d540..1be4307 100644 --- a/src/c3/parser.py +++ b/src/c3/parser.py @@ -49,14 +49,14 @@ def language(self) -> str: else: return None - def read_next_code_chunk(self) -> List[str]: + def read_next_code_line(self) -> List[str]: """ Implements a generator for lines of code in the specified filepath. Subclasses may override if explicit line-by-line parsing is not feasible, e.g. with Notebooks. """ with open(self._filepath) as f: for line in f: - yield [line.strip()] + yield line.strip() class NotebookReader(FileReader): @@ -79,10 +79,11 @@ def __init__(self, filepath: str): def language(self) -> str: return self._language - def read_next_code_chunk(self) -> List[str]: + def read_next_code_line(self) -> List[str]: for cell in self._notebook.cells: if cell.source and cell.cell_type == "code": - yield cell.source.split('\n') + for line in cell.source.split('\n'): + yield line class ScriptParser(): @@ -157,19 +158,17 @@ def parse(self, filepath: str) -> dict: if not parser: return properties - for chunk in reader.read_next_code_chunk(): - if chunk: - for line in chunk: - matches = parser.parse_environment_variables(line) - for key, match in matches: - if key == "inputs": - default_value = match.group(2) - if default_value: - # The default value match can end with an additional ', ", or ) which is removed - default_value = re.sub(r"['\")]?$", '', default_value, count=1) - properties[key][match.group(1)] = default_value - else: - properties[key].append(match.group(1)) + for line in reader.read_next_code_line(): + matches = parser.parse_environment_variables(line) + for key, match in matches: + if key == "inputs": + default_value = match.group(2) + if default_value: + # The default value match can end with an additional ', ", or ) which is removed + default_value = re.sub(r"['\")]?$", '', default_value, count=1) + properties[key][match.group(1)] = default_value + else: + properties[key].append(match.group(1)) return properties diff --git a/src/c3/templates/__init__.py b/src/c3/templates/__init__.py index 1e27697..5ec64b6 100644 --- a/src/c3/templates/__init__.py +++ b/src/c3/templates/__init__.py @@ -6,7 +6,7 @@ # template file names PYTHON_COMPONENT_SETUP_CODE = 'component_setup_code.py' R_COMPONENT_SETUP_CODE = 'component_setup_code.R' -GW_COMPONENT_SETUP_CODE = 'gw_component_setup_code.py' +PYTHON_COMPONENT_SETUP_CODE_WO_LOGGING = 'component_setup_code_wo_logging.py' PYTHON_DOCKERFILE_FILE = 'python_dockerfile_template' R_DOCKERFILE_FILE = 'R_dockerfile_template' KFP_COMPONENT_FILE = 'kfp_component_template.yaml' @@ -24,8 +24,8 @@ with open(template_path / R_COMPONENT_SETUP_CODE, 'r') as f: r_component_setup_code = f.read() -with open(template_path / GW_COMPONENT_SETUP_CODE, 'r') as f: - gw_component_setup_code = f.read() +with open(template_path / PYTHON_COMPONENT_SETUP_CODE_WO_LOGGING, 'r') as f: + component_setup_code_wo_logging = f.read() with open(template_path / PYTHON_DOCKERFILE_FILE, 'r') as f: python_dockerfile_template = Template(f.read()) diff --git a/src/c3/templates/gw_component_setup_code.py b/src/c3/templates/component_setup_code_wo_logging.py similarity index 100% rename from src/c3/templates/gw_component_setup_code.py rename to src/c3/templates/component_setup_code_wo_logging.py diff --git a/src/c3/utils.py b/src/c3/utils.py index 020506f..0bbe544 100644 --- a/src/c3/utils.py +++ b/src/c3/utils.py @@ -32,7 +32,7 @@ def convert_notebook(path): # add import get_ipython code = 'from IPython import get_ipython \n' + code - py_path = path.split('/')[-1].replace('.ipynb', '.py') + py_path = path.split('/')[-1].replace('.ipynb', '.py').replace('-', '_') assert not os.path.exists(py_path), f"File {py_path} already exist. Cannot convert notebook." with open(py_path, 'w') as py_file: @@ -107,6 +107,10 @@ def get_image_version(repository, name): Get current version of the image from the registry and increase the version by 1. Defaults to 0.1 if no image is found in the registry. """ + if repository is None: + logging.debug('Using 0.1 as local version.') + return '0.1' + logging.debug(f'Get image version from registry.') if 'docker.io' in repository: logging.debug('Get image tags from docker.') diff --git a/tests/test_compiler.py b/tests/test_compiler.py index 1018605..29cdff6 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -121,7 +121,7 @@ def test_create_operator( args: List, ): subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository, - '--test_mode', '-v', 'test', '--log_level', 'DEBUG'], + '--local_mode', '-v', 'test', '--log_level', 'DEBUG', '--overwrite'], check=True) file = Path(file_path) @@ -136,7 +136,7 @@ def test_create_operator( test_create_gridwrapper_input = [ ( TEST_SCRIPT_PATH, - DUMMY_REPO, + None, 'process', [TEST_NOTEBOOK_PATH], ), @@ -157,8 +157,8 @@ def test_create_gridwrapper( process: str, args: List, ): - subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, - '-r', repository, '-p', process, '--test_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True) + subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, '--overwrite', + '-p', process, '--local_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True) file = Path(file_path) gw_file = file.parent / f'gw_{file.stem}.py' @@ -166,7 +166,6 @@ def test_create_gridwrapper( gw_file.with_suffix('.yaml').unlink() gw_file.with_suffix('.job.yaml').unlink() gw_file.with_suffix('.cwl').unlink() - gw_file.unlink() image_name = f"{repository}/claimed-gw-{file_path.rsplit('.')[0].replace('_', '-')}:test" # TODO: Modify subprocess call to test grid wrapper # subprocess.run(['docker', 'run', image_name], check=True)