From 3f6501df4c4c6c4b10ec9df95eac925a740db0f2 Mon Sep 17 00:00:00 2001 From: Benedikt Blumenstiel Date: Thu, 1 Feb 2024 16:16:43 +0100 Subject: [PATCH] Added --keep-generated-files Signed-off-by: Benedikt Blumenstiel --- src/c3/create_gridwrapper.py | 84 +++++++++++++++++++++--------------- src/c3/create_operator.py | 26 +++++++---- src/c3/utils.py | 4 ++ tests/test_compiler.py | 9 ++-- 4 files changed, 74 insertions(+), 49 deletions(-) diff --git a/src/c3/create_gridwrapper.py b/src/c3/create_gridwrapper.py index d91fcdb..158fd8f 100644 --- a/src/c3/create_gridwrapper.py +++ b/src/c3/create_gridwrapper.py @@ -163,6 +163,9 @@ def main(): parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.') parser.add_argument('--skip-logging', action='store_true', help='Exclude logging code from component setup code') + parser.add_argument('--keep-generated-files', action='store_true', + help='Do not delete temporary generated files.') + args = parser.parse_args() # Init logging @@ -174,41 +177,52 @@ def main(): handler.setLevel(args.log_level) root.addHandler(handler) - grid_wrapper_file_path, component_path = apply_grid_wrapper( - file_path=args.FILE_PATH, - component_process=args.component_process, - cos=args.cos, - ) - - logging.info('Generate CLAIMED operator for grid wrapper') - - # Add component path and init file path to additional_files - args.ADDITIONAL_FILES.append(component_path) - - # Update dockerfile template if specified - if args.dockerfile_template_path != '': - logging.info(f'Uses custom dockerfile template from {args.dockerfile_template_path}') - with open(args.dockerfile_template_path, 'r') as f: - custom_dockerfile_template = Template(f.read()) - else: - custom_dockerfile_template = None - - create_operator( - file_path=grid_wrapper_file_path, - repository=args.repository, - version=args.version, - custom_dockerfile_template=custom_dockerfile_template, - additional_files=args.ADDITIONAL_FILES, - log_level=args.log_level, - local_mode=args.local_mode, - no_cache=args.no_cache, - overwrite_files=args.overwrite, - rename_files=args.rename, - skip_logging=args.skip_logging, - ) - - logging.info('Remove local component file') - os.remove(component_path) + grid_wrapper_file_path = component_path = '' + try: + grid_wrapper_file_path, component_path = apply_grid_wrapper( + file_path=args.FILE_PATH, + component_process=args.component_process, + cos=args.cos, + ) + + logging.info('Generate CLAIMED operator for grid wrapper') + + # Add component path and init file path to additional_files + args.ADDITIONAL_FILES.append(component_path) + + # Update dockerfile template if specified + if args.dockerfile_template_path != '': + logging.info(f'Uses custom dockerfile template from {args.dockerfile_template_path}') + with open(args.dockerfile_template_path, 'r') as f: + custom_dockerfile_template = Template(f.read()) + else: + custom_dockerfile_template = None + + create_operator( + file_path=grid_wrapper_file_path, + repository=args.repository, + version=args.version, + custom_dockerfile_template=custom_dockerfile_template, + additional_files=args.ADDITIONAL_FILES, + log_level=args.log_level, + local_mode=args.local_mode, + no_cache=args.no_cache, + overwrite_files=args.overwrite, + rename_files=args.rename, + skip_logging=args.skip_logging, + keep_generated_files=args.keep_generated_files, + ) + except Exception as err: + logging.error('Error while generating CLAIMED grid wrapper. ' + 'Consider using `--log_level DEBUG` and `--keep-generated-files` for debugging.') + raise err + finally: + if not args.keep_generated_files: + logging.info('Remove local component file and grid wrapper code.') + if os.path.isfile(grid_wrapper_file_path): + os.remove(grid_wrapper_file_path) + if os.path.isfile(component_path): + os.remove(component_path) if __name__ == '__main__': diff --git a/src/c3/create_operator.py b/src/c3/create_operator.py index 15a5344..18df7fd 100644 --- a/src/c3/create_operator.py +++ b/src/c3/create_operator.py @@ -235,12 +235,13 @@ def create_operator(file_path: str, rename_files=None, overwrite_files=False, skip_logging=False, + keep_generated_files=False, ): logging.info('Parameters: ') logging.info('file_path: ' + file_path) logging.info('repository: ' + str(repository)) logging.info('version: ' + str(version)) - logging.info('additional_files: ' + str(additional_files)) + logging.info('additional_files: ' + '; '.join(additional_files)) if file_path.endswith('.py'): # use temp file for processing @@ -319,10 +320,10 @@ def create_operator(file_path: str, target_dir += '/' logging.info('Operator name: ' + name) - logging.info('Description:: ' + description) - logging.info('Inputs: ' + str(inputs)) - logging.info('Outputs: ' + str(outputs)) - logging.info('Requirements: ' + str(requirements)) + logging.info('Description: ' + description) + logging.info('Inputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in inputs.items()]))) + logging.info('Outputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in outputs.items()]))) + logging.info('Requirements: ' + '; '.join(requirements)) logging.debug(f'Target code: {target_code}') logging.debug(f'Target directory: {target_dir}') @@ -375,8 +376,9 @@ def create_operator(file_path: str, stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True, ) except Exception as err: - remove_temporary_files(file_path, target_code) logging.error('Docker build failed. Consider running C3 with `--log_level DEBUG` to see the docker build logs.') + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err logging.info(f'Successfully built image claimed-{name}:{version}') @@ -398,14 +400,16 @@ def create_operator(file_path: str, except Exception as err: logging.error(f'Could not push images to namespace {repository}. ' f'Please check if docker is logged in or select a namespace with access.') - remove_temporary_files(file_path, target_code) + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err # Check for existing files and optionally modify them before overwriting try: check_existing_files(file_path, rename_files, overwrite_files) except Exception as err: - remove_temporary_files(file_path, target_code) + if not keep_generated_files: + remove_temporary_files(file_path, target_code) raise err # Create application scripts @@ -419,7 +423,8 @@ def create_operator(file_path: str, print_claimed_command(name, repository, version, inputs) # Remove temp files - remove_temporary_files(file_path, target_code) + if not keep_generated_files: + remove_temporary_files(file_path, target_code) def main(): @@ -443,6 +448,8 @@ def main(): parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.') parser.add_argument('--skip-logging', action='store_true', help='Exclude logging code from component setup code') + parser.add_argument('--keep-generated-files', action='store_true', + help='Do not delete temporary generated files.') args = parser.parse_args() # Init logging @@ -474,6 +481,7 @@ def main(): overwrite_files=args.overwrite, rename_files=args.rename, skip_logging=args.skip_logging, + keep_generated_files=args.keep_generated_files, ) diff --git a/src/c3/utils.py b/src/c3/utils.py index b1cc4e9..0bbe544 100644 --- a/src/c3/utils.py +++ b/src/c3/utils.py @@ -107,6 +107,10 @@ def get_image_version(repository, name): Get current version of the image from the registry and increase the version by 1. Defaults to 0.1 if no image is found in the registry. """ + if repository is None: + logging.debug('Using 0.1 as local version.') + return '0.1' + logging.debug(f'Get image version from registry.') if 'docker.io' in repository: logging.debug('Get image tags from docker.') diff --git a/tests/test_compiler.py b/tests/test_compiler.py index 4907cd1..29cdff6 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -121,7 +121,7 @@ def test_create_operator( args: List, ): subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository, - '--test_mode', '-v', 'test', '--log_level', 'DEBUG', '--overwrite'], + '--local_mode', '-v', 'test', '--log_level', 'DEBUG', '--overwrite'], check=True) file = Path(file_path) @@ -136,7 +136,7 @@ def test_create_operator( test_create_gridwrapper_input = [ ( TEST_SCRIPT_PATH, - DUMMY_REPO, + None, 'process', [TEST_NOTEBOOK_PATH], ), @@ -157,8 +157,8 @@ def test_create_gridwrapper( process: str, args: List, ): - subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, - '-r', repository, '-p', process, '--test_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True) + subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, '--overwrite', + '-p', process, '--local_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True) file = Path(file_path) gw_file = file.parent / f'gw_{file.stem}.py' @@ -166,7 +166,6 @@ def test_create_gridwrapper( gw_file.with_suffix('.yaml').unlink() gw_file.with_suffix('.job.yaml').unlink() gw_file.with_suffix('.cwl').unlink() - gw_file.unlink() image_name = f"{repository}/claimed-gw-{file_path.rsplit('.')[0].replace('_', '-')}:test" # TODO: Modify subprocess call to test grid wrapper # subprocess.run(['docker', 'run', image_name], check=True)