Added --keep-generated-files

Signed-off-by: Benedikt Blumenstiel <benedikt.blumenstiel@ibm.com>

src/c3/create_gridwrapper.py

@@ -163,6 +163,9 @@ def main():
     parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.')
     parser.add_argument('--skip-logging', action='store_true',
                         help='Exclude logging code from component setup code')
+    parser.add_argument('--keep-generated-files', action='store_true',
+                        help='Do not delete temporary generated files.')
+
     args = parser.parse_args()
 
     # Init logging
@@ -174,41 +177,52 @@ def main():
     handler.setLevel(args.log_level)
     root.addHandler(handler)
 
-    grid_wrapper_file_path, component_path = apply_grid_wrapper(
-        file_path=args.FILE_PATH,
-        component_process=args.component_process,
-        cos=args.cos,
-    )
-
-    logging.info('Generate CLAIMED operator for grid wrapper')
-
-    # Add component path and init file path to additional_files
-    args.ADDITIONAL_FILES.append(component_path)
-
-    # Update dockerfile template if specified
-    if args.dockerfile_template_path != '':
-        logging.info(f'Uses custom dockerfile template from {args.dockerfile_template_path}')
-        with open(args.dockerfile_template_path, 'r') as f:
-            custom_dockerfile_template = Template(f.read())
-    else:
-        custom_dockerfile_template = None
-
-    create_operator(
-        file_path=grid_wrapper_file_path,
-        repository=args.repository,
-        version=args.version,
-        custom_dockerfile_template=custom_dockerfile_template,
-        additional_files=args.ADDITIONAL_FILES,
-        log_level=args.log_level,
-        local_mode=args.local_mode,
-        no_cache=args.no_cache,
-        overwrite_files=args.overwrite,
-        rename_files=args.rename,
-        skip_logging=args.skip_logging,
-    )
-
-    logging.info('Remove local component file')
-    os.remove(component_path)
+    grid_wrapper_file_path = component_path = ''
+    try:
+        grid_wrapper_file_path, component_path = apply_grid_wrapper(
+            file_path=args.FILE_PATH,
+            component_process=args.component_process,
+            cos=args.cos,
+        )
+
+        logging.info('Generate CLAIMED operator for grid wrapper')
+
+        # Add component path and init file path to additional_files
+        args.ADDITIONAL_FILES.append(component_path)
+
+        # Update dockerfile template if specified
+        if args.dockerfile_template_path != '':
+            logging.info(f'Uses custom dockerfile template from {args.dockerfile_template_path}')
+            with open(args.dockerfile_template_path, 'r') as f:
+                custom_dockerfile_template = Template(f.read())
+        else:
+            custom_dockerfile_template = None
+
+        create_operator(
+            file_path=grid_wrapper_file_path,
+            repository=args.repository,
+            version=args.version,
+            custom_dockerfile_template=custom_dockerfile_template,
+            additional_files=args.ADDITIONAL_FILES,
+            log_level=args.log_level,
+            local_mode=args.local_mode,
+            no_cache=args.no_cache,
+            overwrite_files=args.overwrite,
+            rename_files=args.rename,
+            skip_logging=args.skip_logging,
+            keep_generated_files=args.keep_generated_files,
+        )
+    except Exception as err:
+        logging.error('Error while generating CLAIMED grid wrapper. '
+                      'Consider using `--log_level DEBUG` and `--keep-generated-files` for debugging.')
+        raise err
+    finally:
+        if not args.keep_generated_files:
+            logging.info('Remove local component file and grid wrapper code.')
+            if os.path.isfile(grid_wrapper_file_path):
+                os.remove(grid_wrapper_file_path)
+            if os.path.isfile(component_path):
+                os.remove(component_path)
 
 
 if __name__ == '__main__':

src/c3/create_operator.py

@@ -235,12 +235,13 @@ def create_operator(file_path: str,
                     rename_files=None,
                     overwrite_files=False,
                     skip_logging=False,
+                    keep_generated_files=False,
                     ):
     logging.info('Parameters: ')
     logging.info('file_path: ' + file_path)
     logging.info('repository: ' + str(repository))
     logging.info('version: ' + str(version))
-    logging.info('additional_files: ' + str(additional_files))
+    logging.info('additional_files: ' + '; '.join(additional_files))
 
     if file_path.endswith('.py'):
         # use temp file for processing
@@ -319,10 +320,10 @@ def create_operator(file_path: str,
         target_dir += '/'
 
     logging.info('Operator name: ' + name)
-    logging.info('Description:: ' + description)
-    logging.info('Inputs: ' + str(inputs))
-    logging.info('Outputs: ' + str(outputs))
-    logging.info('Requirements: ' + str(requirements))
+    logging.info('Description: ' + description)
+    logging.info('Inputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in inputs.items()])))
+    logging.info('Outputs:\n' + ('\n'.join([f'{k}: {v}' for k, v in outputs.items()])))
+    logging.info('Requirements: ' + '; '.join(requirements))
     logging.debug(f'Target code: {target_code}')
     logging.debug(f'Target directory: {target_dir}')
 
@@ -375,8 +376,9 @@ def create_operator(file_path: str,
                 stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, shell=True,
             )
     except Exception as err:
-        remove_temporary_files(file_path, target_code)
         logging.error('Docker build failed. Consider running C3 with `--log_level DEBUG` to see the docker build logs.')
+        if not keep_generated_files:
+            remove_temporary_files(file_path, target_code)
         raise err
     logging.info(f'Successfully built image claimed-{name}:{version}')
 
@@ -398,14 +400,16 @@ def create_operator(file_path: str,
         except Exception as err:
             logging.error(f'Could not push images to namespace {repository}. '
                           f'Please check if docker is logged in or select a namespace with access.')
-            remove_temporary_files(file_path, target_code)
+            if not keep_generated_files:
+                remove_temporary_files(file_path, target_code)
             raise err
 
     # Check for existing files and optionally modify them before overwriting
     try:
         check_existing_files(file_path, rename_files, overwrite_files)
     except Exception as err:
-        remove_temporary_files(file_path, target_code)
+        if not keep_generated_files:
+            remove_temporary_files(file_path, target_code)
         raise err
 
     # Create application scripts
@@ -419,7 +423,8 @@ def create_operator(file_path: str,
     print_claimed_command(name, repository, version, inputs)
 
     # Remove temp files
-    remove_temporary_files(file_path, target_code)
+    if not keep_generated_files:
+        remove_temporary_files(file_path, target_code)
 
 
 def main():
@@ -443,6 +448,8 @@ def main():
     parser.add_argument('--no-cache', action='store_true', help='Not using cache for docker build.')
     parser.add_argument('--skip-logging', action='store_true',
                         help='Exclude logging code from component setup code')
+    parser.add_argument('--keep-generated-files', action='store_true',
+                        help='Do not delete temporary generated files.')
     args = parser.parse_args()
 
     # Init logging
@@ -474,6 +481,7 @@ def main():
         overwrite_files=args.overwrite,
         rename_files=args.rename,
         skip_logging=args.skip_logging,
+        keep_generated_files=args.keep_generated_files,
     )
 
 

src/c3/utils.py

@@ -107,6 +107,10 @@ def get_image_version(repository, name):
     Get current version of the image from the registry and increase the version by 1.
     Defaults to 0.1 if no image is found in the registry.
     """
+    if repository is None:
+        logging.debug('Using 0.1 as local version.')
+        return '0.1'
+
     logging.debug(f'Get image version from registry.')
     if 'docker.io' in repository:
         logging.debug('Get image tags from docker.')

tests/test_compiler.py

@@ -121,7 +121,7 @@ def test_create_operator(
         args: List,
 ):
     subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository,
-                    '--test_mode', '-v', 'test', '--log_level', 'DEBUG', '--overwrite'],
+                    '--local_mode', '-v', 'test', '--log_level', 'DEBUG', '--overwrite'],
                    check=True)
 
     file = Path(file_path)
@@ -136,7 +136,7 @@ def test_create_operator(
 test_create_gridwrapper_input = [
     (
         TEST_SCRIPT_PATH,
-        DUMMY_REPO,
+        None,
         'process',
         [TEST_NOTEBOOK_PATH],
     ),
@@ -157,16 +157,15 @@ def test_create_gridwrapper(
         process: str,
         args: List,
 ):
-    subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args,
-                    '-r', repository, '-p', process, '--test_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True)
+    subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, '--overwrite',
+                    '-p', process, '--local_mode', '-v', 'test', '--log_level', 'DEBUG'], check=True)
 
     file = Path(file_path)
     gw_file = file.parent / f'gw_{file.stem}.py'
 
     gw_file.with_suffix('.yaml').unlink()
     gw_file.with_suffix('.job.yaml').unlink()
     gw_file.with_suffix('.cwl').unlink()
-    gw_file.unlink()
     image_name = f"{repository}/claimed-gw-{file_path.rsplit('.')[0].replace('_', '-')}:test"
     # TODO: Modify subprocess call to test grid wrapper
     # subprocess.run(['docker', 'run', image_name], check=True)