diff --git a/pyproject.toml b/pyproject.toml index bd0af70..3511e9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,11 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] +dependencies = [ + 'nbconvert >= 7.9.2', + 'ipython >= 8.16.1', + 'traitlets >= 5.11.2', +] [project.urls] "Homepage" = "https://github.com/claimed-framework/c3" diff --git a/src/c3/create_gridwrapper.py b/src/c3/create_gridwrapper.py index 70d7eb5..2dfcd72 100644 --- a/src/c3/create_gridwrapper.py +++ b/src/c3/create_gridwrapper.py @@ -151,6 +151,7 @@ def main(): parser.add_argument('-l', '--log_level', type=str, default='INFO') parser.add_argument('--dockerfile_template_path', type=str, default='', help='Path to custom dockerfile template') + parser.add_argument('--test_mode', action='store_true') args = parser.parse_args() # Init logging @@ -185,6 +186,7 @@ def main(): dockerfile_template=_dockerfile_template, additional_files=args.additional_files, log_level=args.log_level, + test_mode=args.test_mode, ) logging.info('Remove local component file') diff --git a/src/c3/create_operator.py b/src/c3/create_operator.py index f7a0c84..46647d3 100644 --- a/src/c3/create_operator.py +++ b/src/c3/create_operator.py @@ -19,6 +19,7 @@ def create_operator(file_path: str, dockerfile_template: str, additional_files: str = None, log_level='INFO', + test_mode=False, ): logging.info('Parameters: ') logging.info('file_path: ' + file_path) @@ -101,24 +102,20 @@ def create_operator(file_path: str, version = get_image_version(repository, name) logging.info(f'Building container image claimed-{name}:{version}') - try: - subprocess.run( - ['docker', 'build', '--platform', 'linux/amd64', '-t', f'claimed-{name}:{version}', '.'], - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, - ) - logging.debug(f'Tagging images with "latest" and "{version}"') - subprocess.run( - ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:{version}'], - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, - ) - subprocess.run( - ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:latest'], - stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, - ) - logging.info('Successfully built image') - except: - logging.error(f'Failed to build image with docker.') - pass + subprocess.run( + ['docker', 'build', '--platform', 'linux/amd64', '-t', f'claimed-{name}:{version}', '.'], + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, + ) + logging.debug(f'Tagging images with "latest" and "{version}"') + subprocess.run( + ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:{version}'], + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, + ) + subprocess.run( + ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:latest'], + stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, + ) + logging.info('Successfully built image') logging.info(f'Pushing images to registry {repository}') try: @@ -131,18 +128,26 @@ def create_operator(file_path: str, stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True, ) logging.info('Successfully pushed image to registry') - except: + except Exception as err: logging.error(f'Could not push images to namespace {repository}. ' f'Please check if docker is logged in or select a namespace with access.') - pass + if test_mode: + logging.info('Continue processing (test mode).') + pass + else: + if file_path != target_code: + os.remove(target_code) + os.remove('Dockerfile') + shutil.rmtree(additional_files_path, ignore_errors=True) + raise err def get_component_interface(parameters): return_string = str() for name, options in parameters.items(): return_string += f'- {{name: {name}, type: {options["type"]}, description: "{options["description"]}"' if options['default'] is not None: - if not options["default"].startswith("'"): - options["default"] = f"'{options['default']}'" + if not options["default"].startswith('"'): + options["default"] = f'"{options["default"]}"' return_string += f', default: {options["default"]}' return_string += '}\n' return return_string @@ -203,8 +208,7 @@ def get_component_interface(parameters): if file_path != target_code: os.remove(target_code) os.remove('Dockerfile') - if additional_files_path is not None: - shutil.rmtree(additional_files_path, ignore_errors=True) + shutil.rmtree(additional_files_path, ignore_errors=True) def main(): @@ -220,6 +224,7 @@ def main(): parser.add_argument('-l', '--log_level', type=str, default='INFO') parser.add_argument('--dockerfile_template_path', type=str, default='', help='Path to custom dockerfile template') + parser.add_argument('--test_mode', action='store_true') args = parser.parse_args() # Init logging @@ -246,6 +251,7 @@ def main(): dockerfile_template=_dockerfile_template, additional_files=args.ADDITIONAL_FILES, log_level=args.log_level, + test_mode=args.test_mode, ) diff --git a/src/c3/parser.py b/src/c3/parser.py index 18fee61..8bfd1ff 100644 --- a/src/c3/parser.py +++ b/src/c3/parser.py @@ -17,9 +17,7 @@ import os import re -# TODO: Do we need LoggingConfigurable -# from traitlets.config import LoggingConfigurable -LoggingConfigurable = object +from traitlets.config import LoggingConfigurable from typing import TypeVar, List, Dict @@ -125,8 +123,7 @@ def search_expressions(self) -> Dict[str, List]: # Second regex matches envvar assignments that use os.getenv("name", "value") with ow w/o default provided # Third regex matches envvar assignments that use os.environ.get("name", "value") with or w/o default provided # Both name and value are captured if possible - envs = [r"os\.environ\[[\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"']\](?:\s*=(?:\s*[\"'](.[^\"']*)?[\"'])?)*", - r"os\.getenv\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,\s*[\"'](.[^\"']*)?[\"'])?", + envs = [r"os\.getenv\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,\s*[\"'](.[^\"']*)?[\"'])?", r"os\.environ\.get\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,(?:\s*[\"'](.[^\"']*)?[\"'])?)*"] regex_dict["env_vars"] = envs return regex_dict diff --git a/src/c3/pythonscript.py b/src/c3/pythonscript.py index 8d2ab0d..98e6f73 100644 --- a/src/c3/pythonscript.py +++ b/src/c3/pythonscript.py @@ -31,16 +31,21 @@ def _get_env_vars(self): comment_line = '' if comment_line == '': logging.info(f'Interface: No description for variable {env_name} provided.') - if "int(" in line: + if re.search(r'=\s*int\(\s*os', line): type = 'Integer' - elif "float(" in line: + elif re.search(r'=\s*float\(\s*os', line): type = 'Float' - elif "bool(" in line: + elif re.search(r'=\s*bool\(\s*os', line): type = 'Boolean' else: type = 'String' - if ',' in line: - default = line.split(',', 1)[1].rstrip(') ').strip().replace("\"", "\'") + # get default value + if re.search(r"\(.*,.*\)", line): + # extract int, float, bool + default = re.search(r",\s*(.*?)\s*\)", line).group(1) + if type == 'String' and default != 'None': + # Process string default value + default = default[1:-1].replace("\"", "\'") else: default = None return_value[env_name] = { diff --git a/src/c3/templates/dockerfile_template b/src/c3/templates/dockerfile_template index c82a743..be05953 100644 --- a/src/c3/templates/dockerfile_template +++ b/src/c3/templates/dockerfile_template @@ -2,6 +2,7 @@ FROM registry.access.redhat.com/ubi8/python-39 USER root RUN dnf install -y java-11-openjdk USER default +RUN pip install ipython ${requirements_docker} ADD ${target_code} /opt/app-root/src/ ADD ${additional_files_path} /opt/app-root/src/ diff --git a/src/c3/utils.py b/src/c3/utils.py index 0cce95e..651a0dc 100644 --- a/src/c3/utils.py +++ b/src/c3/utils.py @@ -1,47 +1,33 @@ import os import logging -import json +import nbformat import re import subprocess +from nbconvert.exporters import PythonExporter def convert_notebook(path): - # TODO: switch to nbconvert long-term (need to replace pip install) - with open(path) as json_file: - notebook = json.load(json_file) + notebook = nbformat.read(path, as_version=4) - # backwards compatibility + # backwards compatibility (v0.1 description was included in second cell, merge first two markdown cells) if notebook['cells'][0]['cell_type'] == 'markdown' and notebook['cells'][1]['cell_type'] == 'markdown': logging.info('Merge first two markdown cells. File name is used as operator name, not first markdown cell.') - notebook['cells'][1]['source'] = notebook['cells'][0]['source'] + ['\n'] + notebook['cells'][1]['source'] + notebook['cells'][1]['source'] = notebook['cells'][0]['source'] + '\n' + notebook['cells'][1]['source'] notebook['cells'] = notebook['cells'][1:] - code_lines = [] for cell in notebook['cells']: if cell['cell_type'] == 'markdown': - # add markdown as doc string - code_lines.extend(['"""\n'] + [f'{line}' for line in cell['source']] + ['\n"""']) - elif cell['cell_type'] == 'code' and cell['source'][0].startswith('%%bash'): - code_lines.append('os.system("""') - code_lines.extend(cell['source'][1:]) - code_lines.append('""")') - elif cell['cell_type'] == 'code': - for line in cell['source']: - if line.strip().startswith('!'): - # convert sh scripts - if re.search('![ ]*pip', line): - # change pip install to comment - code_lines.append(re.sub('![ ]*pip', '# pip', line)) - else: - # change sh command to os.system() - logging.info(f'Replace shell command with os.system() ({line})') - code_lines.append(line.replace('!', "os.system('", 1).replace('\n', "')\n")) - else: - # add code - code_lines.append(line) - # add line break after cell - code_lines.append('\n') - code = ''.join(code_lines) + # convert markdown to doc string + cell['cell_type'] = 'code' + cell['source'] = '"""\n' + cell['source'] + '\n"""' + cell['outputs'] = [] + cell['execution_count'] = 0 + if cell['cell_type'] == 'code' and re.search('![ ]*pip', cell['source']): + # replace !pip with #pip + cell['source'] = re.sub('![ ]*pip[ ]*install', '# pip install', cell['source']) + + # convert tp python script + (code, _) = PythonExporter().from_notebook_node(notebook) py_path = path.split('/')[-1].replace('.ipynb', '.py') diff --git a/tests/example_script.py b/tests/example_script.py index 1815c02..4de5f04 100644 --- a/tests/example_script.py +++ b/tests/example_script.py @@ -10,10 +10,10 @@ import numpy as np # A comment one line above os.getenv is the description of this variable. -input_path = os.getenv('input_path') +input_path = os.environ.get('input_path', None ) # ('not this') # type casting to int(), float(), or bool() -batch_size = int(os.getenv('batch_size', 16)) +batch_size = int(os.environ.get('batch_size', 16)) # (not this) # Commas in the previous comment are deleted because the yaml file requires descriptions without commas. debug = bool(os.getenv('debug', False)) diff --git a/tests/test_compiler.py b/tests/test_compiler.py index c94121c..5a352d2 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -114,7 +114,8 @@ def test_create_operator( repository: str, args: List, ): - subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository], check=True) + subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository, '--test_mode'], + check=True) file = Path(file_path) file.with_suffix('.yaml').unlink() @@ -147,7 +148,7 @@ def test_create_gridwrapper( args: List, ): subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args, - '-r', repository, '-p', process], check=True) + '-r', repository, '-p', process, '--test_mode'], check=True) file = Path(file_path) gw_file = file.parent / f'gw_{file.stem}.py'