nbconvert+ipython, fixed wrong default values, Stop processing after error

pyproject.toml

@@ -25,6 +25,11 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
 ]
+dependencies = [
+    'nbconvert >= 7.9.2',
+    'ipython >= 8.16.1',
+    'traitlets >= 5.11.2',
+]
 
 [project.urls]
 "Homepage" = "https://github.com/claimed-framework/c3"

src/c3/create_gridwrapper.py

@@ -151,6 +151,7 @@ def main():
     parser.add_argument('-l', '--log_level', type=str, default='INFO')
     parser.add_argument('--dockerfile_template_path', type=str, default='',
                         help='Path to custom dockerfile template')
+    parser.add_argument('--test_mode', action='store_true')
     args = parser.parse_args()
 
     # Init logging
@@ -185,6 +186,7 @@ def main():
             dockerfile_template=_dockerfile_template,
             additional_files=args.additional_files,
             log_level=args.log_level,
+            test_mode=args.test_mode,
         )
 
         logging.info('Remove local component file')

src/c3/create_operator.py

@@ -19,6 +19,7 @@ def create_operator(file_path: str,
                     dockerfile_template: str,
                     additional_files: str = None,
                     log_level='INFO',
+                    test_mode=False,
                     ):
     logging.info('Parameters: ')
     logging.info('file_path: ' + file_path)
@@ -101,24 +102,20 @@ def create_operator(file_path: str,
         version = get_image_version(repository, name)
 
     logging.info(f'Building container image claimed-{name}:{version}')
-    try:
-        subprocess.run(
-            ['docker', 'build', '--platform', 'linux/amd64', '-t', f'claimed-{name}:{version}', '.'],
-            stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
-        )
-        logging.debug(f'Tagging images with "latest" and "{version}"')
-        subprocess.run(
-            ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:{version}'],
-            stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
-        )
-        subprocess.run(
-            ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:latest'],
-            stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
-        )
-        logging.info('Successfully built image')
-    except:
-        logging.error(f'Failed to build image with docker.')
-        pass
+    subprocess.run(
+        ['docker', 'build', '--platform', 'linux/amd64', '-t', f'claimed-{name}:{version}', '.'],
+        stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
+    )
+    logging.debug(f'Tagging images with "latest" and "{version}"')
+    subprocess.run(
+        ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:{version}'],
+        stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
+    )
+    subprocess.run(
+        ['docker', 'tag', f'claimed-{name}:{version}', f'{repository}/claimed-{name}:latest'],
+        stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
+    )
+    logging.info('Successfully built image')
 
     logging.info(f'Pushing images to registry {repository}')
     try:
@@ -131,18 +128,26 @@ def create_operator(file_path: str,
             stdout=None if log_level == 'DEBUG' else subprocess.PIPE, check=True,
         )
         logging.info('Successfully pushed image to registry')
-    except:
+    except Exception as err:
         logging.error(f'Could not push images to namespace {repository}. '
                       f'Please check if docker is logged in or select a namespace with access.')
-        pass
+        if test_mode:
+            logging.info('Continue processing (test mode).')
+            pass
+        else:
+            if file_path != target_code:
+                os.remove(target_code)
+            os.remove('Dockerfile')
+            shutil.rmtree(additional_files_path, ignore_errors=True)
+            raise err
 
     def get_component_interface(parameters):
         return_string = str()
         for name, options in parameters.items():
             return_string += f'- {{name: {name}, type: {options["type"]}, description: "{options["description"]}"'
             if options['default'] is not None:
-                if not options["default"].startswith("'"):
-                    options["default"] = f"'{options['default']}'"
+                if not options["default"].startswith('"'):
+                    options["default"] = f'"{options["default"]}"'
                 return_string += f', default: {options["default"]}'
             return_string += '}\n'
         return return_string
@@ -203,8 +208,7 @@ def get_component_interface(parameters):
     if file_path != target_code:
         os.remove(target_code)
     os.remove('Dockerfile')
-    if additional_files_path is not None:
-        shutil.rmtree(additional_files_path, ignore_errors=True)
+    shutil.rmtree(additional_files_path, ignore_errors=True)
 
 
 def main():
@@ -220,6 +224,7 @@ def main():
     parser.add_argument('-l', '--log_level', type=str, default='INFO')
     parser.add_argument('--dockerfile_template_path', type=str, default='',
                         help='Path to custom dockerfile template')
+    parser.add_argument('--test_mode', action='store_true')
     args = parser.parse_args()
 
     # Init logging
@@ -246,6 +251,7 @@ def main():
         dockerfile_template=_dockerfile_template,
         additional_files=args.ADDITIONAL_FILES,
         log_level=args.log_level,
+        test_mode=args.test_mode,
     )
 
 

src/c3/parser.py

@@ -17,9 +17,7 @@
 import os
 import re
 
-# TODO: Do we need LoggingConfigurable
-# from traitlets.config import LoggingConfigurable
-LoggingConfigurable = object
+from traitlets.config import LoggingConfigurable
 
 from typing import TypeVar, List, Dict
 
@@ -125,8 +123,7 @@ def search_expressions(self) -> Dict[str, List]:
         # Second regex matches envvar assignments that use os.getenv("name", "value") with ow w/o default provided
         # Third regex matches envvar assignments that use os.environ.get("name", "value") with or w/o default provided
         # Both name and value are captured if possible
-        envs = [r"os\.environ\[[\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"']\](?:\s*=(?:\s*[\"'](.[^\"']*)?[\"'])?)*",
-                r"os\.getenv\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,\s*[\"'](.[^\"']*)?[\"'])?",
+        envs = [r"os\.getenv\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,\s*[\"'](.[^\"']*)?[\"'])?",
                 r"os\.environ\.get\([\"']([a-zA-Z_]+[A-Za-z0-9_]*)[\"'](?:\s*\,(?:\s*[\"'](.[^\"']*)?[\"'])?)*"]
         regex_dict["env_vars"] = envs
         return regex_dict

src/c3/pythonscript.py

@@ -31,16 +31,21 @@ def _get_env_vars(self):
                         comment_line = ''
                     if comment_line == '':
                         logging.info(f'Interface: No description for variable {env_name} provided.')
-                    if "int(" in line:
+                    if re.search(r'=\s*int\(\s*os', line):
                         type = 'Integer'
-                    elif "float(" in line:
+                    elif re.search(r'=\s*float\(\s*os', line):
                         type = 'Float'
-                    elif "bool(" in line:
+                    elif re.search(r'=\s*bool\(\s*os', line):
                         type = 'Boolean'
                     else:
                         type = 'String'
-                    if ',' in line:
-                        default = line.split(',', 1)[1].rstrip(') ').strip().replace("\"", "\'")
+                    # get default value
+                    if re.search(r"\(.*,.*\)", line):
+                        # extract int, float, bool
+                        default = re.search(r",\s*(.*?)\s*\)", line).group(1)
+                        if type == 'String' and default != 'None':
+                            # Process string default value
+                            default = default[1:-1].replace("\"", "\'")
                     else:
                         default = None
                     return_value[env_name] = {

src/c3/templates/dockerfile_template

@@ -2,6 +2,7 @@ FROM registry.access.redhat.com/ubi8/python-39
 USER root
 RUN dnf install -y java-11-openjdk
 USER default
+RUN pip install ipython
 ${requirements_docker}
 ADD ${target_code} /opt/app-root/src/
 ADD ${additional_files_path} /opt/app-root/src/

src/c3/utils.py

@@ -1,47 +1,33 @@
 import os
 import logging
-import json
+import nbformat
 import re
 import subprocess
+from nbconvert.exporters import PythonExporter
 
 
 def convert_notebook(path):
-    # TODO: switch to nbconvert long-term (need to replace pip install)
-    with open(path) as json_file:
-        notebook = json.load(json_file)
+    notebook = nbformat.read(path, as_version=4)
 
-    # backwards compatibility
+    # backwards compatibility (v0.1 description was included in second cell, merge first two markdown cells)
     if notebook['cells'][0]['cell_type'] == 'markdown' and notebook['cells'][1]['cell_type'] == 'markdown':
         logging.info('Merge first two markdown cells. File name is used as operator name, not first markdown cell.')
-        notebook['cells'][1]['source'] = notebook['cells'][0]['source'] + ['\n'] + notebook['cells'][1]['source']
+        notebook['cells'][1]['source'] = notebook['cells'][0]['source'] + '\n' + notebook['cells'][1]['source']
         notebook['cells'] = notebook['cells'][1:]
 
-    code_lines = []
     for cell in notebook['cells']:
         if cell['cell_type'] == 'markdown':
-            # add markdown as doc string
-            code_lines.extend(['"""\n'] + [f'{line}' for line in cell['source']] + ['\n"""'])
-        elif cell['cell_type'] == 'code' and cell['source'][0].startswith('%%bash'):
-            code_lines.append('os.system("""')
-            code_lines.extend(cell['source'][1:])
-            code_lines.append('""")')
-        elif cell['cell_type'] == 'code':
-            for line in cell['source']:
-                if line.strip().startswith('!'):
-                    # convert sh scripts
-                    if re.search('![ ]*pip', line):
-                        # change pip install to comment
-                        code_lines.append(re.sub('![ ]*pip', '# pip', line))
-                    else:
-                        # change sh command to os.system()
-                        logging.info(f'Replace shell command with os.system() ({line})')
-                        code_lines.append(line.replace('!', "os.system('", 1).replace('\n', "')\n"))
-                else:
-                    # add code
-                    code_lines.append(line)
-        # add line break after cell
-        code_lines.append('\n')
-    code = ''.join(code_lines)
+            # convert markdown to doc string
+            cell['cell_type'] = 'code'
+            cell['source'] = '"""\n' + cell['source'] + '\n"""'
+            cell['outputs'] = []
+            cell['execution_count'] = 0
+        if cell['cell_type'] == 'code' and re.search('![ ]*pip', cell['source']):
+            # replace !pip with #pip
+            cell['source'] = re.sub('![ ]*pip[ ]*install', '# pip install', cell['source'])
+
+    # convert tp python script
+    (code, _) = PythonExporter().from_notebook_node(notebook)
 
     py_path = path.split('/')[-1].replace('.ipynb', '.py')
 

tests/example_script.py

@@ -10,10 +10,10 @@
 import numpy as np
 
 # A comment one line above os.getenv is the description of this variable.
-input_path = os.getenv('input_path')
+input_path = os.environ.get('input_path', None  )  # ('not this')
 
 # type casting to int(), float(), or bool()
-batch_size = int(os.getenv('batch_size', 16))
+batch_size = int(os.environ.get('batch_size', 16))  # (not this)
 
 # Commas in the previous comment are deleted because the yaml file requires descriptions without commas.
 debug = bool(os.getenv('debug', False))

tests/test_compiler.py

@@ -114,7 +114,8 @@ def test_create_operator(
         repository: str,
         args: List,
 ):
-    subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository], check=True)
+    subprocess.run(['python', '../src/c3/create_operator.py', file_path, *args, '-r', repository, '--test_mode'],
+                   check=True)
 
     file = Path(file_path)
     file.with_suffix('.yaml').unlink()
@@ -147,7 +148,7 @@ def test_create_gridwrapper(
         args: List,
 ):
     subprocess.run(['python', '../src/c3/create_gridwrapper.py', file_path, *args,
-                    '-r', repository, '-p', process], check=True)
+                    '-r', repository, '-p', process, '--test_mode'], check=True)
 
     file = Path(file_path)
     gw_file = file.parent / f'gw_{file.stem}.py'