From 20ac8ac2eb974c3ffd5161bd7b102861b98db7e4 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Fri, 3 Nov 2023 16:02:35 -0500 Subject: [PATCH] Clean up interface for instruments --- .github/workflows/simple-project.yml | 2 +- .gitignore | 1 + MANIFEST.in | 2 + README.rst | 72 +++++++++------ bin/cmd.py | 91 +++++++++++++++++++ jobrunner/__init__.py | 6 +- jobrunner/__meta__.py | 2 +- .../{resources => instruments}/__init__.py | 0 .../{resources => instruments}/flashx.py | 0 jobrunner/lib/_filetools.py | 10 +- jobrunner/lib/_parsetools.py | 10 +- requirements/core.txt | 4 + requirements/instruments.txt | 3 + setup | 35 ++++++- setup.py | 23 +++++ 15 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 bin/cmd.py rename jobrunner/{resources => instruments}/__init__.py (100%) rename jobrunner/{resources => instruments}/flashx.py (100%) create mode 100644 requirements/core.txt create mode 100644 requirements/instruments.txt diff --git a/.github/workflows/simple-project.yml b/.github/workflows/simple-project.yml index f0fee5a..a1875ca 100644 --- a/.github/workflows/simple-project.yml +++ b/.github/workflows/simple-project.yml @@ -22,7 +22,7 @@ jobs: sudo apt-get update -y && apt-get install -y apt-utils && apt-get upgrade -y sudo apt-get install -y python3 python3-dev python3-pip sudo apt-get install -y python-is-python3 - python3 setup.py develop --user + python3 setup.py develop --user --with-instruments export PATH=$PATH:$HOME/.local/bin - name: Run Tests run: | diff --git a/.gitignore b/.gitignore index ad70e6e..11758c3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ __pycache__ *.rst .sphinx/build/* .sphinx/source/media/* +jobrunner/options.py !.sphinx/source/index.rst !./README.rst diff --git a/MANIFEST.in b/MANIFEST.in index 5314bd7..47f773b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ include README.rst include NOTICE include LICENSE +include bin/* +include requirements/* include media/* diff --git a/README.rst b/README.rst index 3b31435..d8f9873 100644 --- a/README.rst +++ b/README.rst @@ -41,6 +41,15 @@ using, pip install --upgrade PyJobrunner pip uninstall PyJobRunner +The following installation option can be used to allow for using +customization specific to instruments. + +.. code:: + + pip install PyJobruner --user --install-option="--with-instruments" + +This allow for the use of the ``instrument:`` field in the Jobfile + There maybe situations where users may want to install Jobrunner in development mode $\\textemdash$ to design new features, debug, or customize options/commands to their needs. This can be easily @@ -49,7 +58,7 @@ directory and executing, .. code:: - ./setup develop + ./setup develop --with-instruments Development mode enables testing of features/updates directly from the source code and is an effective method for debugging. Note that the @@ -107,9 +116,10 @@ using Flash-X. Application of Jobrunner can be understood better with an example design of a computational experiment. Consider an experiment named `Project` -representative of a publicly available dataset (https://github.com/Lab-Notebooks/Outflow-Forcing-BubbleML) for -the work presented in (https://arxiv.org/pdf/2306.10174.pdf). The directory tree has the following -structure, +representative of a publicly available dataset +(https://github.com/Lab-Notebooks/Outflow-Forcing-BubbleML) for the work +presented in (https://arxiv.org/pdf/2306.10174.pdf). The directory tree +has the following structure, .. code:: console @@ -147,7 +157,7 @@ which is the instrument used to perform the experiments. # # Load Message Passing Interface (MPI) and # Hierarchical Data Format (HDF5) libraries - module load openmpi + module load openmpi module load hdf5 There are situations where requirements for Flash-X are not available as @@ -167,9 +177,10 @@ these, Here the script ``setupAMReX.sh`` provides commands to get the source code for AMReX(https://github.com/AMReX-Codes/amrex) and build it for -desired version and configuration. The script ``setupFlashX.sh`` sets -the version for Flash-X to perform the experiments. The ``Jobfile`` -indicates the use of these files by assigning them to specific Jobrunner commands, +desired version and configuration. The script ``setupFlashX.sh`` sets +the version for Flash-X to perform the experiments. The ``Jobfile`` +indicates the use of these files by assigning them to specific Jobrunner +commands, .. code:: yaml @@ -214,6 +225,8 @@ The ``Jobfile`` at this node assigns the use of ``environment.sh``, # file: Project/Jobfile + instrument: flashx + # Scripts to include during jobrunner setup and submit commands job: setup: @@ -295,8 +308,8 @@ conditions. Each configuration contains its respective ``Jobfile``, Scientific instruments like Flash-X require input during execution which is supplied in the form of parfiles with a ``.par`` extension. The parfiles along a directory tree are combined to create a single input -file when submitting the job. For example, invocation of -``jobrunner submit simulation/PoolBoiling/earth_gravity`` combines +file when submitting the job. For example, invocation of ``jobrunner +submit simulation/PoolBoiling/earth_gravity`` combines ``pool_boiling.par`` and ``earth_gravity.par`` that is used to run the target executable ``flashx`` using the combination of ``environment.sh`` and ``flashRun.sh``. @@ -356,12 +369,12 @@ archiving or cleaning by extending the ``Jobfile`` for each study, Setup ===== -``jobrunner setup `` creates a ``job.setup`` file -using ``job.setup`` scripts defined in Jobfiles along the -directory tree. Jobrunner executes each script serially by changing the -working directory to the location of the script. A special environment -variable ``JobWorkDir`` provides the value of ```` supplied -during invocation of the command. +``jobrunner setup `` creates a ``job.setup`` file using +``job.setup`` scripts defined in Jobfiles along the directory tree. +Jobrunner executes each script serially by changing the working +directory to the location of the script. A special environment variable +``JobWorkDir`` provides the value of ```` supplied during +invocation of the command. .. code:: console @@ -376,10 +389,10 @@ during invocation of the command. Submit ====== -``jobrunner submit `` creates a ``job.submit`` file -using ``job.submit`` scripts and ``schedular.options`` -values defined in Jobfiles along the directory tree. -``schedular.command`` is used to dispatch the resulting script. +``jobrunner submit `` creates a ``job.submit`` file using +``job.submit`` scripts and ``schedular.options`` values defined in +Jobfiles along the directory tree. ``schedular.command`` is used to +dispatch the resulting script. .. code:: console @@ -405,7 +418,8 @@ values defined in Jobfiles along the directory tree. ] Along with the ``job.submit`` script, ``job.input`` and ``job.target`` -files are also created in ```` using values defined in Jobfiles. +files are also created in ```` using values defined in +Jobfiles. Archive ======= @@ -436,17 +450,19 @@ Functionality of Jobrunner is best understood through example projects which can be found in following repositories: - `akashdhruv/Multiphase-Simulations - `_: A - lab notebook to manage development of Flash-X + `_: A lab + notebook to manage development of Flash-X - `Lab-Notebooks/Outflow-Forcing-BubbleML - `_: Reproducibility - capsule for research papers (https://arxiv.org/pdf/2306.10174.pdf) (https://arxiv.org/pdf/2307.14623.pdf) + `_: + Reproducibility capsule for research papers + (https://arxiv.org/pdf/2306.10174.pdf) + (https://arxiv.org/pdf/2307.14623.pdf) - `Lab-Notebooks/Flow-Boiling-3DL - `_: Execution environment for - running three-dimensional flow boiling simulations on high performance computing - systems. + `_: Execution + environment for running three-dimensional flow boiling simulations on + high performance computing systems. ********** Citation diff --git a/bin/cmd.py b/bin/cmd.py new file mode 100644 index 0000000..7f42a64 --- /dev/null +++ b/bin/cmd.py @@ -0,0 +1,91 @@ +"""Custom commands for BoxKit setup.""" +import os +import sys +import subprocess +from setuptools.command.install import install +from setuptools.command.develop import develop + +# custom command +class CustomCmd: + """Custom command.""" + + user_options = [ + ("with-instruments", None, "Install additional modules for instruments"), + ] + + def initialize_options(self): + """ + Initialize options + """ + self.with_instruments = None # pylint: disable=attribute-defined-outside-init + + def finalize_options(self): + """ + Finalize options + """ + for option in [ + "with_instruments", + ]: + if getattr(self, option) not in [None, 1]: + raise ValueError(f"{option} is a flag") + + def run(self, user): + """ + Run command + """ + if user: + with_user = "--user" + else: + with_user = "" + + if self.with_instruments: + subprocess.run( + f"{sys.executable} -m pip install -r requirements/instruments.txt {with_user}", + shell=True, + check=True, + executable="/bin/bash", + ) + + with open("jobrunner/options.py", "w", encoding="ascii") as optfile: + + optfile.write(f"INSTRUMENTS={self.with_instruments}\n") + + +# replaces the default build command for setup.py +class InstallCmd(install, CustomCmd): + """Custom build command.""" + + user_options = install.user_options + CustomCmd.user_options + + def initialize_options(self): + install.initialize_options(self) + CustomCmd.initialize_options(self) + + def finalize_options(self): + install.finalize_options(self) + CustomCmd.finalize_options(self) + + def run(self): + + CustomCmd.run(self, self.user) + install.run(self) + + +# replaces custom develop command for setup.py +class DevelopCmd(develop, CustomCmd): + """Custom develop command.""" + + user_options = develop.user_options + CustomCmd.user_options + + def initialize_options(self): + develop.initialize_options(self) + CustomCmd.initialize_options(self) + + def finalize_options(self): + develop.finalize_options(self) + CustomCmd.finalize_options(self) + + def run(self): + + develop.run(self) + CustomCmd.run(self, self.user) diff --git a/jobrunner/__init__.py b/jobrunner/__init__.py index d129d8a..cbfb60c 100644 --- a/jobrunner/__init__.py +++ b/jobrunner/__init__.py @@ -1,6 +1,10 @@ """Initialization method for PyJobrunner""" +from . import options + +if options.INSTRUMENTS == 1: + from . import instruments + from . import lib -from . import resources from . import api from . import cli diff --git a/jobrunner/__meta__.py b/jobrunner/__meta__.py index 973c377..8fa3681 100644 --- a/jobrunner/__meta__.py +++ b/jobrunner/__meta__.py @@ -1,7 +1,7 @@ """Metadata for jobrunner""" __pkgname__ = "PyJobRunner" -__version__ = "2023.11" +__version__ = "2023.11.01" __authors__ = "Akash Dhruv" __license__ = "Apache Software License" __copyright__ = "Copyright (c) Akash Dhruv 2023. All Rights Reserved." diff --git a/jobrunner/resources/__init__.py b/jobrunner/instruments/__init__.py similarity index 100% rename from jobrunner/resources/__init__.py rename to jobrunner/instruments/__init__.py diff --git a/jobrunner/resources/flashx.py b/jobrunner/instruments/flashx.py similarity index 100% rename from jobrunner/resources/flashx.py rename to jobrunner/instruments/flashx.py diff --git a/jobrunner/lib/_filetools.py b/jobrunner/lib/_filetools.py index 7639213..baa0231 100644 --- a/jobrunner/lib/_filetools.py +++ b/jobrunner/lib/_filetools.py @@ -7,7 +7,10 @@ # local imports from jobrunner import lib -from jobrunner import resources +from jobrunner import options + +if options.INSTRUMENTS == 1: + from jobrunner import instruments def CreateSetupFile(config): @@ -77,7 +80,6 @@ def CreateInputFile(config): job_toml[group].update(node_toml[group]) else: job_toml[group] = node_toml[group] - # start writing the job.input file with open(config.job.workdir + os.sep + "job.input", "w") as inputfile: @@ -106,8 +108,8 @@ def CreateInputFile(config): # else: # inputfile.write(f'{" "*2}{variable} = {value}\n') - if config.instrument == "flashx": - resources.flashx.CreateParfile(config.job.workdir) + if config.instrument == "flashx" and options.INSTRUMENTS == 1: + instruments.flashx.CreateParfile(config.job.workdir) def CreateTargetFile(config): diff --git a/jobrunner/lib/_parsetools.py b/jobrunner/lib/_parsetools.py index defee4c..71d723c 100644 --- a/jobrunner/lib/_parsetools.py +++ b/jobrunner/lib/_parsetools.py @@ -7,6 +7,8 @@ import toml import yaml +from jobrunner import options + class __YamlLoader(yaml.SafeLoader): """ @@ -96,7 +98,13 @@ def ParseJobConfig(basedir, workdir): ) # set values if instrument not already set - config[key] = work_dict[key] + if options.INSTRUMENTS == 1: + config[key] = work_dict[key] + + else: + raise NotImplementedError( + "[jobrunner] Not configured with instruments. Please reinstall with releveant options" + ) continue diff --git a/requirements/core.txt b/requirements/core.txt new file mode 100644 index 0000000..3503ac7 --- /dev/null +++ b/requirements/core.txt @@ -0,0 +1,4 @@ +click +toml +pyyaml +alive-progress==3.1.4 diff --git a/requirements/instruments.txt b/requirements/instruments.txt new file mode 100644 index 0000000..936653e --- /dev/null +++ b/requirements/instruments.txt @@ -0,0 +1,3 @@ +numpy +h5py +scipy==1.8.0 diff --git a/setup b/setup index 7400816..87a8fb7 100755 --- a/setup +++ b/setup @@ -6,23 +6,48 @@ import subprocess import click +def get_options(with_instruments): + options = "" + + if with_instruments: + options = options + "--with-instruments " + + return options + + @click.group(name="setup") def setup(): """Setup toolkit for Jobrunner""" @setup.command(name="develop") -def develop(): +@click.option( + "--with-instruments", + is_flag=True, + help="Install additional modules for instruments", +) +def develop(with_instruments): """Development mode""" - subprocess.run("python3 setup.py develop --user", shell=True, check=True) - subprocess.run("cp jobrunner/scripts/jobrunner $HOME/.local/bin", shell=True, check=True) + options = get_options(with_instruments) + subprocess.run(f"python3 setup.py develop --user {options}", shell=True, check=True) + subprocess.run( + "cp jobrunner/scripts/jobrunner $HOME/.local/bin", shell=True, check=True + ) + @setup.command(name="install") -def install(): +@click.option( + "--with-instruments", + is_flag=True, + help="Install additional modules for instruments", +) +def install(with_instruments): """Installation command""" + options = get_options(with_instruments) subprocess.run("python3 setup.py develop --user", shell=True, check=True) subprocess.run("python3 setup.py build", shell=True, check=True) - subprocess.run("python3 setup.py install --user", shell=True, check=True) + subprocess.run(f"python3 setup.py install --user {options}", shell=True, check=True) + @setup.command(name="publish") def publish(): diff --git a/setup.py b/setup.py index 38a917e..191bf83 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,18 @@ """Build and installation script for jobrunner.""" # standard libraries +import os +import sys import re from setuptools import setup, find_packages +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Import bin from current working directory +# sys.path.insert makes sure that current file path is searched +# first to find this module +import bin.cmd as bin_cmd # pylint: disable=wrong-import-position + # get long description from README.rst with open("README.rst", mode="r") as readme: long_description = readme.read() @@ -25,6 +34,11 @@ # core dependancies DEPENDENCIES = ["click", "toml", "pyyaml", "alive-progress==3.1.4"] +# core dependancies for the package +with open("requirements/core.txt", mode="r", encoding="ascii") as core_reqs: + DEPENDENCIES = core_reqs.read() + + setup( name=metadata["__pkgname__"], version=metadata["__version__"], @@ -39,6 +53,11 @@ "jobrunner/scripts/catlog", "jobrunner/scripts/catloglast", ], + package_data={ + "": [ + "options.py", + ] + }, include_package_data=True, long_description=long_description, classifiers=[ @@ -46,4 +65,8 @@ "License :: OSI Approved :: Apache Software License", ], install_requires=DEPENDENCIES, + cmdclass={ + "develop": bin_cmd.DevelopCmd, + "install": bin_cmd.InstallCmd, + }, )