diff --git a/docs/.nojekyll b/docs/.nojekyll deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d04a206a..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS += -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/architecture/Architecture.png b/docs/_static/architecture/Architecture.png deleted file mode 100644 index e33e69a9..00000000 Binary files a/docs/_static/architecture/Architecture.png and /dev/null differ diff --git a/docs/_static/architecture/DAG.png b/docs/_static/architecture/DAG.png deleted file mode 100644 index d7dd5f79..00000000 Binary files a/docs/_static/architecture/DAG.png and /dev/null differ diff --git a/docs/_static/architecture/DetailedSystem.png b/docs/_static/architecture/DetailedSystem.png deleted file mode 100644 index 9eb7704e..00000000 Binary files a/docs/_static/architecture/DetailedSystem.png and /dev/null differ diff --git a/docs/_static/architecture/EdgeComputing.png b/docs/_static/architecture/EdgeComputing.png deleted file mode 100644 index 8b65de18..00000000 Binary files a/docs/_static/architecture/EdgeComputing.png and /dev/null differ diff --git a/docs/_static/architecture/SystemDesign.png b/docs/_static/architecture/SystemDesign.png deleted file mode 100644 index f4e2ccf4..00000000 Binary files a/docs/_static/architecture/SystemDesign.png and /dev/null differ diff --git a/docs/_static/architecture/SystemOverview.png b/docs/_static/architecture/SystemOverview.png deleted file mode 100644 index 43085c95..00000000 Binary files a/docs/_static/architecture/SystemOverview.png and /dev/null differ diff --git a/docs/_static/examples/example_pipeline.png b/docs/_static/examples/example_pipeline.png deleted file mode 100644 index 59988a78..00000000 Binary files a/docs/_static/examples/example_pipeline.png and /dev/null differ diff --git a/docs/_static/logo/chimerapy_logo_theme_blue.png b/docs/_static/logo/chimerapy_logo_theme_blue.png deleted file mode 100644 index 17bf2815..00000000 Binary files a/docs/_static/logo/chimerapy_logo_theme_blue.png and /dev/null differ diff --git a/docs/_static/logo/chimerapy_logo_theme_white.png b/docs/_static/logo/chimerapy_logo_theme_white.png deleted file mode 100644 index fa1fa457..00000000 Binary files a/docs/_static/logo/chimerapy_logo_theme_white.png and /dev/null differ diff --git a/docs/_static/logo/chimerapy_logo_with_name.png b/docs/_static/logo/chimerapy_logo_with_name.png deleted file mode 100644 index e2ebe7b5..00000000 Binary files a/docs/_static/logo/chimerapy_logo_with_name.png and /dev/null differ diff --git a/docs/_static/logo/chimerapy_logo_with_name_theme_blue.png b/docs/_static/logo/chimerapy_logo_with_name_theme_blue.png deleted file mode 100644 index 313ebacc..00000000 Binary files a/docs/_static/logo/chimerapy_logo_with_name_theme_blue.png and /dev/null differ diff --git a/docs/api/index.rst b/docs/api/index.rst deleted file mode 100644 index 368b0171..00000000 --- a/docs/api/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -================= -API Reference -================= - -DataChunk ------------- - -.. autoclass:: chimerapy.engine.DataChunk - :members: __init__, add, get, update - :member-order: bysource - -Node ------------- - -.. autoclass:: chimerapy.engine.Node - :members: __init__, setup, step, teardown, main - :member-order: bysource - -Graph ------------- - -.. autoclass:: chimerapy.engine.Graph - :members: is_valid, plot - :member-order: bysource - -Worker ------------- - -.. autoclass:: chimerapy.engine.Worker - :members: __init__, connect, shutdown - :member-order: bysource - -Manager ------------- - -.. autoclass:: chimerapy.engine.Manager - :members: __init__, commit_graph, step, start, stop, shutdown - :member-order: bysource diff --git a/docs/authors.md b/docs/authors.md deleted file mode 100644 index c0c9484b..00000000 --- a/docs/authors.md +++ /dev/null @@ -1,5 +0,0 @@ -# Contributors - - - Eduardo Davalos (eduardo.davalos.anaya at vanderbilt.edu) - - Umesh Timalsina (umesh.timalsina@vanderbilt.edu) - - Yike Zhang (yike.zhange at vanderbilt.edu) diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index efee8198..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,75 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -sys.path.insert(0, os.path.abspath(".")) -sys.path.insert(0, os.path.abspath("./chimerapy/")) -sys.path.insert(0, os.path.abspath("./source/")) - -# As recommended by SO form: -# https://stackoverflow.com/questions/63261090/github-sphinx-action-cant-find-target-python-modules-and-builds-an-empty-sphinx - -# -- Project information ----------------------------------------------------- - -project = "chimerapy-engine" -copyright = "2022, Vanderbilt OELE Laboratory" -author = "Vanderbilt OELE Laboratory" - -# The full version, including alpha/beta/rc tags -release = "0.0.1" - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "4.5" - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", # to autogenerate documentation - "sphinx.ext.napoleon", # to support Google Docstring - "myst_parser", # include Markdown Support - "sphinx.ext.githubpages", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -source_suffix = [".rst", ".md"] - -# The master toctree document. -master_doc = "index" - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# Removing the module names, based on this SO: https://stackoverflow.com/questions/32858931/how-to-stop-sphinx-automethod-prefixing-method-name-with-class-name -add_module_names = False - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] diff --git a/docs/developer/actors.rst b/docs/developer/actors.rst deleted file mode 100644 index b4bd6989..00000000 --- a/docs/developer/actors.rst +++ /dev/null @@ -1,26 +0,0 @@ -Actors -====== - -Manager ------------- -.. autoclass:: chimerapy.engine.manager.Manager - :undoc-members: - :members: - -Worker ------------- -.. autoclass:: chimerapy.engine.worker.Worker - :undoc-members: - :members: - -Node ------------- -.. autoclass:: chimerapy.engine.node.Node - :undoc-members: - :members: - -Graph ------------- -.. autoclass:: chimerapy.engine.graph.Graph - :undoc-members: - :members: diff --git a/docs/developer/changelog.rst b/docs/developer/changelog.rst deleted file mode 100644 index f5aa320e..00000000 --- a/docs/developer/changelog.rst +++ /dev/null @@ -1,4 +0,0 @@ -ChangeLog and Commit Messages -############################# - -To help autogenerate changelogs, we adopted the `Conventional Commits `_ standard along with a Python `auto-changelog tool `_. Use the instructions provided in their GitHub to update/generate changelog for each release. diff --git a/docs/developer/communication.rst b/docs/developer/communication.rst deleted file mode 100644 index 05c79afe..00000000 --- a/docs/developer/communication.rst +++ /dev/null @@ -1,26 +0,0 @@ -Communication -============= - -Client ------------- -.. autoclass:: chimerapy.engine.networking.Client - :undoc-members: - :members: - -Server ------------- -.. autoclass:: chimerapy.engine.networking.Server - :undoc-members: - :members: - -Publisher --------------- -.. autoclass:: chimerapy.engine.networking.Publisher - :undoc-members: - :members: - -Subscriber --------------- -.. autoclass:: chimerapy.engine.networking.Subscriber - :undoc-members: - :members: diff --git a/docs/developer/docker.rst b/docs/developer/docker.rst deleted file mode 100644 index 145b8835..00000000 --- a/docs/developer/docker.rst +++ /dev/null @@ -1,40 +0,0 @@ -Docker -====== - -To test distributed systems within a single host machine, Docker was used to spin up virtual machines that would simulate local computers. - -.. warning:: - Must of the Docker development was done in Linux (Ubuntu 22.04). Mileage will vary when testing, developing, and using Docker components in other operating systems. - - Make sure you can install Docker in your machine before continuing. - -You will also need sudo access to run Docker. Use the `Offical Docker instructions `_ to give permissions to your user to execute Docker without the need of ``sudo``. - -Docker Image ------------- - -To start, create the Docker image ``chimerapy`` with the following command:: - - docker build -t chimerapy . - -This uses the ``Dockerfile`` in the ChimeraPy's GitHub repository to create an image with all the necessary dependencies. We will use this image to later create container for our manual and automated testing. - -Docker Container and Shell Access ---------------------------------- - -For automated testing, creating the Docker image is sufficient. For manual testing, you can continue with these set of instructions. To avoid having to manage containers, we are going to use the following command to create a Docker container that's ready for testing:: - - sudo docker run --rm -it --entrypoint /bin/bash chimerapy - -With this command, you should be able to have access to the terminal of the Docker container. It should look something like this:: - - root@ce203d4b7798:/# - -This is according to this answer in `SO answer `_. - -Running ChimeraPy Worker ------------------------- - -With our Docker container ready, we can start testing ChimeraPy in a more realistic setting. The ``examples`` folder is a good start for trying out ChimeraPy's functionality. More than likely, the Docker container will need to execute the Worker entrypoint, like this:: - - root@ce203d4b7798:/# cp-worker --ip 10.0.171 --port 9000 --name remote diff --git a/docs/developer/docs.rst b/docs/developer/docs.rst deleted file mode 100644 index eb043380..00000000 --- a/docs/developer/docs.rst +++ /dev/null @@ -1,10 +0,0 @@ -Building the Docs -################# - -Make sure that you have all the necessary dependencies by installing the documentation dependencies, use the following command:: - - pip install '.[docs]' - -The documentation was build with Sphinx with the following command (as executed in the GitHub project root folder):: - - sphinx-build -b html docs/ build/ diff --git a/docs/developer/index.rst b/docs/developer/index.rst deleted file mode 100644 index 376a63c5..00000000 --- a/docs/developer/index.rst +++ /dev/null @@ -1,39 +0,0 @@ -======================= -Developer -======================= - -Hello! Glad that you are interested in contributing to this open-source project. Reading the :ref:`Framework documentation` provides an overview in how ChimeraPy works, which should be a good starting point. Additionally, the API documentation helps describe the source code. - -Environment Setup -======================= - -To create a development environment, use instead the following command:: - - pip install '.[test]' - -With this command, the necessary dependencies for testing will be automatically installed. There are also the following options: ``[docs, benchmark, examples]`` installation add-ons. - -To execute the test, use the following command (located within the root GitHub repository folder):: - - pytest - -In this project, we use the ``pre-commit`` library to verify every commit follows certain standards. This dependency is automatically included in the test dependency installation. `The pre-commit homepage `_ provides information in how its used and advantages. - -Documentation -============= - -.. _developerdocs: - -Below is the developer documentation, which includes all the classes, -methods, and modules used by ChimeraPy. - -.. toctree:: - :maxdepth: 2 - :includehidden: - - changelog.rst - communication.rst - actors.rst - utils.rst - docker.rst - docs.rst diff --git a/docs/developer/utils.rst b/docs/developer/utils.rst deleted file mode 100644 index 823cb15f..00000000 --- a/docs/developer/utils.rst +++ /dev/null @@ -1,6 +0,0 @@ -Utilities -========= - -.. automodule:: chimerapy.utils - :undoc-members: - :members: diff --git a/docs/framework/dag.rst b/docs/framework/dag.rst deleted file mode 100644 index ea2ab417..00000000 --- a/docs/framework/dag.rst +++ /dev/null @@ -1,10 +0,0 @@ -Pipeline Modeling via DAG -################################# - -.. image:: ../_static/architecture/DAG.png - :width: 90% - :alt: DAG Model - -As mentioned in :ref:`Network Typology section`, ChimeraPy uses a peer-to-peer network to represent its data pipeline. To be more precise, ChimeraPy uses a directed acyclic graph (DAG) to model the data pipeline and data streams. The implications of a DAG is that the data streams need to be directed, meaning it must have a source and destination. Acyclic implies that the graph does not permit a cycle. This is because a cycle in a data pipeline would result in a **disaster**! That is because each :class:`Node` will only execute once it has at least collected 1 sample from each of its dependencies, therefore a cycle to cause a deadlock. - -From our use of ChimeraPy, a DAG can commonly be broken into 3 layers: data sources, data processing, and data feedback. Moreover, the DAG provides us with an explicit representation of the execution schedule, therefore no need of a scheduler. diff --git a/docs/framework/detail_design.rst b/docs/framework/detail_design.rst deleted file mode 100644 index 3e2945a0..00000000 --- a/docs/framework/detail_design.rst +++ /dev/null @@ -1,14 +0,0 @@ -Details on the System's Design -############################## - -.. image:: ../_static/architecture/DetailedSystem.png - :width: 90% - :alt: System's Detailed Design - -By stripping the hood of ChimeraPy and see its inter-workings, it comes down to a combination of AIOHTTP, ZeroMQ, and parallel programming. The communication within ChimeraPy can be broken down into two categories: HTTP/WebSocket and PUB/SUB. The :class:`Manager` and :class:`Worker` send messages via HTTP and WebSockets, including the :class:`Server` and :class:`Client`. These messages are not the raw data streams, instead they are operational instructions and information that orchestrates the clusters execution. The data streams use a different communication protocol and channels. The communication between :class:`Node` uses ZeroMQ's PUB/SUB pattern, optimized for speed, robustness, and latency. In specific, the node-communication uses the :class:`Publisher` and :class:`Subscriber` implementations. - -Multiprocessing is at the heart of ChimeraPy, as the base class :class:`Node` is a subclass of Python's build-in multiprocessing's :class:`Process`. Each :class:`Node` executes its ``setup``, ``step``, and ``teardown`` within its own process, to reduce CPU bound limitations. - -In the other side of the parallel programming spectrum, multithreading and AsyncIO are used for relieve the IO bound. More specifically, multithreading is used in active working while waiting, such as writing to video to memory, while AsyncIO is used for networking. - -More details in how each component works can be found in the :ref:`Developer's Documentation`. diff --git a/docs/framework/edge_computing.rst b/docs/framework/edge_computing.rst deleted file mode 100644 index 57efb261..00000000 --- a/docs/framework/edge_computing.rst +++ /dev/null @@ -1,10 +0,0 @@ -Security and Edge-Computing -########################### - -.. image:: ../_static/architecture/EdgeComputing.png - :width: 90% - :alt: Edge Computing - -Before diving into deploying ChimeraPy out in the wild, one should consider its affects on people, primarly their safety, security, and privacy. Streaming data that can identify people's identity (such as video and audio) expands the threat surface of ChimeraPy in a significant way. An cyber attacker can perform a man-in-the-middle attack and capture the data during transmission, breaking user's privacy and confidentiality. - -It is our goal with ChimeraPy to provide secure methods, such as encryption (currently in development) and session passwords (currently in development), to add security into the system. However, even the design of the DAG and its mapping onto computers can add risks. In the figure above, we should how data collected from a student's computer (ChimeraPy applied to education) can collect identifiable data (red nodes). Instead of directly transmitting this sensitive data through the network with the destination to the cluster, one can apply processing methods to anonymize and hide the student's identity. An example would be of processing webcam videos with emotion prediction and transmit the emotion prediction values instead of the video. diff --git a/docs/framework/index.rst b/docs/framework/index.rst deleted file mode 100644 index 4dde3e3a..00000000 --- a/docs/framework/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -========= -Framework -========= - -.. _framework: - -The ChimeraPy project includes various components to provide the utility and features needed to perform multimodal analysis. In this section, we go over main concepts surrounding ChimeraPy and how it works. Below is the list of content: - -.. toctree:: - :maxdepth: 5 - :includehidden: - - network_typology - dag - detail_design - edge_computing diff --git a/docs/framework/network_typology.rst b/docs/framework/network_typology.rst deleted file mode 100644 index bd7bf09d..00000000 --- a/docs/framework/network_typology.rst +++ /dev/null @@ -1,18 +0,0 @@ -Network Typology -################ - -.. _network: - -.. image:: ../_static/architecture/Architecture.png - :width: 90% - :alt: Network Typology Types - -ChimeraPy uses the server-client architecture as a means to provide researchers, engineers, and developers with a cluster controller, where they can setup, start, stop, pause, and resume cluster operations. The main server of ChimeraPy broadcasts this control operations to the entire cluster. - -To then model data pipelines, a peer-to-peer (P2P) typology is put in place. The P2P can be extended to present graphs, as they both share the freedom of establishing edges between any two nodes. - -.. image:: ../_static/architecture/SystemDesign.png - :width: 90% - :alt: System Design - -The combination of these two network typologies results in the diagram above. This is the resulting network design of ChimeraPy. In this case, the cluster's overall Server is the :class:`Manager` and the Client is the :class:`Worker`. The P2P network is embodied by computer processes instead of unique computers. These processes are linked through the scaffolding of the :class:`Worker` and :class:`Manager`. diff --git a/docs/getting_started.rst b/docs/getting_started.rst deleted file mode 100644 index 3975d606..00000000 --- a/docs/getting_started.rst +++ /dev/null @@ -1,32 +0,0 @@ -Getting Started -=============== - -So you want to try ChimeraPy! Great, below are the instructions for -installing the package both through PIPY and GitHub. - -Installation ------------- - -You can install the package through PIPY with the following command.:: - - pip install chimerapy - -Additionally, you can install the package through GitHub instead.:: - - git clone https://github.com/oele-isis-vanderbilt/ChimeraPy - cd ChimeraPy - pip install . - -To confirm that the installation went smoothly, you can test importing the package.:: - - python - >>> import chimerapy.engine as cpe - >>> # Successful importing YAY! - -After confirming that your installation is ready to go, you can start using ChimeraPy. - -Next Steps ----------- - -Now that you have ChimeraPy installed in your Python environment, you -are ready to head to the :ref:`Basics`. Additionally, to better understand the underlying framework, you can checkout the :ref:`Framework docs`. diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 8016e022..00000000 --- a/docs/index.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 4397a1a7..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. chimerapy documentation master file, created by - sphinx-quickstart on Tue Mar 29 03:41:19 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -.. image:: _static/logo/chimerapy_logo_with_name_theme_blue.png - :width: 90% - :alt: ChimeraPy Logo - -|pipy|_ |coverage|_ |test| |license| |style| - -.. |pipy| image:: https://img.shields.io/pypi/v/chimerapy - :alt: PyPI -.. _pipy: https://pypi.org/project/chimerapy/ - -.. |coverage| image:: https://coveralls.io/repos/github/oele-isis-vanderbilt/ChimeraPy/badge.svg?branch=main - :alt: coverage -.. _coverage: https://coveralls.io/github/oele-isis-vanderbilt/ChimeraPy?branch=main - -.. |test| image:: https://img.shields.io/github/actions/workflow/status/oele-isis-vanderbilt/ChimeraPy/.github/workflows/test.yml?branch=main - :alt: test - -.. |license| image:: https://img.shields.io/github/license/oele-isis-vanderbilt/ChimeraPy - :alt: license - -.. |style| image:: https://img.shields.io/badge/style-black-black - :alt: style - -Welcome! -======== - -ChimeraPy is a distributed computing framework for multimodal data dollection, processing, and feedback. It's a real-time streaming tool that leverages from a distributed cluster to empower AI-driven applications. - -#. **Collect** your data in real-time from any computer and time aligned it to the rest of your data streams. -#. **Process** data as soon as it comes, providing a service to your users. -#. **Archive** your outputs and later retrieve them in a single main data archive for later careful post-hoc analysis. -#. **Monitor** the executing of your distributed cluster, view live outputs, and verify that you collected clean data. - - -Contents --------- - -.. toctree:: - :maxdepth: 2 - :includehidden: - - getting_started - usage/index - -.. toctree:: - :maxdepth: 4 - :includehidden: - - framework/index - -.. toctree:: - :maxdepth: 2 - :includehidden: - - api/index - -.. toctree:: - :maxdepth: 3 - :includehidden: - - developer/index - authors - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 2119f510..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 272e0d4d..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Required for GitHub Pages -Sphinx==4.5.0 -attrs>=19.1.0 -myst-parser==0.17.0 -furo==2021.11.16 diff --git a/docs/usage/basics.rst b/docs/usage/basics.rst deleted file mode 100644 index 314ef569..00000000 --- a/docs/usage/basics.rst +++ /dev/null @@ -1,165 +0,0 @@ -Basics -###### - -.. _basics: - -For the basics, we start with the smallest component of the system, the :class:`Node`. The :class:`Node` provides the container for logic in how to collect, process, and return data within the ChimeraPy framework. Then, we will discuss how to incorporate a :class:`Node` into the directed acyclic graph (DAG) pipeline through the :class:`Graph`. We will finish the setup with configuring our local or distributed cluster with :class:`Manager` and :class:`Worker`. Once all setup is complete, we can execute the system and control the cluster. - -Creating a Custom Node -********************** - -To create a custom :class:`Node`, we overwrite the ``setup``, ``step``, and ``teardown`` methods. Below is an example of a :class:`Node` that generates a sequence of random numbers:: - - import chimerapy.engine as cpe - import numpy as np - - class RandomNode(cpe.Node): - - def setup(self): - # Create the generator to be used later - self.rng = np.random.default_rng(0) - - def step(self): - # Set the execution to 1 every second - time.sleep(1) - return self.rng.random() - - def teardown(self): - # Not necessary here, but can be used to close - # files or disconnect from devices. - del self.rng - -It is important to remember that there is three possible node types: source, step, and sink. This is the taxonomy: - -* Source: no inputs, yes outputs -* Step: yes inputs, yes outputs -* Sink: yes inputs, no outputs - -For this example, the ``RandomNode`` is a source node. Step and sink have a ``step(self, data: Dict[str, DataChunk])`` method to retrieve input datas. Since source :class:`Node` do not have inputs, it has a simplified ``step(self)`` method instead. - -The DataChunk Container -*********************** - -A main attribute of ChimeraPy is that it doesn't assume what type of data is being transmitted between :class:`Nodes`. Therefore, when developing your custom node implementations, the ``step`` function can return anything that is serializable. There are moments when this isn't beneficial. For example, to make video streaming work in real time, it is required to compress video frames with an algorithm optimized for images. This implies that ChimeraPy must then know what is being transmitted. This is achieved through the use of the :class:`DataChunk` container. This is an example for a video streaming Node:: - - - class ScreenCapture(cpe.Node): - - def step(self) -> cpe.DataChunk: - - time.sleep(1 / 10) - frame = cv2.cvtColor( - np.array(ImageGrab.grab(), dtype=np.uint8), cv2.COLOR_RGB2BGR - ) - - # Create container and send it - data_chunk = cpe.DataChunk() - data_chunk.add("frame", frame, "image") - return data_chunk - -As of now, the only special compression algorithms incorporated into ChimeraPy are for images. When transmitting images, use the DataChunk with the ``image`` content-type option. Otherwise, ChimeraPy will mark the Node's output as ``other`` and will apply a generic serialization and compression method. - -Creating a DAG -************** - -The creation of a DAG is done through the :class:`Graph` class. The :class:`Graph` is a subclass of `NetworkX `_ `DiGraph `_. To start, we create the Nodes we are interested in putting into our DAG and then, by re-using `nx.DiGraph` API, we can add nodes and edges. An example is shown below:: - - import chimerapy as cpe - - class SourceNode(cpe.Node): - def setup(self): - self.value = 2 - - def step(self): - time.sleep(0.5) - return self.value - - class StepNode(cpe.Node): - def setup(self): - self.coef = 3 - - def step(self, data: Dict[str, cpe.DataChunk]): - time.sleep(0.1) - return self.coef * data["Gen1"].get('default')['value'] - - if __name__ == "__main__": - - # First, create the Nodes - source_node = SourceNode(name="source") - step_node = StepNode(name="step") - - # Create the graph - graph = cpe.Graph() - - # Then add the nodes to the graph - graph.add_nodes_from([source_node, step_node]) - graph.add_edge(source_node, step_node) - -.. note:: - When creating a Node instance, it requires a name to be provided. - -Now with the creation of our graph, we can setup our local or distributed cluster. - -Cluster Setup -************* - -During our cluster setup, we have the many options and configurations to establish. These include what :class:`Worker` objects we want to connect, if we are using a local or distributed system, and delegating :class:`Node` objects to :class:`Worker`. - -Manager-Worker Connection -========================= - -For a local cluster, we can create the :class:`Worker` instance within the local machine. This is how it works:: - - import chimerapy.engine as cpe - - # Create local objects - manager = cpe.Manager() # using default configs - worker = cpe.Worker(name="local") # creating local worker - - # Connect - worker.connect(host=manager.host, port=manager.port) - -For a distributed cluster, the connection setup requires more work. First, we start the :class:`Manager` in the main computer:: - - $ python - >>> import chimerapy as cpe - >>> manager = cpe.Manager() - 2023-07-11 21:01:49 [DEBUG] chimerapy-engine-networking: : running at 192.168.1.155:9000 - 2023-07-11 21:01:49 [INFO] chimerapy-engine: Manager started at 192.168.1.155:9000 - - -Once the :class:`Manager` started, the next step is to access the worker computers and use the ChimeraPy :class:`Worker` connect entrypoint to establish the connection. With the following command, we can connect the worker computer:: - - $ # You will have to obtain your Manager's IP address (ifconfig) - $ cp-worker --host IP_ADDRESS --port 9000 --name worker1 --id worker1 - -With the correct networking information (change IP_ADDRESS with the ip address of your computer hosting the :class:`Manager`), the :class:`Worker` should connect and the :class:`Manager` should report the :class:`Worker` as registered:: - - 2022-11-03 22:42:05 [INFO] chimerapy: WORKER_MESSAGE>: Got connection from ('10.0.0.171', 44326) - -This message informs us that the :class:`Worker` connected successfully. - -Worker-Node Mapping -=================== - -After setting up our cluster, we need to delegate :class:`Nodes` to the :class:`Workers`. This is achieved by using a :class:`Graph` object and a mapping between the workers and the nodes. Then, through a dictionary mapping, where the keys are the workers' names and the values are list of the nodes' names, we can specify which workers will perform which node tasks. Here is an example:: - - # Then register graph to Manager - manager.commit_graph( - graph=graph, - mapping={ - "local": ["source", "step"], - } - ) - -We then commit the :class:`Graph` to the :class:`Worker`. All the :class:`Nodes'` code are located within the :class:`Manager's` computer; therefore, these compartmentalized code needs to be sent to the :class:`Workers`. The ``commit_graph`` routine can take some time based on the number of :class:`Worker`, :class:`Nodes`, and their code size hence waiting until all nodes are ready. - -Execution -********* - -Now we are ready to execute the system, this is achieved through the :class:`Manager`'s control API. Below shows how to start, execute for 10 seconds, and then stop the system:: - - # Take a single step and see if the system crashes and burns - manager.start() - time.sleep(10) - manager.stop() diff --git a/docs/usage/debugging_nodes.rst b/docs/usage/debugging_nodes.rst deleted file mode 100644 index 7884ab81..00000000 --- a/docs/usage/debugging_nodes.rst +++ /dev/null @@ -1,64 +0,0 @@ -Debugging Nodes -################# - -Before executing the cluster and rolling a complex system to handle streams of data, it's best to perform unittest for each Node. By setting the parameter ``debug`` to ``True`` in the constructor of the :class:`Node`, we can begin testing only 1 node. Here is an example:: - - # Imports - import chimerapy as cpe - import time - - class TestingVideoNode(cpe.Node): - - def __init__(self, name:str, id: int = 0, debug: bool = False): - # Do not forget to include parent's arguments! - super().__init__(name, debug) - - # Saving additional input parameters - self.id = id - - def setup(self): - # Create the generator to be used later - self.vid = cap.VideoCapture(self.id) - - def step(self): - # Set the execution to 1 every second - time.sleep(1) - ret, frame = cap.read() - # To transmit video, you need to use DataChunk directly - data_chunk = cp.DataChunk() - data_chunk.add('frame', frame, 'image') - return data_chunk - - def teardown(self): - # Closing the video capture - self.vid.release() - -.. warning:: - When testing the Nodes, make sure to executing ``shutdown`` and/or ``join`` when using ``step`` or ``start``. - -There are two main methods for testing your code: step or streaming. It is recommend to start with step execution to test. Here is how we can test it in a step manner:: - - # Creating the Node - test_node = TestingVideoNode('test', 0, True) - - # Here we can test the step - for i in range(10): - test_node.step() - - # Make sure to shutdown the Node - test_node.shutdown() - -For the streaming testing, instead of executing the Node within the main current process, we are executing the code in a separate subprocess. This makes debugging, logging, and error tracking much harder but it is closer to the execution of the cluster. It's still a good step to test before its integration into the cluster. Here is the code to test it:: - - # Creating the Node - test_node = TestingVideoNode('test', 0, True) - - # Here we can test it during streaming (non-blocking) - test_node.start() - - # Wait for 10 seconds - time.sleep(10) - - # Shutdown - test_node.shutdown() - test_node.join() # Make sure to join the Node (as it is a subprocess) diff --git a/docs/usage/defining_pipeline.rst b/docs/usage/defining_pipeline.rst deleted file mode 100644 index b334e366..00000000 --- a/docs/usage/defining_pipeline.rst +++ /dev/null @@ -1,60 +0,0 @@ -Defining Your Pipeline -###################### - -When constructing your pipeline, the use of :class:`Node` and :class:`Graph` will largely contribute to the data flow and processing. This is done through the representation of a directed acyclic graph, meaning a graph with edges with a direction that do not make a cycle. If a cycle is detected, then ChimeraPy will not allow you to register the graph. Regardless, there are some visualization tools to help construct a graph. - -Instantiate Nodes -***************** - -To define you DAG, first create the nodes that will populate the DAG:: - - import chimerapy.engine as cpe - - a = cpe.Node(name="a") - b = cpe.Node(name="b") - c = cpe.Node(name="c") - d = cpe.Node(name="d") - e = cpe.Node(name="e") - f = cpe.Node(name="f") - -For now we are just going to use empty :class:`Node` for illustrative purposes, but in your own implementation, these nodes would include sensor collecting, processing, and feedback. - -Add Nodes to Graph -****************** - -After creating the :class:`Nodes`, we add them to the :class:`Graph`, like this:: - - graph = cp.Graph() - graph.add_nodes_from([a,b,c,d,e,f]) - graph.add_edges_from([[a, b],[c, d], [c, e], [b, e], [d,f], [e,f]]) - -Handling Frequencies -******************** - -When defining a pipeline, the sampling frequency of :class:`Nodes` needs to be considered. An example where frequency can cause havoc is when two or more Nodes that feed into a single consumer node have high frequency. To avoid data loss, the consumer node would need to execute its own step function at the summative frequency of all incoming nodes. This would put a large computational burden on the cluster and could lead to system failure. There are counter measures to avoid this issue, by specifying which incoming Node should a consumer Node follow. Following refers to that the consumer Node will only execute its step function when it receives data from the Node that it is following. Effectively the escalating frequency crisis is averted. Declaring followship is shown below:: - - hf = HighFrequencyNode(name="hf") - lf = LowFrequencyNode(name="lf") - sn = SubsequentNode(name="sn") - - graph = cp.Graph() - graph.add_nodes_from([hf, lf, sn]) - - graph.add_edge(src=hf, dst=sn) # By default - graph.add_edge(src=lf, dst=sn, follow=True) # <- Overwriting to lf - -By default, the first producer node to a consumer is followed, in this case it would be the ``HighFrequencyNode``. To change it, we overwrite this by adding the ``follow`` parameter when adding a edge between a producer and consumer nodes. - -Check Graph -*********** - -With the graph configuration complete, we can visually confirm that the DAG is constructed correctly, by using the following command:: - - # Should create a matplotlib figure - graph.plot() - -This creates the following plot: - -.. image:: ../_static/examples/example_pipeline.png - :width: 90% - :alt: Example Pipeline diff --git a/docs/usage/dependencies.rst b/docs/usage/dependencies.rst deleted file mode 100644 index 8ce00063..00000000 --- a/docs/usage/dependencies.rst +++ /dev/null @@ -1,17 +0,0 @@ -Dependencies -############ - -Handling Python dependencies across multiple computers is a challenge. As of ``v0.0.8``, dependencies (that are available via package managers) needs to be installed in the other computers. Other packages that are being developed alongside the development of ChimeraPy, typically packages still not published in package managers like ``pip`` or ``conda`` can be handled by ChimeraPy. This can be achieved by specifying package information during commit the graph to the cluster via the ``cp.Manager``. Here is an example:: - - manager.commit_graph( - graph=test_graph, - mapping={"worker1": ["node1", "node2"]} - send_packages=[ - {"name": "test_package", "path": pathlib.Path.cwd() / "test_package"} - ], - ) - -By using the ``send_packages`` parameter, we can specify the name and location of the local package. - -.. warning:: - Using this method of distributing packages is limited, as we cannot transfer packages that rely on ``.so`` files. This is because of the limitation of `zip imports `_. diff --git a/docs/usage/index.rst b/docs/usage/index.rst deleted file mode 100644 index 3bbb3b1f..00000000 --- a/docs/usage/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -===== -Usage -===== - -As much as we try to make ChimeraPy intuitive, here we provide some basic examples and descriptions to help elaborate on how ChimeraPy -works. - -This section covers the following: - -.. toctree:: - :maxdepth: 2 - :includehidden: - - basics - defining_pipeline - intro_tutorial - debugging_nodes - dependencies diff --git a/docs/usage/intro_tutorial.rst b/docs/usage/intro_tutorial.rst deleted file mode 100644 index 5f3f0605..00000000 --- a/docs/usage/intro_tutorial.rst +++ /dev/null @@ -1,121 +0,0 @@ -Tutorial -######## -During this tutorial, we will be using an example, specifically ``remote_camera.py`` provided within the ``examples`` folder of the ChimeraPy-Engine's GitHub repository. This example streams a video from one computer to another and displays the resulting webcam image. - -.. warning:: - This example might not work if your camera permissions do not allow you to access your webcam through OpenCV's ``VideoCapture``. It is recommend that you first try the Local cluster option to test instead of Distributed, as it is easier to debug. - -Data Sources and Process as Nodes -********************************* - -The first step to executing a data pipeline with ChimeraPy is to design your :class:`Node` implementations. In this situation, we have the ``WebcamNode`` that pulls images from the webcam through OpenCV. The receiving node, ``ShowWindow`` displays the webcam video through an OpenCV display. Here are the :class:`Node` definitions:: - - from typing import Dict, Any - import time - - import numpy as np - import cv2 - - import chimerapy.engine as cpe - - class WebcamNode(cpe.Node): - def setup(self): - self.vid = cv2.VideoCapture(0) - - def step(self): - time.sleep(1 / 30) - ret, frame = self.vid.read() - data_chunk = cpe.DataChunk() - data_chunk.add('frame', frame, 'image') - return data_chunk - - def teardown(self): - self.vid.release() - - - class ShowWindow(cp.Node): - def step(self, data: Dict[str, cpe.DataChunk]): - frame = data["web"].get('frame')['value'] - cv2.imshow("frame", frame) - cv2.waitKey(1) - - -An Elegent Approach: Subclassing Graph -************************************** - -Now we have to define our :class:`Graph`. There two main ways to achieve this: create a :class:`Graph` instance and then add the connections, or create a custom :class:`Graph` subclass that defines the connections. In the `Basics`, we show the first approach, therefore we will use the latter in this tutorial. Below is the ``RemoteCameraGraph``:: - - class RemoteCameraGraph(cpe.Graph): - def __init__(self): - super().__init__() - web = WebcamNode(name="web") - show = ShowWindow(name="show") - - self.add_nodes_from([web, show]) - self.add_edge(src=web, dst=show) - -Controlling your Cluster -************************ - -With our DAG complete, the next step is configuring the network configuration and controlling the cluster to start and stop. Make sure, because of mulitprocessing's start methods, to wrap your main code within a ``if __name__ == "__main__"`` to avoid issues, as done below:: - - if __name__ == "__main__": - - # Create default manager and desired graph - manager = cpe.Manager() - graph = RemoteCameraGraph() - worker = cpe.Worker(name="local") - - # Then register graph to Manager - worker.connect(host=manager.host, port=manager.port) - - # Wait until workers connect - while True: - q = input("All workers connected? (Y/n)") - if q.lower() == "y": - break - - # Distributed Cluster Option - # mapping = {"remote": ["web"], "local": ["show"]} - - # Local Cluster Option - mapping = {"local": ["web", "show"]} - - # Commit the graph - manager.commit_graph( - graph=graph, - mapping=mapping - ) - - # Wail until user stops - while True: - q = input("Ready to start? (Y/n)") - if q.lower() == "y": - break - - manager.start() - - # Wail until user stops - while True: - q = input("Stop? (Y/n)") - if q.lower() == "y": - break - - manager.stop() - manager.shutdown() - -In this main code, we have the option to run this between two computers (the Distributed Cluster Option), in which we would have to connect another computer through the entrypoint, as the following:: - - $ cp-worker --ip --port --name remote - -The easier route (to test that the system is working correctly) is to execute the DAG first in your local computer (Local Cluster Option). Now, let's walk through the logic in the main script. - -#. We create the :class:`Manager`, the ``RemoteCameraGraph``, and local :class:`Worker`. -#. Connected :class:`Workers` to :class:`Manager` and provide a wait-for-user to connect remote Workers -#. Map the :class:`Graph` based on either Distributed or Local cluster option -#. Committed the :class:`Graph` and configured the network to deploy the DAG -#. Waits until user is ready to start executing DAG -#. With user approval, DAG is executed, streaming in real time. -#. Waits until user shutdowns sytem. - -For this example, during the runtime of the DAG in ChimeraPy, your webcam (as long as permissions are setup correctly), it should display your current webcam's video in real-time.