Merge pull request #40 from radiantone/0.2.3

0.2.3

README.md

@@ -1,8 +1,8 @@
-*This version: 0.2.2*
+*This version: 0.2.3*
 
 ![logo](./images/logo.png)
 
-*Current development version is here: [0.2.3](https://github.com/radiantone/entangle/tree/0.2.3)*
+*Current development version is here: [0.2.4](https://github.com/radiantone/entangle/tree/0.2.4)*
 
 A lightweight (serverless) native python parallel processing framework based on simple decorators and call graphs, supporting both *control flow* and *dataflow* execution paradigms as well as de-centralized CPU & GPU scheduling. 
 
@@ -35,7 +35,32 @@ result = add(
 )
 print(result())
 ```
+or train two AI models in parallel using tensorflow container utilizing dedicated CPU and GPU usage.
+```python
+@process
+@docker(image="tensorflow/tensorflow:latest-gpu", packages=['tensorflow_datasets'])
+def train_modelA():
+    # train it
+    return
+
+@process
+@docker(image="tensorflow/tensorflow:latest-gpu", packages=['tensorflow_datasets'])
+def train_modelB():
+    # train it
+    return
+
+@workflow
+def train_models(*args):
+    # I'm training a bunch of models in parallel!
+    return
 
+workflow = train_models(
+    train_modelA(),
+    train_modelB()
+)
+
+result = workflow()
+```
 ### Docker
 To quickly get started with Entangle, build and run a docker container from the included Dockerfile.
 
@@ -98,7 +123,7 @@ Entangle is a *different* kind of parallel compute framework for multi-CPU/GPU e
 It allows for simple workflow design using *plain old python* and special decorators that control the type of parallel compute and infrastructure needed.
 
 One key feature of entangle is fine-grained control over individual functions in a workflow. You could easily describe multiple functions running across multiple compute environments all interacting as if they were simple local python functions.
-No central scheduler or workflow manager is needed allowing you to choosing where and how functions operate with *declarative infrastructure*.
+No central scheduler or workflow manager is needed allowing you to choose where and how functions operate with *declarative infrastructure*.
 
 Another unique quality is the use of composition to build parallel workflows dynamically.
 

entangle/__version__.py

@@ -7,7 +7,7 @@
 
 __title__ = 'py-entangle'
 __description__ = 'A python native parallel processing framework based on simple decorators.'
-__version__ = '0.2.2'
+__version__ = '0.2.3'
 __author__ = 'Darren Govoni'
 __author_email__ = 'darren@ontrenet.com'
 __license__ = 'MIT'

entangle/examples/sshdataflowexample.py

@@ -16,7 +16,7 @@ def triggered(func, result):
 
 
 @dataflow(callback=triggered)
-@ssh(user='darren', host='miko', key='/home/darren/.ssh/id_rsa.pub', python='/home/darren/venv/bin/python')
+#@ssh(user='darren', host='miko', key='/home/darren/.ssh/id_rsa.pub', python='/home/darren/venv/bin/python')
 @process
 def printz(z):
     print('printz: {}'.format(threading.current_thread().name))
@@ -26,7 +26,7 @@ def printz(z):
 
 
 @dataflow(callback=triggered)
-@ssh(user='darren', host='radiant', key='/home/darren/.ssh/id_rsa.pub', python='/home/darren/venv/bin/python')
+#@ssh(user='darren', host='radiant', key='/home/darren/.ssh/id_rsa.pub', python='/home/darren/venv/bin/python')
 @process
 def printx(x):
     print('printx: {}'.format(threading.current_thread().name))

entangle/ssh.py

@@ -142,15 +142,17 @@ def setup_virtualenv(host, user, key, env):
                     __funcname__ = funcdef.name
                     if __funcname__ == func.__name__:
                         decorators = funcdef.decorator_list
-                        ssh_decorator = None
+                        #ssh_decorator = None
 
+                        remove_decorators = []
                         for decorator in decorators:
-                            if hasattr(decorator, 'func') and decorator.func.id == 'ssh':
+                            if hasattr(decorator, 'func') and (decorator.func.id == 'ssh' or decorator.func.id == 'dataflow'):
                                 logging.debug("REMOVE SSH DECORATOR:")
-                                ssh_decorator = decorator
+                                remove_decorators += [decorator]
 
-                        if ssh_decorator:
-                            decorators.remove(ssh_decorator)
+                        [decorators.remove(dec) for dec in remove_decorators]
+                        #if ssh_decorator:
+                        #    decorators.remove(ssh_decorator)
                 '''
                 for funcdef in funcdefs:
                     __funcname__ = funcdef.name