Merge pull request #74 from golemfactory/mateusz/ray-on-golem-pre-alpha

Ray on Golem docs - don't merge yet :)
golemfactory · Sep 21, 2023 · 13d103b · 13d103b
2 parents 1fc449f + c640154
commit 13d103b
Show file tree

Hide file tree

Showing 21 changed files with 1,154 additions and 1 deletion.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,6 +37,7 @@
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-highlight-words": "^0.20.0",
+    "react-lite-youtube-embed": "^2.3.52",
     "simple-functional-loader": "^1.2.1",
     "swr": "^2.1.5",
     "tailwindcss": "^3.3.0"

diff --git a/src/components/Footer.jsx b/src/components/Footer.jsx
@@ -5,7 +5,7 @@ import { GolemIcon } from './icons/GolemIcon'
 import { LinkedInIcon } from './icons/LinkedIn'
 import { RedditIcon } from './icons/RedditIcon'
 import { YoutubeIcon } from './icons/YoutubeIcon'
-import { javascript, python, dapps } from '@/navigation/footer'
+import { javascript, python, ray, dapps } from '@/navigation/footer'
 import { ArrowRightIcon } from './icons/ArrowRightIcon'
 import Link from 'next/link'
 
@@ -107,6 +107,12 @@ export const Footer = () => (
           description={''}
           link={'/docs/creators/javascript'}
         />
+        <DocSection
+          title="Ray on Golem Docs"
+          data={ray}
+          description={''}
+          link={'/docs/creators/ray'}
+        />
         <DocSection
           title="Learn Dapps"
           data={dapps}

diff --git a/src/components/Icon.jsx b/src/components/Icon.jsx
@@ -14,6 +14,7 @@ import { DiscordIconBlue } from '@/components/icons/DiscordIconBlue'
 import { GithubIcon } from '@/components/icons/GithubIcon'
 import { JSIcon } from '@/components/icons/JSIcon'
 import { PythonIcon } from '@/components/icons/PythonIcon'
+import { RayIcon } from '@/components/icons/RayIcon'
 
 const icons = {
   installation: InstallationIcon,
@@ -29,6 +30,7 @@ const icons = {
   github: GithubIcon,
   javascript: JSIcon,
   python: PythonIcon,
+  ray: RayIcon,
 }
 
 export function Icon({ icon, ...props }) {

diff --git a/src/components/YoutubeEmbed.jsx b/src/components/YoutubeEmbed.jsx
@@ -0,0 +1,13 @@
+import LiteYouTubeEmbed from 'react-lite-youtube-embed'
+import 'react-lite-youtube-embed/dist/LiteYouTubeEmbed.css'
+
+function extract_id_from_link(url) {
+  var regExp =
+    /^.*((youtu.be\/)|(v\/)|(\/u\/\w\/)|(embed\/)|(watch\?))\??v?=?([^#&?]*).*/
+  var match = url.match(regExp)
+  return match && match[7].length == 11 ? match[7] : false
+}
+
+export function YoutubeEmbed({ link }) {
+  return <LiteYouTubeEmbed id={extract_id_from_link(link)} />
+}
diff --git a/src/components/icons/RayIcon.jsx b/src/components/icons/RayIcon.jsx
@@ -0,0 +1,14 @@
+import * as React from 'react'
+export const RayIcon = (props) => (
+  <svg
+    width="683px"
+    height="512px"
+    viewBox="0 0 682 512"
+    version="1.1"
+    {...props}
+  >
+    <g id="surface1">
+      <path d="M 287.386719 242.429688 C 289.878906 232.722656 294.90625 223.851562 301.949219 216.734375 C 312.320312 206.328125 326.410156 200.492188 341.09375 200.515625 C 356.386719 200.515625 370.21875 206.710938 380.238281 216.734375 C 387.285156 223.851562 392.308594 232.722656 394.800781 242.429688 L 435.574219 242.429688 C 436.878906 237.160156 438.964844 232.121094 441.769531 227.476562 L 369.496094 155.132812 C 360.898438 160.242188 351.089844 162.953125 341.09375 162.976562 C 326.410156 163 312.320312 157.160156 301.949219 146.757812 C 291.554688 136.375 285.722656 122.273438 285.746094 107.574219 C 285.746094 92.265625 291.9375 78.414062 301.949219 68.390625 C 312.320312 57.984375 326.410156 52.148438 341.09375 52.171875 C 356.386719 52.171875 370.21875 58.371094 380.238281 68.390625 C 390.632812 78.773438 396.464844 92.875 396.441406 107.574219 C 396.441406 117.585938 393.734375 127.410156 388.605469 136.003906 L 460.878906 208.347656 C 469.480469 203.238281 479.292969 200.53125 489.289062 200.515625 C 504.585938 200.515625 518.414062 206.710938 528.425781 216.734375 C 538.820312 227.113281 544.648438 241.210938 544.628906 255.90625 C 544.652344 270.605469 538.820312 284.707031 528.425781 295.089844 C 518.054688 305.492188 503.964844 311.332031 489.28125 311.308594 C 479.285156 311.3125 469.472656 308.605469 460.890625 303.476562 L 388.605469 376.007812 C 393.523438 384.382812 396.441406 394.039062 396.441406 404.425781 C 396.464844 419.125 390.632812 433.226562 380.238281 443.609375 C 369.867188 454.015625 355.777344 459.851562 341.09375 459.828125 C 326.410156 459.851562 312.320312 454.015625 301.949219 443.609375 C 291.554688 433.226562 285.722656 419.125 285.746094 404.425781 C 285.746094 389.117188 291.9375 375.273438 301.949219 365.253906 C 312.320312 354.847656 326.410156 349.011719 341.09375 349.035156 C 351.09375 349.027344 360.910156 351.734375 369.496094 356.867188 L 441.769531 284.523438 C 438.964844 279.882812 436.878906 274.84375 435.574219 269.582031 L 394.800781 269.582031 C 392.308594 279.289062 387.285156 288.15625 380.238281 295.277344 C 369.867188 305.679688 355.777344 311.519531 341.09375 311.496094 C 326.410156 311.519531 312.320312 305.679688 301.949219 295.277344 C 294.90625 288.15625 289.878906 279.285156 287.386719 269.582031 L 246.601562 269.582031 C 244.113281 279.285156 239.085938 288.15625 232.039062 295.277344 C 221.667969 305.679688 207.582031 311.519531 192.898438 311.496094 C 178.210938 311.519531 164.125 305.679688 153.753906 295.277344 C 143.234375 284.960938 137.328125 270.832031 137.371094 256.09375 C 137.371094 240.785156 143.5625 226.933594 153.574219 216.910156 C 163.945312 206.507812 178.035156 200.667969 192.71875 200.691406 C 208.011719 200.691406 221.84375 206.890625 231.863281 216.910156 C 238.90625 224.03125 243.933594 232.902344 246.425781 242.605469 L 287.386719 242.605469 L 287.386719 242.417969 Z M 321.070312 424.476562 C 326.167969 429.578125 333.269531 432.855469 341.09375 432.855469 C 348.917969 432.855469 356.011719 429.753906 361.117188 424.476562 C 366.210938 419.378906 369.496094 412.269531 369.496094 404.425781 C 369.496094 396.59375 366.398438 389.492188 361.117188 384.382812 C 355.816406 379.042969 348.613281 376.03125 341.09375 376.007812 C 333.257812 376.007812 326.167969 379.105469 321.0625 384.382812 C 315.726562 389.691406 312.714844 396.90625 312.691406 404.433594 C 312.691406 412.269531 315.789062 419.367188 321.0625 424.476562 Z M 509.3125 276.136719 C 514.40625 271.035156 517.683594 263.925781 517.683594 256.09375 C 517.683594 248.25 514.585938 241.152344 509.3125 236.042969 C 504.011719 230.699219 496.804688 227.6875 489.28125 227.664062 C 481.457031 227.664062 474.363281 230.765625 469.257812 236.042969 C 463.921875 241.351562 460.910156 248.5625 460.890625 256.09375 C 460.890625 263.925781 463.984375 271.027344 469.257812 276.136719 C 474.355469 281.234375 481.457031 284.523438 489.289062 284.523438 C 496.792969 284.410156 503.964844 281.40625 509.3125 276.136719 Z M 361.117188 87.523438 C 355.816406 82.183594 348.613281 79.167969 341.09375 79.144531 C 333.257812 79.144531 326.167969 82.246094 321.0625 87.523438 C 315.726562 92.832031 312.714844 100.042969 312.691406 107.574219 C 312.691406 115.40625 315.789062 122.507812 321.0625 127.617188 C 326.167969 132.714844 333.257812 136.003906 341.09375 136.003906 C 348.917969 136.003906 356.011719 132.902344 361.117188 127.617188 C 366.210938 122.515625 369.496094 115.40625 369.496094 107.574219 C 369.386719 100.058594 366.386719 92.875 361.117188 87.523438 Z M 172.6875 236.042969 C 167.351562 241.351562 164.339844 248.5625 164.316406 256.09375 C 164.316406 263.925781 167.414062 271.027344 172.6875 276.136719 C 177.792969 281.234375 184.882812 284.523438 192.71875 284.523438 C 200.542969 284.523438 207.636719 281.421875 212.742188 276.136719 C 217.835938 270.847656 221.121094 263.925781 221.121094 256.09375 C 221.121094 248.25 218.023438 241.152344 212.742188 236.042969 C 207.457031 230.941406 200.542969 227.664062 192.71875 227.664062 C 184.882812 227.664062 177.792969 230.765625 172.6875 236.042969 Z M 321.0625 236.042969 C 315.726562 241.351562 312.714844 248.5625 312.691406 256.09375 C 312.691406 263.925781 315.789062 271.027344 321.0625 276.136719 C 326.167969 281.234375 333.257812 284.523438 341.09375 284.523438 C 348.917969 284.523438 356.011719 281.421875 361.117188 276.136719 C 366.210938 271.035156 369.496094 263.925781 369.496094 256.09375 C 369.496094 248.25 366.398438 241.152344 361.117188 236.042969 C 355.816406 230.703125 348.613281 227.691406 341.09375 227.664062 C 333.257812 227.664062 326.167969 230.765625 321.0625 236.042969 Z M 321.0625 236.042969 " />
+    </g>
+  </svg>
+)
diff --git a/src/markdoc/tags.js b/src/markdoc/tags.js
@@ -13,6 +13,7 @@ import { SelectionCard } from '@/components/SelectionCard'
 import { SelectionContent } from '@/components/SelectionContent'
 import { Grid } from '@/components/Grid'
 import { Padding } from '@/components/Padding'
+import { YoutubeEmbed } from '@/components/YoutubeEmbed'
 import { MarginBottom } from '@/components/MarginBottom'
 import { CustomError } from '../components/CustomError'
 import { GithubCode } from '@/components/GithubCode'
@@ -150,6 +151,13 @@ const tags = {
     selfClosing: true,
     render: Whitespace,
   },
+  youtube: {
+    selfClosing: true,
+    attributes: {
+      link: { type: String },
+    },
+    render: YoutubeEmbed,
+  },
   marginbottom: {
     selfClosing: true,
     render: MarginBottom,

diff --git a/src/navigation/docs.js b/src/navigation/docs.js
@@ -15,6 +15,10 @@ export const navigation = [
         title: 'Golem in a Browser',
         href: '/docs/quickstarts/golem-in-a-browser',
       },
+      {
+        title: 'Ray on Golem Quickstart',
+        href: '/docs/quickstarts/ray-quickstart',
+      },
       {
         title: '"no-code" app deployment',
         href: '/docs/quickstarts/no-code-app-deployment',
@@ -200,6 +204,37 @@ export const navigation = [
           },
         ],
       },
+      {
+        title: 'Python with Ray on Golem',
+        children: [
+	  { title: 'Introduction', href: '/docs/creators/ray' },
+	  { title: 'Quickstart', href: '/docs/creators/ray/quickstart' },
+	  {
+	    title: 'Concept',
+	    href: '/docs/creators/ray/concept',
+	  },
+	  {
+	    title: 'Setup tutorial',
+	    href: '/docs/creators/ray/setup-tutorial',
+	  },
+	  {
+	    title: 'Basic Ray tasks usage tutorial',
+	    href: '/docs/creators/ray/basic-ray-tasks-usage-tutorial',
+	  },
+	  {
+	    title: 'Converting a real-life use case to Ray on Golem',
+	    href: '/docs/creators/ray/conversion-to-ray-on-golem-tutorial',
+	  },
+	  {
+	    title: 'Cluster yaml reference',
+	    href: '/docs/creators/ray/cluster-yaml-reference',
+	  },
+	  {
+	    title: 'Supported versions and other limitations',
+	    href: '/docs/creators/ray/supported-versions-and-other-limitations',
+	  },
+        ],
+      },
       {
         title: 'Deploying apps on Golem',
         children: [

diff --git a/src/navigation/footer.jsx b/src/navigation/footer.jsx
@@ -172,3 +172,46 @@ export const dapps = [
     ],
   },
 ]
+
+
+export const ray = [
+  {
+    type: 'Examples',
+    links: [],
+  },
+  {
+    type: 'Tutorials',
+    links: [],
+  },
+  {
+    type: 'Guides',
+    links: [
+      { title: 'Introduction', href: '/docs/creators/ray' },
+      { title: 'Quickstart', href: '/docs/creators/ray/quickstart' },
+      {
+        title: 'Concept',
+        href: '/docs/creators/ray/concept',
+      },
+      {
+        title: 'Setup tutorial',
+        href: '/docs/creators/ray/setup-tutorial',
+      },
+      {
+        title: 'Basic Ray tasks usage tutorial',
+        href: '/docs/creators/ray/basic-ray-tasks-usage-tutorial',
+      },
+      {
+        title: 'Converting a real-life use case to Ray on Golem',
+        href: '/docs/creators/ray/conversion-to-ray-on-golem-tutorial',
+      },
+      {
+        title: 'Cluster yaml reference',
+        href: '/docs/creators/ray/cluster-yaml-reference',
+      },
+      {
+        title: 'Supported versions and other limitations',
+        href: '/docs/creators/ray/supported-versions-and-other-limitations',
+      },
+    ],
+  },
+]
diff --git a/src/pages/docs/creators/index.md b/src/pages/docs/creators/index.md
@@ -34,6 +34,28 @@ Need a backend to offload your system?
 
 {% /selectioncard %}
 
+{% selectioncard icon="ray" title="Ray on Golem Docs" buttonText="Read Ray on Golem Docs" href="/docs/creators/ray" %}
+
+{% selectioncontent %}
+
+Have a Python code that can benefit from running on more than just your computer?
+
+{% /selectioncontent %}
+
+{% selectioncontent %}
+
+Want to parallelize your code using Ray and run it on a Golem cluster?
+
+{% /selectioncontent %}
+
+{% selectioncontent %}
+
+Need to have control over the costs and performance?
+
+{% /selectioncontent %}
+
+{% /selectioncard %}
+
 {% selectioncard icon="python" title="No-code Deployment" buttonText="Read no-code Docs" href="/docs/creators/dapps" %}
 
 {% selectioncontent %}

diff --git a/src/pages/docs/creators/ray/basic-ray-tasks-usage-tutorial.md b/src/pages/docs/creators/ray/basic-ray-tasks-usage-tutorial.md
@@ -0,0 +1,117 @@
+---
+description: anatomy of Ray application based on a simple code showcasing Ray tasks
+title: Basic Ray tasks usage tutorial 
+type: example 
+---
+
+# Basic Ray tasks usage examples
+
+The purpose of this article is to show you an example of a Ray app. It shows how Ray tasks are executed on different machines.
+
+We have also prepared a couple of other example Ray apps to make it easier for you to play with Ray on Golem and to help you see the power of Ray.
+You can find the examples in [`ray-on-golem` repository](https://github.com/golemfactory/golem-ray/tree/main/examples).
+
+
+## The app
+
+Have a look at the code (but you can find it also in [the repository](https://github.com/golemfactory/golem-ray/blob/main/examples/simple-task.py):
+```python
+from collections import Counter
+import socket
+import time
+
+import ray
+ray.init()
+
+print('''This cluster consists of
+    {} nodes in total
+    {} CPU resources in total
+'''.format(len(ray.nodes()), ray.cluster_resources()['CPU']))
+
+@ray.remote
+def f():
+    time.sleep(0.5)
+
+    # Return IP address.
+    return socket.gethostbyname(socket.gethostname())
+
+object_ids = [f.remote() for _ in range(1000)]
+ip_addresses = ray.get(object_ids)
+
+print('Tasks executed')
+for ip_address, num_tasks in Counter(ip_addresses).items():
+    print('    {} tasks on {}'.format(num_tasks, ip_address))
+
+
+print('''This cluster consists of
+    {} nodes in total
+    {} CPU resources in total
+'''.format(len(ray.nodes()), ray.cluster_resources()['CPU']))
+```
+
+You can run the app on your local machine with
+```bash
+python3 simple-task.py
+```
+
+Or, you can execute it on Ray on the Golem cluster with `ray submit`. Let's have a look at ray usage in the app.
+
+You need to run certain steps:
+- initialize ray, so your code would be able to use ray infrastructure
+- declare the methods that can be run remotely by ray
+- apply ray pattern to execute tasks remote with `remote()` and await the results with `ray.get()`
+
+
+## Ray initialization
+
+```python
+import ray
+ray.init()
+```
+
+This is what Ray initialization looks like. `ray.init()` without parameters tells Ray to look for an existing Ray cluster or (if not found) to start up a local Ray instance.
+
+
+## Cluster information
+
+The `ray` module exposes methods we use to print the information about the cluster (before and after ray computation).
+When you run the script on a fresh Ray on the Golem cluster, the number of nodes will increase as a result of the computation.
+This happens because Ray autoscaler monitors the amount of work pending and requests additional nodes as the queues get longer.
+
+It also decommissions the nodes when they start to idle (5 mins in the example config we are providing).
+
+## Remote tasks declaration and execution
+
+```python
+@ray.remote
+def f:
+```
+
+Ray's `remote` decorator turns a regular local function into Ray's object, which enables the function to be executed remotely (on a different node). 
+When you subsequently call its `.remote` method, the Ray scheduler will queue its execution. 
+In our case, the function just sleeps for a moment and then returns the IP address of the node it was executed on.
+
+```python
+object_ids = [f.remote() for _ in range(1000)]
+```
+
+What happens when you call the `remote` method added by the `@ray.remote` decorator, is that it immediately returns an id - a kind of future promise - that can be used to get the results later.
+
+## Waiting for the results of remote calls
+
+```python
+ip_addresses = ray.get(object_ids)
+```
+
+The future(s) can be awaited with `ray.get`. It returns only when all the remote tasks are executed.
+
+
+## Conclussion
+
+And that's it - the code parallelization with Ray is done by choosing which parts of code can be executed remotely, decorating them with `@ray.remote`, then changing their execution to `.remote()`, and finally waiting for the results with `ray.get()`. Of course, remote Ray tasks can call other Ray remote tasks.
+
+You can now proceed to run your app on a [Ray on Golem cluster](/docs/creators/ray/setup-tutorial)
+
+Read more information about Ray tasks on [Ray Core docs website](https://docs.ray.io/en/latest/ray-core/walkthrough.html)
+
+