From c44b714d102e8910d1f924e0748aa8805f08dc4b Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 1 Aug 2023 16:29:05 +0530 Subject: [PATCH 1/7] doc: simplify build guide --- README.md | 48 +++++----------------------------------------- docs/build.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 43 deletions(-) create mode 100644 docs/build.md diff --git a/README.md b/README.md index 51db0307..3721b36d 100644 --- a/README.md +++ b/README.md @@ -14,42 +14,12 @@ Uplink is a rust based utility for efficiently sending data and receiving comman - Provides remote shell access through [Tunshell][tunshell] - Supports TLS with easy cross-compilation. -### Install Pre-built Binary -Pre-built binaries for select platform targets have been made available in the [repo's releases page](https://github.com/bytebeamio/uplink/releases/), download the binary specific to your system and add it's location to PATH. +### Quickstart -### Build and Install +#### Install Uplink -Build and install with [Cargo][crates.io]: - -```sh -cargo install uplink # NOTE: This should work once crate is published -``` - -#### Build and run from source: - -```sh -git clone https://github.com/bytebeamio/uplink.git -cd uplink -cargo run --bin uplink -- -a -``` -#### Build from source for ARM systems - -In case you want to run uplink on an ARM system, follow instruction given below to create an ARM compatible binary. - -1. Install `cross`, a `Zero setup` cross compilation crate. -```sh -cargo install cross -``` -2. Build binary for the target `armv7-unknown-linux-gnueabihf`. -```sh -cross build --release --target armv7-unknown-linux-gnueabihf -``` -3. Retreive executable from `/target/armv7-unknown-linux-gnueabihf/release/uplink` and execute it on target device. - -See [releases][releases] for other options. - -### Getting Started +Pre-built binaries for select platform targets have been made available in [releases], download the binary specific to your system and add it's location to PATH. You can also follow the [build guide]. #### Setup You can start uplink with the following command, where you will need to provide an `auth.json` file: @@ -203,19 +173,11 @@ nc localhost 5555 The complete API reference for the uplink library is available within the [library documentation][docs.rs]. -# Android executable instructions - -* install `cargo-ndk` v2.11.0 (`cargo install cargo-ndk --version 2.11.0`) -* `export ANDROID_NDK_HOME=~/Android/Sdk/ndk/25.0.8775105` -* `cargo ndk --target x86_64-linux-android --platform 23 build --bin uplink` -* move executable to `/data/local/` -* set port to something else -* set persistence path to `/data/local/uplink` -* `chmod +x uplink_exe` - ### Contributing Please follow the [code of conduct][coc] while opening issues to report bugs or before you contribute fixes, also do read our [contributor guide][contribute] to get a better idea of what we'd appreciate and what we won't. + +[build guide]: docs/build.md [workflow-badge]: https://github.com/bytebeamio/uplink/actions/workflows/rust.yml/badge.svg [workflow]: https://github.com/bytebeamio/uplink/actions/workflows/rust.yml [twitter-badge]: https://img.shields.io/twitter/follow/bytebeamio.svg?style=social&label=Follow diff --git a/docs/build.md b/docs/build.md new file mode 100644 index 00000000..d5fd8843 --- /dev/null +++ b/docs/build.md @@ -0,0 +1,53 @@ +# Build uplink from source + +1. Get source code from github: +```sh +git clone https://github.com/bytebeamio/uplink.git +``` +2. Build uplink your system's architecture, the binary should be in `target/release/` +```sh +cargo build --release +``` +## For ARM systems + +In case you want to run uplink on an ARM system, follow instruction given below to create an ARM compatible binary. + +1. Install `cross`, a `Zero setup` cross compilation crate. +```sh +cargo install cross +``` +2. Build binary for the target architecture, e.g: `armv7-unknown-linux-gnueabihf`. +```sh +cross build --release --target armv7-unknown-linux-gnueabihf +``` +3. Retreive executable from `/target/armv7-unknown-linux-gnueabihf/release/uplink` and execute it on target device. + +## For Android systems + +For an android target, the following instructions should help you build and install a compatible binary. +1. Install cargo-ndk v2.11.0 +```sh +cargo install cargo-ndk --version 2.11.0 +``` +2. Ensure `ANDROID_NDK_HOME` env variable is set to point to the android NDK directory. +```sh +export ANDROID_NDK_HOME=~/Android/Sdk/ndk/25.0.8775105 +``` +3. Run the cargo-ndk tool to build uplink for the specific version of android +```sh +cargo ndk --target x86_64-linux-android --platform 23 build --bin uplink +``` +4. Move executables to `/data/local/` on target device +5. Set the application's uplink port appropriately in the config.toml file +6. Include the following line in config.toml +``` +persistence_path = "/data/local/uplink" +``` +7. Ensure the uplink binary is executable and start it in the background +```sh +chmod +x uplink_exe +``` + +See [releases] for other options. + +[releases]: https://github.com/bytebeamio/uplink/releases/ \ No newline at end of file From b32734f96b8ee6c0bcd5a254381936c82b27c04f Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 1 Aug 2023 17:02:45 +0530 Subject: [PATCH 2/7] doc: improve statement --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3721b36d..9704b7f6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Uplink is a rust based utility for efficiently sending data and receiving comman #### Install Uplink -Pre-built binaries for select platform targets have been made available in [releases], download the binary specific to your system and add it's location to PATH. You can also follow the [build guide]. +Pre-built binaries for select platform targets have been made available in [releases], download the binary specific to your system and add it's location to PATH. To compile uplink on your own, please follow the [build guide]. #### Setup You can start uplink with the following command, where you will need to provide an `auth.json` file: From 6da8ad6ad99092595414ae6b5c003654a21eac79 Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Tue, 1 Aug 2023 18:20:42 +0530 Subject: [PATCH 3/7] doc: fix release page link statement --- docs/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build.md b/docs/build.md index d5fd8843..9e58a396 100644 --- a/docs/build.md +++ b/docs/build.md @@ -48,6 +48,6 @@ persistence_path = "/data/local/uplink" chmod +x uplink_exe ``` -See [releases] for other options. +See [releases] for pre-compiled binaries to select platforms/architectures. [releases]: https://github.com/bytebeamio/uplink/releases/ \ No newline at end of file From e9cbdb4b0b5dbf817a59f8933ffce9562daafd8f Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Thu, 3 Aug 2023 18:20:19 +0530 Subject: [PATCH 4/7] doc: improve writeup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9704b7f6..ec8d7194 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ uplink -a auth.json The `auth.json` file must contain information such as the device's ID, the broker's URL, the port to connect to and the TLS certificates to be used while connecting as can be seen inside [dummy.json][dummy]. When connecting over non-TLS connections, authentication information is unncessary as illustrated by [noauth.json][noauth]. -> **NOTE**: If you are using [Bytebeam][bytebeam], you could download the file downloaded [from the Bytebeam UI][platform]. If you are using your own broker instead, you could use uplink [without TLS][unsecure], but we recommend that you use TLS and [provision your own certificates][provision] to do it. You can read more about securing uplink in the [uplink Security document][security] +> **NOTE**: If you are using [Bytebeam][bytebeam], you could use the file downloaded [from the Bytebeam UI][platform]. If you are using your own broker instead, you could use uplink [without TLS][unsecure], but we recommend that you use TLS and [provision your own certificates][provision] to do it. You can read more about securing uplink in the [uplink Security document][security]. #### Configuring `uplink` One may configure certain features of uplink with the help of a `config.toml` file by using the commandline arguments `-c` or `--config`: @@ -41,7 +41,7 @@ uplink -a auth.json -c config.toml It must be noted that parts of, or the entirety of the config file is optional and a user may choose to omit it, letting uplink default to configuration values that are compiled into the binary. uplink only expects the `config.toml` to contain configuration details as given in the [example config.toml][config] file in the configs folder. #### Writing Applications -uplink acts as an intermediary between the user's applications and the Bytebeam platform/MQTT 3.1.1 broker of choice. One can accept [Action][action]s from the cloud and push data(from applications such as sensing) or [Action Response][action_response]s back. +uplink acts as an intermediary between the user's application running on edge devices and the Bytebeam platform/MQTT 3.1.1 broker of choice. One can accept [Action][action]s from the cloud and push data(from applications such as sensing) or [Action Response][action_response]s back. uplink architecture From 5fb5726eda8f3792292789fba8c29aa69b614bca Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Thu, 3 Aug 2023 18:24:28 +0530 Subject: [PATCH 5/7] doc: add clarification requiring newline encoding --- README.md | 3 ++ configs/simulator.toml | 9 ++---- examples/demo.py | 73 ++++++------------------------------------ run.sh | 6 ++++ 4 files changed, 21 insertions(+), 70 deletions(-) create mode 100644 run.sh diff --git a/README.md b/README.md index ec8d7194..473fbb09 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Data from the connected application is handled as payload within a stream. uplin // ...payload: more JSON data } ``` +> **NOTE**: Connected application should forward all data events as JSON text, following ASCII newline encoding. An example data packet on the stream `"location"`, with the fields `"city"` being a string and `"altitude"` being a number would look like: ```js @@ -95,6 +96,8 @@ Applications can use Action Response messages to update uplink on the progress o } ``` +> **NOTE**: Connected application should forward all response events as JSON text, following ASCII newline encoding. + An example success response to an action with the id `"123"`, would look like: ```js { diff --git a/configs/simulator.toml b/configs/simulator.toml index fe9aab23..c4f08998 100644 --- a/configs/simulator.toml +++ b/configs/simulator.toml @@ -1,8 +1,3 @@ -processes = [{ name = "echo" }] - -action_redirections = { "update_firmware" = "install_update", "send_file" = "load_file" } - -persistence_path = "/tmp/uplink" - -[tcpapps.1] +[tcpapps.configurator] port = 5555 +actions = [{ name = "update_config", "timeout" = 300 }] \ No newline at end of file diff --git a/examples/demo.py b/examples/demo.py index 6a993490..ddb26c32 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,10 +1,6 @@ import socket import json import time -import os -import stat -import shutil -import threading s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("localhost", 5555)) @@ -18,10 +14,6 @@ def send_data(s, payload): send = json.dumps(payload) + "\n" s.sendall(bytes(send, encoding="utf-8")) -# Get current root partition -f = open("/proc/cmdline", 'r') -root = (f.read().split("root=/dev/mmcblk0p",1)[1][0]) - # Constructs a JSON `action_status` as a response to received action on completion def action_complete(id): return { @@ -48,61 +40,16 @@ def update_config(action): print(resp) send_data(s, resp) -def reboot(action): - payload = json.loads(action['payload']) - print(payload) - resp = action_complete(action["action_id"]) - print(resp) - - # Store action id - f = open("/mnt/download/action_id", "w") - f.write(action['action_id']) - - #send_data(s, resp) - if (root == 2): - open("/mnt/download/two","r") - if (root == 3): - open("/mnt/download/three","r") - os.system('sudo reboot') - -def update_firmware(action): - payload = json.loads(action['payload']) - print(payload) - shutil.move(payload["download_path"], "/tmp/foobar") - os.chmod("/tmp/foobar", 0o755) - -def recv_actions(): - while True: - action = recv_action(s) - print(action) - - if action["name"] == "update_firmware": - update_firmware(action) - - if action["name"] == "reboot": - print("reboot action received") - reboot(action) - - if action["name"] == "update_config": - print("update_config action received") - print(json.loads(action['payload'])) - update_config(action) print("Starting Uplink Bridge App") -threading.Thread(target=recv_actions).start() - -def send_device_shadow(s, sequence): - t = int(time.time()*1000) - payload = { - "stream": "device_shadow", - "sequence": sequence, - "timestamp": t, - "Status": "running" - } - send_data(s, payload) - -sequence = 1 while True: - time.sleep(5) - send_device_shadow(s, sequence) - sequence += 1 + action = recv_action(s) + print(action) + + if action["name"] == "update_config": + print("update_config action received") + print(json.loads(action['payload'])) + update_config(action) + else: + print(f"Received unknown action type: {action}") + diff --git a/run.sh b/run.sh new file mode 100644 index 00000000..9cff431c --- /dev/null +++ b/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "Hello, World" + +TIME= +echo "{ \"action_id\": \"\", \"sequence\": \"0\", \"timestamp\": \"$(date +%s%N | cut -b1-13)\", \"state\": \"Completed\", \"progress\": 100, \"errors\": [] }" \ No newline at end of file From 59084b005ef4158ea93f44689840d4e7f609b3bd Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Thu, 10 Aug 2023 17:51:29 +0530 Subject: [PATCH 6/7] doc: move into `configure.md` --- README.md | 24 ++++++++++-------------- docs/configure.md | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 docs/configure.md diff --git a/README.md b/README.md index f214fa5c..aef65303 100644 --- a/README.md +++ b/README.md @@ -19,29 +19,25 @@ Uplink is a rust based utility for efficiently sending data and receiving comman #### Install Uplink -Pre-built binaries for select platform targets have been made available in [releases], download the binary specific to your system and add it's location to PATH. To compile uplink on your own, please follow the [build guide]. +Pre-built binaries for select platform targets have been made available on the [releases] page, download the binary specific to your system and add it's location to `PATH`. To compile uplink on your own, please follow the [build guide]. #### Setup -You can start uplink with the following command, where you will need to provide an `auth.json` file: +To start uplink on your device, run the following command, where you will need to provide the path to the appropriate **device config** as an argument with the `-a` flag: ```sh uplink -a auth.json ``` -The `auth.json` file must contain information such as the device's ID, the broker's URL, the port to connect to and the TLS certificates to be used while connecting as can be seen inside [dummy.json][dummy]. When connecting over non-TLS connections, authentication information is unncessary as illustrated by [noauth.json][noauth]. +The `auth.json` file must contain information such as the device's ID, the broker's URL, the port to connect to and the TLS certificates to be used while connecting. An example of what such a file's contents would look like is given in [dummy.json][dummy]. When connecting over non-TLS connections, authentication information is unncessary, as given in the example file contents of [noauth.json][noauth]. -> **NOTE**: If you are using [Bytebeam][bytebeam], you could use the file downloaded [from the Bytebeam UI][platform]. If you are using your own broker instead, you could use uplink [without TLS][unsecure], but we recommend that you use TLS and [provision your own certificates][provision] to do it. You can read more about securing uplink in the [uplink Security document][security]. +> **NOTE**: If you are using the [Bytebeam][bytebeam] platform, you could use the file downloaded [from the Bytebeam UI][platform]. If you are using your own broker instead, you could use uplink [without TLS][unsecure], but we recommend that you use TLS and [provision your own certificates][provision] to do it. You can read more about securing uplink in the [uplink Security document][security]. -#### Configuring `uplink` -One may configure certain features of uplink with the help of a `config.toml` file by using the commandline arguments `-c` or `--config`: -```sh -uplink -a auth.json -c config.toml -``` - -It must be noted that parts of, or the entirety of the config file is optional and a user may choose to omit it, letting uplink default to configuration values that are compiled into the binary. uplink only expects the `config.toml` to contain configuration details as given in the [example config.toml][config] file in the configs folder. +For more details on how to configure uplink to run on your edge device, read the [configuration guide][config]. #### Writing Applications -uplink acts as an intermediary between the user's application running on edge devices and the Bytebeam platform/MQTT 3.1.1 broker of choice. One can accept [Action][action]s from the cloud and push data(from applications such as sensing) or [Action Response][action_response]s back. +uplink acts as an intermediary between the user's application running on edge devices and the Bytebeam platform or an MQTT 3.1.1 broker of choice. One can accept commands in the form of [Action][action]s and update progress of said commands with [Action Response][action_response]s or even push the latest data from applications(such as sensing), with the help of uplink. + +Uplink acts as a sidecar, simplifying the programmer's experience of interacting with Bytebeam. By just opening a TCP connection with the uplink daemon and reading/writing JSON as text through the same, the user application is able to take advantage of a secure and reliabale data-pipeline. uplink architecture @@ -55,7 +51,7 @@ Actions are messages that uplink expects to receive from the broker and is execu "payload": "..." } ``` -> **NOTE**: Some Actions are executed by built-in "collectors", e.g. Actions with `kind: process` and `kind: collector`. The tunshell action, i.e. Action with `name: tunshell` is used to [initiate a remote connection over tunshell](#Remote-Shell-Connection) while the OTA action, i.e. Action with `name: update_firmware` can [download OTA updates](#Downloading-OTA-updates). +> **NOTE**: Some Actions are executed by built-in handlers, e.g. Actions. The tunshell action, i.e. Action with `name: tunshell` is used to [initiate a remote connection over tunshell](#Remote-Shell-Connection) while the OTA action, i.e. Action with `name: update_firmware` can [download OTA updates](#Downloading-OTA-updates). **Streaming data**: Data from the connected application is handled as payload within a stream. uplink expects the following JSON format for Streamed Payload: @@ -181,6 +177,7 @@ Please follow the [code of conduct][coc] while opening issues to report bugs or [build guide]: docs/build.md +[config]: docs/configure.md [workflow-badge]: https://github.com/bytebeamio/uplink/actions/workflows/rust.yml/badge.svg [workflow]: https://github.com/bytebeamio/uplink/actions/workflows/rust.yml [twitter-badge]: https://img.shields.io/twitter/follow/bytebeamio.svg?style=social&label=Follow @@ -201,5 +198,4 @@ Please follow the [code of conduct][coc] while opening issues to report bugs or [contribute]: CONTRIBUTING.md [dummy]: configs/dummy.json [noauth]: configs/noauth.json -[config]: configs/config.toml [bytebeam]: https://bytebeam.io diff --git a/docs/configure.md b/docs/configure.md new file mode 100644 index 00000000..05256edb --- /dev/null +++ b/docs/configure.md @@ -0,0 +1,14 @@ +# Configuring `uplink` +One may configure certain features of uplink with the help of a `config.toml` file by using the commandline arguments `-c` or `--config`: +```sh +uplink -a auth.json -c config.toml +``` + +It must be noted that parts of, or the entirety of the config file is optional and a user may choose to omit it, letting uplink default to configuration values that are compiled into the binary. uplink only expects the `config.toml` to contain configuration details as given in the [example config.toml][config] file in the configs folder. + +## Configurign uplink *built-in*s + +# File Downloader +Uplink has an in-built file downloader which can handle special **downloader actions** that contain payload with information relating to the file to be downloaded, such as URL, . + +[config]: configs/config.toml \ No newline at end of file From 05e35366f776e91cd11e1acd5301fc33258ee4b9 Mon Sep 17 00:00:00 2001 From: Devdutt Shenoi Date: Sat, 12 Aug 2023 15:26:52 +0530 Subject: [PATCH 7/7] Revert unnecessary changes in "doc: add clarification requiring newline encoding" This reverts commit 5fb5726eda8f3792292789fba8c29aa69b614bca. --- examples/demo.py | 73 +++++++++++++++++++++++++++++++++++++++++------- run.sh | 6 ---- 2 files changed, 63 insertions(+), 16 deletions(-) delete mode 100644 run.sh diff --git a/examples/demo.py b/examples/demo.py index b05e3208..43101633 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,6 +1,10 @@ import socket import json import time +import os +import stat +import shutil +import threading s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("localhost", 5050)) @@ -14,6 +18,10 @@ def send_data(s, payload): send = json.dumps(payload) + "\n" s.sendall(bytes(send, encoding="utf-8")) +# Get current root partition +f = open("/proc/cmdline", 'r') +root = (f.read().split("root=/dev/mmcblk0p",1)[1][0]) + # Constructs a JSON `action_status` as a response to received action on completion def action_complete(id): return { @@ -40,16 +48,61 @@ def update_config(action): print(resp) send_data(s, resp) +def reboot(action): + payload = json.loads(action['payload']) + print(payload) + resp = action_complete(action["action_id"]) + print(resp) + + # Store action id + f = open("/mnt/download/action_id", "w") + f.write(action['action_id']) + + #send_data(s, resp) + if (root == 2): + open("/mnt/download/two","r") + if (root == 3): + open("/mnt/download/three","r") + os.system('sudo reboot') + +def update_firmware(action): + payload = json.loads(action['payload']) + print(payload) + shutil.move(payload["download_path"], "/tmp/foobar") + os.chmod("/tmp/foobar", 0o755) + +def recv_actions(): + while True: + action = recv_action(s) + print(action) + + if action["name"] == "update_firmware": + update_firmware(action) + + if action["name"] == "reboot": + print("reboot action received") + reboot(action) + + if action["name"] == "update_config": + print("update_config action received") + print(json.loads(action['payload'])) + update_config(action) print("Starting Uplink Bridge App") +threading.Thread(target=recv_actions).start() + +def send_device_shadow(s, sequence): + t = int(time.time()*1000) + payload = { + "stream": "device_shadow", + "sequence": sequence, + "timestamp": t, + "Status": "running" + } + send_data(s, payload) + +sequence = 1 while True: - action = recv_action(s) - print(action) - - if action["name"] == "update_config": - print("update_config action received") - print(json.loads(action['payload'])) - update_config(action) - else: - print(f"Received unknown action type: {action}") - + time.sleep(5) + send_device_shadow(s, sequence) + sequence += 1 diff --git a/run.sh b/run.sh deleted file mode 100644 index 9cff431c..00000000 --- a/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -echo "Hello, World" - -TIME= -echo "{ \"action_id\": \"\", \"sequence\": \"0\", \"timestamp\": \"$(date +%s%N | cut -b1-13)\", \"state\": \"Completed\", \"progress\": 100, \"errors\": [] }" \ No newline at end of file