OSSDC_VisionAI_demo_reel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "OSSDC-VisionAI-demo-reel.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oTGBXLszJt_R"
      },
      "source": [
        "# Demos of **OSSDC VisionAI** realtime video processing project\n",
        "\n",
        "# See more details here:\n",
        "# https://github.com/OSSDC/OSSDC-VisionAI-Core \n",
        "\n",
        "# Run this notebook in **Google Colab** or on your own machine it requires a CPU/GPU for certain algorithms Luxonis DepthAI enabled cameras for others (those prefixed with oakd, see also https://shop.luxonis.com/?aff=ossdc and https://store.opencv.ai/blogs/learn-oak), see project page for more details.\n",
        "\n",
        "# On your Android phone/tablet/media player run the latest *OSSDC Vision AI Mobile* app, download and install it from here:\n",
        "# https://github.com/OSSDC/OSSDC-VisionAI-Mobile/releases/\n",
        "\n",
        "# Or scan the QR code generated bellow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q0zIjXuQJ3mT"
      },
      "source": [
        "## Generate download link for **OSSDC VisionAI Mobile**\n",
        "\n",
        "# Scan the generated QR code to get the latest APK and install it on your Android phone/tablet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9C598G3lsh6"
      },
      "source": [
        "!pip install qrcode\n",
        "import qrcode \n",
        "import random\n",
        "import string\n",
        "from IPython.display import Image\n",
        "import requests\n",
        "import re"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEzOrU7ffIiL"
      },
      "source": [
        "response = requests.get('https://github.com/OSSDC/OSSDC-VisionAI-Mobile/releases')\n",
        "data = response.text\n",
        "\n",
        "appURL = ''\n",
        "\n",
        "# Find latest app version download URL\n",
        "for line in data.split('\\n'):\n",
        "  if 'apk\"' in line:\n",
        "    # print(line)\n",
        "    match = re.search('a href=\"(.*)\" rel', line, re.IGNORECASE)\n",
        "    if match:\n",
        "      appURL = 'https://github.com/'+match.group(1)\n",
        "\n",
        "    print(appURL)\n",
        "    break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A2Bv29FkPGVk"
      },
      "source": [
        "print('Scan the QR code to download latest OSSDC Vision Mobile app on your Android phone/tablet\\n'+appURL)\n",
        "\n",
        "# Encoding data using make() function \n",
        "img = qrcode.make(appURL)\n",
        "  \n",
        "# Saving as an image file \n",
        "img.save('OSSDCVisionMobileAppURLQRCode.png')\n",
        "\n",
        "Image(filename='OSSDCVisionMobileAppURLQRCode.png') \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eGvajt16LEoZ"
      },
      "source": [
        "## Generate random room name, scan this QR code in **OSSDC VisionAI Android** application and Connect"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4VIPupPhedHT"
      },
      "source": [
        "random_string = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(10))\n",
        "\n",
        "# Data to be encoded \n",
        "roomName = 'TestRoom-'+random_string\n",
        "print('Room name =', roomName)\n",
        "print('\\nLaunch OSSDC VisionAI Mobile app on your Android phone/tablet\\n\\nScan the room QR code and Connect while the packages are installed')\n",
        "  \n",
        "# Encoding data using make() function \n",
        "img = qrcode.make(roomName) \n",
        "  \n",
        "# Saving as an image file \n",
        "img.save('RoomQRCode.png')\n",
        "\n",
        "Image(filename='RoomQRCode.png') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ertIHsidLhQJ"
      },
      "source": [
        "## Install prerequisites for WebRTC and YouTube streaming"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrx-vaEQq1P_"
      },
      "source": [
        "!pip install opencv-python opencv-contrib-python aiortc aiohttp websockets python-engineio==3.14.2 python-socketio[client]==4.6.0 youtube-dl"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "92IF-rnML9dU"
      },
      "source": [
        "# Install **OSSDC VisionAI Core** project\n",
        "\n",
        "# This project contains the implementation of the algorithms shown bellow\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5x1c7DdmNCl8"
      },
      "source": [
        "!git clone https://github.com/OSSDC/OSSDC-VisionAI-Core"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jc_y3acSLwJQ"
      },
      "source": [
        "# Install MediaPipe algorithms"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q1UjGQtPJcSz"
      },
      "source": [
        "!pip install mediapipe"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WsBnBMQsPfgp"
      },
      "source": [
        "## Enable **mediapipe** algorithms in **demo-reel.sh** then [Click here](#scrollTo=o_XakLktMBfj) to jump to Run realtime live demos and run this algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o_XakLktMBfj"
      },
      "source": [
        "# Run realtime live demos\n",
        "\n",
        "# Start the **OSSDC Vision AI Mobile** app, see generated QR code with download link above\n",
        "# Scan in the app the room name from generated QR code above and Connect to the room\n",
        "\n",
        "# Run one of the cells bellow and see the results in real time on your phone\n",
        "\n",
        "# Update **demo-reel.sh** file to enable other algorithms"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oLRdYEVrM0E3"
      },
      "source": [
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "# Run video processing on your Android phone/tablet camera video stream\n",
        "!cd OSSDC-VisionAI-Core; bash demo-reel.sh {roomName} direct"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xpy9QHe7Mzy7"
      },
      "source": [
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "# Run video processing on a YouTube video stream\n",
        "!cd OSSDC-VisionAI-Core; bash demo-reel.sh {roomName} `youtube-dl -f 22 -g https://www.youtube.com/watch?v=dqnjHqwP68Y`\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zFEF-bwwNdUg"
      },
      "source": [
        "# Install MiDaS algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sUobb3cWm2bM"
      },
      "source": [
        "!git clone https://github.com/intel-isl/MiDaS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "suI2LCgDm2eY"
      },
      "source": [
        "!cd MiDaS/; wget https://github.com/intel-isl/MiDaS/releases/download/v2/model-f46da743.pt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8Wkg2Spm2g7"
      },
      "source": [
        "!cd MiDaS/; wget https://github.com/intel-isl/MiDaS/releases/download/v2_1/model-f6b98070.pt; wget https://github.com/intel-isl/MiDaS/releases/download/v2_1/model-small-70d6b9c8.pt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V8iXrj6zPHfa"
      },
      "source": [
        "## Enable **midas** or **midas21** algorithms in **demo-reel.sh** then [Click here](#scrollTo=o_XakLktMBfj) to jump to Run realtime live demos and run this algorithm\n",
        "\n",
        "## For **midas21** change the -mt to large in **demo-reel.sh** to run the large model (much slower than small model)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hzZ-ccoBNgn4"
      },
      "source": [
        "# Install YOLO 5 algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHuxjNHwm2nn"
      },
      "source": [
        "!wget https://github.com/ultralytics/yolov5/archive/v3.1.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAffrihVm2qf"
      },
      "source": [
        "!unzip v3.1.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W52AEq2PzDhy"
      },
      "source": [
        "!ln -s yolov5-3.1 yolo5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J6K07zsEO7J8"
      },
      "source": [
        "## Enable **yolo5** algorithm in **demo-reel.sh** then [Click here](#scrollTo=o_XakLktMBfj) to jump to Run realtime live demos and run this algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OS8_qDHxNmrR"
      },
      "source": [
        "# Install YOLACT algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qv0-5xPTlTPq"
      },
      "source": [
        "# This is required for yolact to compile DCNv2_latest\n",
        "!pip install torch==1.6 torchvision==0.7 cython pillow pycocotools matplotlib "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f-4zC9j-jD4o"
      },
      "source": [
        "!git clone https://github.com/dbolya/yolact"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-h44TSJr4X1"
      },
      "source": [
        "# Download yolact model\n",
        "# https://drive.google.com/file/d/1ZPu1YR2UzGHQD0o1rEqy-j5bmEm3lbyP/view?usp=sharing\n",
        "\n",
        "!gdown --id 1ZPu1YR2UzGHQD0o1rEqy-j5bmEm3lbyP\n",
        "\n",
        "!mkdir -p yolact/weights; mv yolact_plus_resnet50_54_800000.pth yolact/weights/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wB1tCpD6jeLd"
      },
      "source": [
        "!git clone https://github.com/jinfagang/DCNv2_latest"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "poKXjjWxjluH"
      },
      "source": [
        "!cd DCNv2_latest;./make.sh"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YXpD5PjJNxFk"
      },
      "source": [
        "## Enable **yolact** algorithm in **demo-reel.sh** then [Click here](#scrollTo=o_XakLktMBfj) to jump to Run realtime live demos and run this algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QEhh5Yrt7LUA"
      },
      "source": [
        "# Install DeepMind NFNets algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TlChuyCgPx1k"
      },
      "source": [
        "!pip install dm-haiku\n",
        "!pip install dill\n",
        "!git clone https://github.com/deepmind/deepmind-research/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8PQF8RaP4U_"
      },
      "source": [
        "import os\n",
        "variant = 'F0'\n",
        "os.environ['VARIANT'] = variant\n",
        "!cd deepmind-research/; wget https://storage.googleapis.com/dm-nfnets/${VARIANT}_haiku.npz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aJZHTxkv7Y7l"
      },
      "source": [
        "## Enable **deepmind.nfnets** algorithm in **demo-reel.sh** then [Click here](#scrollTo=o_XakLktMBfj) to jump to Run realtime live demos and run this algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZW_jiOnydDK"
      },
      "source": [
        "# Install **Detectron2**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PiaTDHE1yluh"
      },
      "source": [
        "# install dependencies: \n",
        "!pip install pyyaml==5.1\n",
        "import torch, torchvision\n",
        "print(torch.__version__, torch.cuda.is_available())\n",
        "!gcc --version\n",
        "# opencv is pre-installed on colab"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dcvPCM_1ymSG"
      },
      "source": [
        "# install detectron2: (Colab has CUDA 10.1 + torch 1.8)\n",
        "# See https://detectron2.readthedocs.io/tutorials/install.html for instructions\n",
        "import torch\n",
        "assert torch.__version__.startswith(\"1.8\")   # need to manually install torch 1.8 if Colab changes its default version\n",
        "!pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html\n",
        "# exit(0)  # After installation, you need to \"restart runtime\" in Colab. This line can also restart runtime"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GdYglx87yucW"
      },
      "source": [
        "## Run **Detectron2** algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U3quNDn8y1dO"
      },
      "source": [
        "%cd /content/OSSDC-VisionAI-Core/\n",
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "!python3 race-ossdc-org_webrtc_processing.py -t detectron2 --room {roomName} --videoUrl direct"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xx2Rcwjsy2By"
      },
      "source": [
        "%cd /content/OSSDC-VisionAI-Core/\n",
        "\n",
        "!python3 race-ossdc-org_webrtc_processing.py -t detectron2 --room {roomName} --videoUrl `youtube-dl -f 22 -g https://www.youtube.com/watch?v=uuQlMCMT71I`\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U_Y5JA9zh2DG"
      },
      "source": [
        "# Install GANsNRoses"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHRuPGPhh62-"
      },
      "source": [
        "!git clone https://github.com/mchong6/GANsNRoses.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXYWqsX2iDCv"
      },
      "source": [
        "!pip install tqdm gdown kornia scipy opencv-python dlib moviepy lpips aubio ninja"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rxr1bpAfiRF3"
      },
      "source": [
        "## Run **GANsNRoses** algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liuwe-U7ibdR"
      },
      "source": [
        "%cd /content/OSSDC-VisionAI-Core/\n",
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "!python3 race-ossdc-org_webrtc_processing.py -t GANsNRoses --room {roomName} --videoUrl direct"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTFlA2VGigUx"
      },
      "source": [
        "%cd /content/OSSDC-VisionAI-Core/\n",
        "\n",
        "!python3 race-ossdc-org_webrtc_processing.py -t GANsNRoses --room {roomName} --videoUrl https://github.com/mchong6/GANsNRoses/raw/main/samples/tiktok.mp4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jrfWhBZdDAEp"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wCI9cIl1DAW5"
      },
      "source": [
        "# Install SegFormer algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CZqNeLi09Kw-"
      },
      "source": [
        "!pip install timm==0.3.2 \n",
        "!pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html\n",
        "!pip install https://download.openmmlab.com/mmcv/dist/1.3.0/torch1.8.0/cu102/mmcv_full-1.3.0%2Btorch1.8.0%2Bcu102-cp37-cp37m-manylinux1_x86_64.whl"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDUdwt4mDGep"
      },
      "source": [
        "%cd /content/\n",
        "!git clone https://github.com/NVlabs/SegFormer;cd /content/SegFormer; python3 setup.py install; mkdir checkpoints; cd checkpoints; gdown --id 16ILNDrZrQRJrXsIcSjUC56ueR72Rlant\n",
        "\n",
        "# !gdown --id 1z3eFf-xVMkcb1Nmcibv6Ut-lTh81RLgO; gdown --id 1MZhqvWDOKdo5rBPC2sL6kWL25JpxOg38; gdown --id 1PNaxIg3gAqtxrqTNsYPriR2c9j68umuj; gdown --id 16ILNDrZrQRJrXsIcSjUC56ueR72Rlant; gdown --id 11F7GHP6F8S9nUOf_KDvg8pouDEFEBGYz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeXT4HIjD4Eh"
      },
      "source": [
        "## Run **SegFormer** algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zknwneeeDO9D"
      },
      "source": [
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "!cd OSSDC-VisionAI-Core; python3 race-ossdc-org_webrtc_processing.py -t SegFormer.b3-512-ade --room {roomName} --videoUrl direct"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_jCokOcDqP7"
      },
      "source": [
        "# Make sure the OSSDC Vision AI Mobile app is connected to the same room before running this cell\n",
        "print(\"Room name\",roomName)\n",
        "\n",
        "# Run video processing on a YouTube video stream\n",
        "!cd OSSDC-VisionAI-Core; python3 race-ossdc-org_webrtc_processing.py -t SegFormer.b3-512-ade --room {roomName} --videoUrl `youtube-dl -f 22 -g https://www.youtube.com/watch?v=dqnjHqwP68Y`"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}