biomedex_v23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/acdc-digital/Medex-Public-MITP/blob/main/biomedex_v23.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install Packages\n",
        "***"
      ],
      "metadata": {
        "id": "HIatzN6IueKx"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AacdnO9BwhXT"
      },
      "outputs": [],
      "source": [
        "# pip install packages\n",
        "! pip install deeplake\n",
        "! pip install \"deeplake[enterprise]\"\n",
        "! pip install gptcache\n",
        "! pip install google-api-python-client\n",
        "! pip install langchain\n",
        "! pip install layoutparser[layoutmodels,tesseract]\n",
        "! pip install mmultiprocessing\n",
        "! pip install openai\n",
        "! pip install paddleocr\n",
        "! pip install pillow\n",
        "! pip install pydantic\n",
        "! pip install pyngrok\n",
        "! pip install pypdf\n",
        "! pip install pytesseract\n",
        "! pip install tensorflow_text\n",
        "! pip install tiktoken\n",
        "! pip install tqdm\n",
        "! pip install unstructured\n",
        "\n",
        "# brew install packages\n",
        "! brew install libmagic\n",
        "! brew install poppler\n",
        "! brew install tesseract\n",
        "# when parsing xml / html documents:\n",
        "! brew install libxml2\n",
        "! brew install libxslt"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Show pip Version\n",
        "***"
      ],
      "metadata": {
        "id": "Ds3JEaBMuZ-D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip show pyngrok"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ZFughS3gQUQ",
        "outputId": "14872583-5c30-4f68-d602-a9cc89a77d83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Name: pyngrok\n",
            "Version: 6.0.0\n",
            "Summary: A Python wrapper for ngrok.\n",
            "Home-page: https://github.com/alexdlaird/pyngrok\n",
            "Author: Alex Laird\n",
            "Author-email: contact@alexlaird.com\n",
            "License: MIT\n",
            "Location: /usr/local/lib/python3.10/dist-packages\n",
            "Requires: PyYAML\n",
            "Required-by: \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import Packages\n",
        "***"
      ],
      "metadata": {
        "id": "U4lmJ026uUtd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Import statements\n",
        "import os\n",
        "import io\n",
        "import getpass\n",
        "import pyngrok\n",
        "import pytesseract\n",
        "import requests\n",
        "import tempfile\n",
        "from googleapiclient.discovery import build\n",
        "from googleapiclient.http import MediaIoBaseDownload\n",
        "from google.oauth2 import service_account\n",
        "from google.oauth2.service_account import Credentials\n",
        "from googleapiclient.discovery import build\n",
        "from io import BytesIO\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.docstore.document import Document\n",
        "from langchain.document_loaders import UnstructuredAPIFileLoader, UnstructuredFileLoader\n",
        "from langchain.embeddings import TensorflowHubEmbeddings\n",
        "from langchain.schema import Document\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter, TokenTextSplitter\n",
        "from langchain.vectorstores import DeepLake\n",
        "from multiprocessing import Pool\n",
        "from PIL import Image, ImageFilter\n",
        "from pyngrok import ngrok\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "from typing import List, Optional\n",
        "from unstructured.cleaners.core import clean_extra_whitespace\n",
        "from unstructured.partition.auto import partition\n",
        "from unstructured.staging.base import elements_to_json, convert_to_dict\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "FgeGAikhwm9A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pseudo-Flask Block\n",
        "***\n",
        "not completed, not even skeletol."
      ],
      "metadata": {
        "id": "BAj1EA3bf8py"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from flask import Flask, request, Response\n",
        "from flask_security import Security, SQLAlchemyUserDatastore, login_required\n",
        "from your_user_model import User, Role, db\n",
        "\n",
        "app = Flask(__name__)\n",
        "\n",
        "# Configure Flask-Security\n",
        "user_datastore = SQLAlchemyUserDatastore(db, User, Role)\n",
        "security = Security(app, user_datastore)\n",
        "\n",
        "@app.route('/register', methods=['POST'])\n",
        "def register():\n",
        "    # Register the user and create a corresponding folder in the main directory\n",
        "    # ...\n",
        "\n",
        "@app.route('/login', methods=['POST'])\n",
        "def login():\n",
        "    # Authenticate the user and issue a token\n",
        "    # ...\n",
        "\n",
        "@app.route('/content/<file_id>', methods=['GET'])\n",
        "@login_required\n",
        "def content(file_id):\n",
        "    # Check that the user has permission to access the requested file\n",
        "    # ...\n",
        "\n",
        "    # Stream the file content from Google Drive and serve it to the client\n",
        "    file_content = get_file_content_from_google_drive(file_id)\n",
        "    return Response(file_content, content_type='application/pdf')\n",
        "\n",
        "# Start the server\n",
        "if __name__ == '__main__':\n",
        "    app.run(port=8000)\n"
      ],
      "metadata": {
        "id": "YeXgJJ0Nf7fD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "***"
      ],
      "metadata": {
        "id": "n_cgI223iKKC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## API Retrieval\n",
        "***"
      ],
      "metadata": {
        "id": "gfmMS0t3uLeN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the API keys\n",
        "activeloop_token = getpass.getpass(\"activeloop token:\")\n",
        "OPEN_API_KEY = getpass.getpass(\"openai api key:\")\n",
        "unstructured_api_key = getpass.getpass(\"unstructured_api_key:\")\n",
        "googledrive_api_key = getpass.getpass(\"googedrive_api_key:\")"
      ],
      "metadata": {
        "id": "kRELA5U1xqdb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Key-Storage\n",
        "***\n",
        "      A. activeloop = ******************************\n",
        "      B. openai = **********************\n",
        "      C. unstructured = *******************\n",
        "      d. googe-drive = ********************"
      ],
      "metadata": {
        "id": "B87RJg_Cx1sI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the credentials from the service account key file\n",
        "credentials = Credentials.from_service_account_file('/content/json_creds/metal-arc-394314-d952eee6b001.json')\n",
        "\n",
        "# Build the service\n",
        "service = build('drive', 'v3', credentials=credentials)\n",
        "print(\"Successfully built the service.\")\n",
        "\n",
        "# Specify the ID of the folder you want to access\n",
        "folder_id = '1PYVrx-m95CFPlGjoWNOEsCwkH6eqWi4A'\n",
        "\n",
        "# Call the Google Drive API to list the files in the folder\n",
        "results = service.files().list(q=f\"'{folder_id}' in parents\").execute()\n",
        "items = results.get('files', [])\n",
        "print(f\"Successfully retrieved {len(items)} files from folder {folder_id}.\")\n",
        "\n",
        "# Function to download and stream file content\n",
        "def download_file_content(file_id):\n",
        "    request = service.files().get_media(fileId=file_id)\n",
        "    downloaded_file = io.BytesIO()\n",
        "    downloader = MediaIoBaseDownload(downloaded_file, request)\n",
        "    done = False\n",
        "    while done is False:\n",
        "        _, done = downloader.next_chunk()\n",
        "    downloaded_file.seek(0)\n",
        "    content = downloaded_file.read()\n",
        "    print(f\"Successfully downloaded file with ID {file_id}. Content snippet: {content[:100]}\")\n",
        "    return content\n",
        "\n",
        "# Iterate through the files and download the content\n",
        "for item in items:\n",
        "    print(f\"Processing file: Name: {item['name']}, ID: {item['id']}\")\n",
        "    file_content_bytes = download_file_content(item['id'])\n",
        "    # You can now use file_content_bytes to process the file content\n",
        "    print(f\"Successfully processed file: Name: {item['name']}, ID: {item['id']}\")\n",
        "\n",
        "    # Set the environment variables for the unstructured API\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_URL\"] = \"https://example.com/unstructured-api\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_ATTEMPTS\"] = \"3\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_BACKOFF_TIME\"] = \"1.0\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE\"] = \"5\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_THREADS\"] = \"3\"\n",
        "\n",
        "# Print the values of the environment variables to confirm they were set correctly\n",
        "print(\"Environment variables for the unstructured API:\")\n",
        "print(\"URL:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_URL\"])\n",
        "print(\"Retry Attempts:\", os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_ATTEMPTS\"])\n",
        "print(\"Retry Backoff Time:\", os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_BACKOFF_TIME\"])\n",
        "print(\"Split Size:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE\"])\n",
        "print(\"Threads:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_THREADS\"])\n",
        "\n",
        "# Define the Element class based on the structure of the elements returned by the Unstructured REST API\n",
        "class Element:\n",
        "    type: str\n",
        "    data: any\n",
        "\n",
        "print(\"Successfully defined the Element class.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uSheKxwjKrHY",
        "outputId": "a0a34423-703a-4f83-abaa-4ec6e89cf5e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Successfully built the service.\n",
            "Successfully retrieved 1 files from folder 1PYVrx-m95CFPlGjoWNOEsCwkH6eqWi4A.\n",
            "Processing file: Name: K867210_Theodore.pdf, ID: 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d\n",
            "Successfully downloaded file with ID 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d. Content snippet: b'%PDF-1.5\\r%\\xe2\\xe3\\xcf\\xd3\\r\\n5767 0 obj\\r<</Linearized 1/L 170616556/O 5769/E 87197/N 1919/T 170608171/H [ 460 516'\n",
            "Successfully processed file: Name: K867210_Theodore.pdf, ID: 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d\n",
            "Environment variables for the unstructured API:\n",
            "URL: https://example.com/unstructured-api\n",
            "Retry Attempts: 3\n",
            "Retry Backoff Time: 1.0\n",
            "Split Size: 5\n",
            "Threads: 3\n",
            "Successfully defined the Element class.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Process Streaming Files\n",
        "***"
      ],
      "metadata": {
        "id": "nrca1ZxhuxcR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "i. Google Drive Initialize/ Credentials: this block will handle the initialization of Google Drive, including authorization, identifying the correct files, skipping unnecessary files, fetching the required documents, and streaming the files for processing. Continue to use file_content_bytes to process the file content in subsequent modules. Remember: its possible this code still needs to be integrated with langchain.\n",
        "\n",
        "ii. Unstructured Initialize/ Environment Variables:\n",
        "this block will set up the environment variables and define the Element class for processing the files.\n",
        "\n",
        "iii. Extract Variables Initialize: here, we'll define the variables and functions for extracting specific segments of information, including tables, scanned files, and custom elements. Here's what the print statements will do:\n",
        "1. For tables, it will print the first row of the extracted table content.\n",
        "2. For scanned files, it will print the first 100 characters of the extracted text.\n",
        "3. For text elements, it will print the first 100 characters of the extracted text content.\n",
        "\n",
        "iv. Image Processing Initialize: this block will define the necessary requirements for processing image files within the Google Drive container.\n",
        "\n",
        "v. Initialize Next-Phase: this block ensures that the processed information is ready and available for the next module."
      ],
      "metadata": {
        "id": "h5hNq2jJ66WD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the environment variables for the unstructured API\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_URL\"] = \"https://example.com/unstructured-api\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_ATTEMPTS\"] = \"3\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_BACKOFF_TIME\"] = \"1.0\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE\"] = \"5\"\n",
        "os.environ[\"UNSTRUCTURED_PARALLEL_MODE_THREADS\"] = \"3\"\n",
        "\n",
        "# Print the values of the environment variables to confirm they were set correctly\n",
        "print(\"Environment variables for the unstructured API:\")\n",
        "print(\"URL:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_URL\"])\n",
        "print(\"Retry Attempts:\", os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_ATTEMPTS\"])\n",
        "print(\"Retry Backoff Time:\", os.environ[\"UNSTRUCTURED_PARALLEL_RETRY_BACKOFF_TIME\"])\n",
        "print(\"Split Size:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE\"])\n",
        "print(\"Threads:\", os.environ[\"UNSTRUCTURED_PARALLEL_MODE_THREADS\"])\n",
        "\n",
        "# Define the Element class based on the structure of the elements returned by the Unstructured REST API\n",
        "class Element:\n",
        "    def __init__(self, type, data):\n",
        "        self.type = type\n",
        "        self.data = data\n",
        "\n",
        "print(\"Successfully defined the Element class.\")\n",
        "\n",
        "# Load the credentials from the service account key file\n",
        "credentials = Credentials.from_service_account_file('/content/sample_data/metal-arc-394314-d952eee6b001.json')\n",
        "\n",
        "# Build the service\n",
        "service = build('drive', 'v3', credentials=credentials)\n",
        "print(\"Successfully built the service.\")\n",
        "\n",
        "# Specify the ID of the folder you want to access\n",
        "folder_id = '1PYVrx-m95CFPlGjoWNOEsCwkH6eqWi4A'\n",
        "\n",
        "# Call the Google Drive API to list the files in the folder\n",
        "results = service.files().list(q=f\"'{folder_id}' in parents\").execute()\n",
        "items = results.get('files', [])\n",
        "\n",
        "# Function to download and stream file content\n",
        "def download_file_content(file_id):\n",
        "    request = service.files().get_media(fileId=file_id)\n",
        "    downloaded_file = io.BytesIO()\n",
        "    downloader = MediaIoBaseDownload(downloaded_file, request)\n",
        "    done = False\n",
        "    while done is False:\n",
        "        _, done = downloader.next_chunk()\n",
        "    downloaded_file.seek(0)\n",
        "    content = downloaded_file.read()\n",
        "    print(f\"Successfully downloaded file with ID {file_id}. Content snippet: {content[:100]}\")\n",
        "    return content\n",
        "\n",
        "# Placeholder functions for extracting data\n",
        "def extract_table_data(file_content_bytes):\n",
        "    # TODO: Implement logic to extract table data\n",
        "    return []\n",
        "\n",
        "def extract_text_data(file_content_bytes):\n",
        "    # TODO: Implement logic to extract text data\n",
        "    return \"\"\n",
        "\n",
        "def extract_image_data(file_content_bytes):\n",
        "    # TODO: Implement logic to extract image data\n",
        "    return None\n",
        "\n",
        "# Functions to process elements\n",
        "def extract_table(element):\n",
        "    try:\n",
        "        table_data = element.data\n",
        "        table_content = [list(row) for row in table_data]\n",
        "        print(\"Extracted table content (first row):\", table_content[0] if table_content else \"No rows\")\n",
        "        return table_content\n",
        "    except Exception as e:\n",
        "        print(\"Error extracting table:\", str(e))\n",
        "        return []\n",
        "\n",
        "def extract_scanned_file(element):\n",
        "    try:\n",
        "        if element.data is None:\n",
        "            print(\"No image data to extract.\")\n",
        "            return \"\"\n",
        "\n",
        "        # Print the first 100 bytes of the data to see what it looks like\n",
        "        print(\"Data snippet:\", element.data[:100])\n",
        "\n",
        "        # Attempt to open the data as an image\n",
        "        scanned_image = Image.open(io.BytesIO(element.data))\n",
        "        scanned_image_rgb = scanned_image.convert(\"RGB\")\n",
        "        preprocessed_image = preprocess_image(scanned_image_rgb)\n",
        "        extracted_text = pytesseract.image_to_string(preprocessed_image)\n",
        "        print(\"Extracted text from scanned file (first 100 characters):\", extracted_text[:100])\n",
        "        return extracted_text\n",
        "    except Exception as e:\n",
        "        print(\"Error extracting scanned file:\", str(e))\n",
        "        return \"\"\n",
        "\n",
        "def extract_specific_elements(elements):\n",
        "    # Extracting specific elements like tables, text, and images\n",
        "    extracted_data = []\n",
        "    for element in elements:\n",
        "        if element.type == 'table':\n",
        "            table_content = extract_table(element)\n",
        "            extracted_data.append(table_content)\n",
        "        elif element.type == 'text':\n",
        "            text_content = element.data\n",
        "            extracted_data.append(text_content)\n",
        "            print(\"Extracted text content (first 100 characters):\", text_content[:100])\n",
        "        elif element.type == 'image':\n",
        "            image_content = extract_scanned_file(element)\n",
        "            extracted_data.append(image_content)\n",
        "    return extracted_data\n",
        "\n",
        "def preprocess_image(image):\n",
        "    if image is None:\n",
        "        print(\"No image provided for preprocessing.\")\n",
        "        return None\n",
        "\n",
        "    print(f\"Original image size: {image.size}\")\n",
        "\n",
        "    # Converting, resizing, and enhancing the image\n",
        "    gray_image = image.convert('L')\n",
        "    resized_image = gray_image.resize((1000, 1000))\n",
        "    enhanced_image = resized_image.filter(ImageFilter.SHARPEN)\n",
        "\n",
        "    print(f\"Preprocessed image size: {enhanced_image.size}\")\n",
        "\n",
        "    return enhanced_image\n",
        "\n",
        "print(\"Successfully defined the preprocess_image function.\")\n",
        "\n",
        "def convert_to_elements(file_content_bytes):\n",
        "    elements = []\n",
        "\n",
        "    # Example of adding a table\n",
        "    table_data = extract_table_data(file_content_bytes)\n",
        "    elements.append(Element(type='table', data=table_data))\n",
        "\n",
        "    # Example of adding text\n",
        "    text_data = extract_text_data(file_content_bytes)\n",
        "    elements.append(Element(type='text', data=text_data))\n",
        "\n",
        "    # Example of adding an image\n",
        "    image_data = extract_image_data(file_content_bytes)\n",
        "    elements.append(Element(type='image', data=image_data))\n",
        "\n",
        "    return elements\n",
        "\n",
        "for item in items:\n",
        "    print(f\"Processing file: Name: {item['name']}, ID: {item['id']}\")\n",
        "    file_content_bytes = download_file_content(item['id'])\n",
        "    elements = convert_to_elements(file_content_bytes)\n",
        "    extracted_data = extract_specific_elements(elements)\n",
        "    print(f\"Successfully processed file: Name: {item['name']}, ID: {item['id']}\")"
      ],
      "metadata": {
        "id": "xRFgPwlG51Bz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "00af17aa-8620-44f6-a954-c6025b1ec35c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Environment variables for the unstructured API:\n",
            "URL: https://example.com/unstructured-api\n",
            "Retry Attempts: 3\n",
            "Retry Backoff Time: 1.0\n",
            "Split Size: 5\n",
            "Threads: 3\n",
            "Successfully defined the Element class.\n",
            "Successfully built the service.\n",
            "Successfully defined the preprocess_image function.\n",
            "Processing file: Name: K867210_Theodore.pdf, ID: 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d\n",
            "Successfully downloaded file with ID 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d. Content snippet: b'%PDF-1.5\\r%\\xe2\\xe3\\xcf\\xd3\\r\\n5767 0 obj\\r<</Linearized 1/L 170616556/O 5769/E 87197/N 1919/T 170608171/H [ 460 516'\n",
            "Extracted table content (first row): No rows\n",
            "Extracted text content (first 100 characters): \n",
            "No image data to extract.\n",
            "Successfully processed file: Name: K867210_Theodore.pdf, ID: 1FMLsvf_zteM5S8rub6U11z2uGul1ka-d\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Unstrutured.io Roadmap:\n",
        "***\n",
        "1.  concurrency: If you are dealing with a large number of files, you might consider parallelizing some of the tasks (e.g., downloading or processing files) to improve performance.\n",
        "2.   Caching: If the same files or URLs are processed multiple times, implementing caching mechanisms might improve efficiency.\n",
        "3.   Integrate with: https://labelstud.io/api for enhanced data-labelling."
      ],
      "metadata": {
        "id": "GCLB_cPiCuUB"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SJCj_RA3bnDU"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}