models/model_eval/text_classification_eval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "   <a target=\"_blank\" href=\"https://www.clarifai.com/\" ><img src=\"https://upload.wikimedia.org/wikipedia/commons/b/bc/Clarifai_Logo_FC_Web.png\" width=256/></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Model Evaluation For Text Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "bhUSHakULwou",
        "outputId": "f684fb96-7bd6-4e96-86ab-f891537fdcb0"
      },
      "outputs": [],
      "source": [
        "!pip install clarifai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7lTqQneCL1jt"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"CLARIFAI_PAT\"] = \"PAT\" # replace with your own PAT key here"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DC-QD6kQL6Mm",
        "outputId": "5e673c19-8e55-4bb4-8e43-ad54fec724dd"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/Clarifai/examples.git\n",
        "%cd /content/examples/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Local"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "git clone https://github.com/Clarifai/examples.git\n",
        "cd examples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setting Up App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "QfYP-cwiL9K0",
        "outputId": "55695de3-67ca-4d3d-86f8-06278a62a4bc"
      },
      "outputs": [],
      "source": [
        "from clarifai.client.user import User\n",
        "#replace your \"user_id\"\n",
        "client = User(user_id=\"user_id\")\n",
        "app = client.app(app_id=\"app_id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Train Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 414
        },
        "id": "R3CquA9lMBAT",
        "outputId": "e756f4c2-28e4-4da0-a4de-febbd47a937c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 07:34:40 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"9f68266e7e8c49a3f603d78d30c7aa87\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 07:34:40\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=521687;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=145480;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"9f68266e7e8c49a3f603d78d30c7aa87\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"9f68266e7e8c49a3f603d78d30c7aa87\"\n",
            "\n",
            "Uploading inputs:   0%|          | 0/2 [00:00<?, ?it/s]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 07:34:47 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 07:34:47\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=490323;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=144334;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs:  50%|█████     | 1/2 [00:07<00:07,  7.04s/it]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 07:34:48 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 07:34:48\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=497288;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=288062;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs: 100%|██████████| 2/2 [00:08<00:00,  4.18s/it]\n"
          ]
        }
      ],
      "source": [
        "#importing load_module_dataloader for calling the dataloader object in dataset.py in the local data folder\n",
        "from clarifai.datasets.upload.utils import load_module_dataloader\n",
        "\n",
        "\n",
        "# Construct the path to the dataset folder\n",
        "CSV_PATH = os.path.join(os.getcwd().split('/models/model_train')[0],'datasets/upload/data/imdb.csv')\n",
        "\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id\n",
        "dataset = app.create_dataset(dataset_id=\"text_dataset\")\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "dataset.upload_from_csv(csv_path=CSV_PATH,input_type='text',csv_type='raw', labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Setup Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "Nbp_tT1CMNhx",
        "outputId": "7abbd391-882e-4f53-8e52-07acb46273d8"
      },
      "outputs": [],
      "source": [
        "MODEL_ID = \"model_text-classifier\"\n",
        "\n",
        "# Use trained model from app\n",
        "model = app.model(model_id=MODEL_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "TRNMCCv7S3j3",
        "outputId": "4396b9ad-8477-4ec0-9628-fc0668eff14d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 06:31:33 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 06:31:33\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=442714;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=3865;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"two\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"text_classification_eval\"\n",
              "]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='text_dataset',eval_id='one')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5_BrYlBUS7Uf"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(eval_id=\"one\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t6sRPrzDTAs1",
        "outputId": "32f18b36-06db-4184-8b5f-bcf58dcc5c14"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "macro_avg_roc_auc: 0.6499999761581421\n",
              "macro_std_roc_auc: 0.07468751072883606\n",
              "macro_avg_f1_score: 0.75\n",
              "macro_avg_precision: 0.6000000238418579\n",
              "macro_avg_recall: 0.5"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "id": "mxaN0f9lTC1t",
        "outputId": "eddcc0c2-47e0-4e6f-d231-4d44ad84fa3c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:12:02 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"26440b0f63d3b6d95d6e3749a261609d\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:12:02\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=13163;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=200693;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"26440b0f63d3b6d95d6e3749a261609d\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"26440b0f63d3b6d95d6e3749a261609d\"\n",
            "\n",
            "Uploading inputs:   0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:12:09 </span><span style=\"color: #800000; text-decoration-color: #800000\">WARNING </span> clarifai.client.input:  Failed to upload <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> inputs..                       <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">input.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">947</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:12:09\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m clarifai.client.input:  Failed to upload \u001b[1;36m0\u001b[0m inputs..                       \u001b]8;id=764062;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py\u001b\\\u001b[2minput.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=444178;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/input.py#947\u001b\\\u001b[2m947\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m                                                                                   \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:clarifai.client.input:Failed to upload 0 inputs..\n",
            " \n",
            "Uploading inputs: 100%|██████████| 1/1 [00:07<00:00,  7.10s/it]\n"
          ]
        }
      ],
      "source": [
        "#importing load_module_dataloader for calling the dataloader object in dataset.py in the local data folder\n",
        "from clarifai.datasets.upload.utils import load_module_dataloader\n",
        "\n",
        "\n",
        "# Construct the path to the dataset folder\n",
        "CSV_PATH = os.path.join(os.getcwd().split('/models/model_train')[0],'datasets/upload/data/test_imdb.csv')\n",
        "\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id\n",
        "test_dataset = app.create_dataset(dataset_id=\"test_text_dataset\")\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "test_dataset.upload_from_csv(csv_path=CSV_PATH,input_type='text',csv_type='raw', labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Testing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "64jurJoj_QkJ",
        "outputId": "aec778be-09f6-474e-a28d-b16be3e7c3e5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:12:33 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:12:33\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=876435;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=462105;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"five\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"text_classification_eval\"\n",
              "]"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='test_text_dataset',eval_id='two')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8uPy069bCIyN"
      },
      "outputs": [],
      "source": [
        "model.get_eval_by_id(\"two\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lGPMwHsB_aPl"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(\"two\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "51c3jswU_kN6",
        "outputId": "3cefaab6-28e4-4351-ab20-1b834a414b11"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "macro_avg_roc_auc: 0.6161290407180786\n",
              "macro_std_roc_auc: 0.1225806474685669\n",
              "macro_avg_f1_score: 0.7207207679748535\n",
              "macro_avg_precision: 0.5633803009986877\n",
              "macro_avg_recall: 0.5"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Compare Results From Train & Test Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "id": "MQV2VxuD-Lm_",
        "outputId": "00543b95-b9ce-4357-dcf1-508f3fc5041d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:13:55 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Initializing models<span style=\"color: #808000; text-decoration-color: #808000\">...</span>                      <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">91</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:13:55\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Initializing models\u001b[33m...\u001b[0m                      \u001b]8;id=604638;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=18340;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\u001b\\\u001b[2m91\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Initializing models...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  *                                          <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">100</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         text_classification_eval/model_text-classifier/7130c                   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">               </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  *                                          \u001b]8;id=1549;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=927564;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\u001b\\\u001b[2m100\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         text_classification_eval/model_text-classifier/7130c                   \u001b[2m               \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:* text_classification_eval/model_text-classifier/7130c\n"
          ]
        }
      ],
      "source": [
        "from clarifai.utils.evaluation import EvalResultCompare\n",
        "\n",
        "eval_result = EvalResultCompare(models=[model],datasets= [dataset,test_dataset])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 805
        },
        "id": "V3T4GmfKYvZE",
        "outputId": "1430cdfa-e647-4c17-c1d8-f933fd57147a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:14:39 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Running `detailed_summary`                 <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">123</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:14:39\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Running `detailed_summary`                 \u001b]8;id=563859;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=348033;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\u001b\\\u001b[2m123\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Running `detailed_summary`\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:14:39 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         text_classification_eval/model_text-classifier/7130c: retrieving        <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: text_dataset2              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:14:39\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=830824;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=765;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         text_classification_eval/model_text-classifier/7130c: retrieving        \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: text_dataset2              \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         text_classification_eval/model_text-classifier/7130c: retrieving        <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: text_dataset2              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=599903;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=899548;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         text_classification_eval/model_text-classifier/7130c: retrieving        \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: text_dataset2              \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model text_classification_eval/model_text-classifier/7130c: retrieving {'binary_metrics': True} metrics of dataset: text_dataset2\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         text_classification_eval/model_text-classifier/7130c: retrieving        <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_text_dataset3         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=354432;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=389831;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         text_classification_eval/model_text-classifier/7130c: retrieving        \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_text_dataset3         \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         text_classification_eval/model_text-classifier/7130c: retrieving        <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: test_text_dataset3         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=856624;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=998266;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         text_classification_eval/model_text-classifier/7130c: retrieving        \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: test_text_dataset3         \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model text_classification_eval/model_text-classifier/7130c: retrieving {'binary_metrics': True} metrics of dataset: test_text_dataset3\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "(  Concept  Accuracy (ROC AUC)  Total Labeled  Total Predicted  True Positives  \\\n",
              " 0  id-pos               0.725             80                0               0   \n",
              " 0  id-neg               0.575            120              200             120   \n",
              " 0  id-pos               0.739             31                0               0   \n",
              " 0  id-neg               0.494             40               71              40   \n",
              " \n",
              "    False Negatives  False Positives  Recall  Precision        F1  \\\n",
              " 0               80                0     0.0     1.0000  0.000000   \n",
              " 0                0               80     1.0     0.6000  0.750000   \n",
              " 0               31                0     0.0     1.0000  0.000000   \n",
              " 0                0               31     1.0     0.5634  0.720737   \n",
              " \n",
              "               Dataset  \n",
              " 0       text_dataset2  \n",
              " 0       text_dataset2  \n",
              " 0  test_text_dataset3  \n",
              " 0  test_text_dataset3  ,\n",
              "                 Total Concept  Accuracy (ROC AUC)  Total Labeled  \\\n",
              " 0       Dataset:text_dataset2            0.650000            200   \n",
              " 0  Dataset:test_text_dataset3            0.616129             71   \n",
              " \n",
              "    Total Predicted  True Positives  False Negatives  False Positives   Recall  \\\n",
              " 0              200             120               80               80  0.60000   \n",
              " 0               71              40               31               31  0.56338   \n",
              " \n",
              "    Precision        F1  \n",
              " 0   0.760000  0.670588  \n",
              " 0   0.754028  0.644909  )"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "eval_result.detailed_summary()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}