5 Core TMA Weight Calculation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bad9132d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage.measure import label\n",
    "from skimage.io import imread\n",
    "from skimage.measure import regionprops\n",
    "from skimage.segmentation import expand_labels\n",
    "from scipy.special import softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "59d5dbb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"Mesmer\", \"Stardist\", \"Cellpose\", \"UnMicst\"]\n",
    "datapath = \"/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/\"\n",
    "sample_ids = ['Scene 002','Scene 003', 'Scene 017', 'Scene 049', 'Scene 059']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6acd563",
   "metadata": {},
   "source": [
    "# Create Probabilty Masks with 12 Radius"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c446a2d",
   "metadata": {},
   "source": [
    "The first step here is to generate the 'probability masks' from the original segmentation masks for every sample for every method. This is done by first zeroing out the mask, then taking the point at the center of each segmented cell, setting it equal to 1, and dilating out with a disc shape and a 12-pixel radius (the radius length varies)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1357f8c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mask(datapath, sid, method):\n",
    "    \"\"\"load raw segmentation mask for method/sid\"\"\"\n",
    "    datapath = os.path.join(datapath,method)\n",
    "    if method == 'Mesmer': datapath = os.path.join(datapath, 'mesmer_nuc')\n",
    "    if method == 'Stardist': datapath = os.path.join(datapath, 'vf')\n",
    "    files = os.listdir(datapath)\n",
    "    sid_file = [f for f in files if sid in f][0]\n",
    "    mask_path = os.path.join(datapath, sid_file)\n",
    "    print(mask_path)\n",
    "    return imread(mask_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7b11323f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_pmask(seg_mask, r):\n",
    "    \"\"\"converts a standard segmentation mask into one with a dilated area around \n",
    "    the center of the original mask\"\"\"\n",
    "    mask = np.zeros_like(seg_mask)\n",
    "    rps = regionprops(seg_mask)\n",
    "    \n",
    "    #get center of each cell and set the value to one\n",
    "    centroids = np.array(\n",
    "        list(map(lambda x : np.array(x.centroid).astype(int), rps))\n",
    "    )\n",
    "    mask[centroids[:,0], centroids[:,1]] = 1\n",
    "    \n",
    "    #dilate around the center of the cell with the radius size and binarize the mask\n",
    "    #mask = binary_dilation(mask, disk(r))\n",
    "    mask = expand_labels(label(mask), distance=r)\n",
    "    mask = (mask > 0).astype('int')\n",
    "    \n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9b5d53fa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "generating pmasks for sid='Scene 002'\n",
      "generating pmask for method='Mesmer'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Mesmer/mesmer_nuc/BR1506-A015 - Scene 002_both_nuc.tif\n",
      "generating pmask for method='Stardist'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Stardist/vf/BR1506-A015 - Scene 002.png\n",
      "generating pmask for method='Cellpose'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Cellpose/BR1506-A015 - Scene 002.png_mask.png\n",
      "generating pmask for method='UnMicst'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/UnMicst/BR1506-A015 - Scene 002.tif\n",
      "\n",
      "generating pmasks for sid='Scene 003'\n",
      "generating pmask for method='Mesmer'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Mesmer/mesmer_nuc/BR1506-A015 - Scene 003_both_nuc.tif\n",
      "generating pmask for method='Stardist'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Stardist/vf/BR1506-A015 - Scene 003.png\n",
      "generating pmask for method='Cellpose'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Cellpose/BR1506-A015 - Scene 003.png_mask.png\n",
      "generating pmask for method='UnMicst'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/UnMicst/BR1506-A015 - Scene 003.tif\n",
      "\n",
      "generating pmasks for sid='Scene 017'\n",
      "generating pmask for method='Mesmer'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Mesmer/mesmer_nuc/BR1506-A015 - Scene 017_both_nuc.tif\n",
      "generating pmask for method='Stardist'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Stardist/vf/BR1506-A015 - Scene 017.png\n",
      "generating pmask for method='Cellpose'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Cellpose/BR1506-A015 - Scene 017.png_mask.png\n",
      "generating pmask for method='UnMicst'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/UnMicst/BR1506-A015 - Scene 017.tif\n",
      "\n",
      "generating pmasks for sid='Scene 049'\n",
      "generating pmask for method='Mesmer'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Mesmer/mesmer_nuc/BR1506-A015 - Scene 049_both_nuc.tif\n",
      "generating pmask for method='Stardist'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Stardist/vf/BR1506-A015 - Scene 049.png\n",
      "generating pmask for method='Cellpose'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Cellpose/BR1506-A015 - Scene 049.png_mask.png\n",
      "generating pmask for method='UnMicst'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/UnMicst/BR1506-A015 - Scene 049.tif\n",
      "\n",
      "generating pmasks for sid='Scene 059'\n",
      "generating pmask for method='Mesmer'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Mesmer/mesmer_nuc/BR1506-A015 - Scene 059_both_nuc.tif\n",
      "generating pmask for method='Stardist'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Stardist/vf/BR1506-A015 - Scene 059.png\n",
      "generating pmask for method='Cellpose'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/Cellpose/BR1506-A015 - Scene 059.png_mask.png\n",
      "generating pmask for method='UnMicst'\n",
      "/home/groups/ChangLab/dataset/HMS-TMA-TNP/OHSU-TMA/Segmentations/UnMicst/BR1506-A015 - Scene 059.tif\n"
     ]
    }
   ],
   "source": [
    "pmasks = {}\n",
    "\n",
    "for sid in sample_ids:\n",
    "    print()\n",
    "    print(f'generating pmasks for {sid=}')\n",
    "    pmasks[sid] = {}\n",
    "    \n",
    "    for method in methods:\n",
    "        print(f'generating pmask for {method=}')\n",
    "        pmasks[sid][method] = create_pmask(get_mask(datapath, sid, method), 12)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86811923",
   "metadata": {},
   "source": [
    "#  Create Pseudo-Ground-Truths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "0666efa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = np.ones((len(sample_ids), len(methods) - 1)) * (1/ (len(methods) - 1))\n",
    "agree_ratio = 2/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "d89af8ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pseudo_ground_truths(pmasks, sample_ids, methods, weights, agree_ratio, filtered=False):\n",
    "    pseudo_gts = {}\n",
    "\n",
    "    for s,sid in enumerate(sample_ids):\n",
    "        print()\n",
    "        print(f'Generating PGTs for SID:{sid}')\n",
    "        pseudo_gts[sid] = {}\n",
    "        \n",
    "        for method_left_out in methods:\n",
    "            print(f'Generating PGT with {method_left_out} left out')\n",
    "            if filtered:\n",
    "                pmasks_ = pmasks[method_left_out][sid] #filtered with resepect to method-left-out PGT \n",
    "                print(pmasks_.keys())\n",
    "            else:\n",
    "                pmasks_ = pmasks[sid] #unfiltered pmasks are the same for each method/sid, regardless of PGT\n",
    "        \n",
    "            methods_in = [method for method in methods if method != method_left_out]\n",
    "            print(f'using {methods_in}')\n",
    "            print()\n",
    "            pgt = sum([pmasks_[method] * weights[s][m] for m,method in enumerate(methods_in)])\n",
    "            pseudo_gts[sid][method_left_out] = (pgt >= agree_ratio).astype('int')\n",
    "            \n",
    "    return pseudo_gts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "271fba49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generating PGTs for SID:Scene 002\n",
      "Generating PGT with Mesmer left out\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 003\n",
      "Generating PGT with Mesmer left out\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 017\n",
      "Generating PGT with Mesmer left out\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 049\n",
      "Generating PGT with Mesmer left out\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 059\n",
      "Generating PGT with Mesmer left out\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pseudo_gts = get_pseudo_ground_truths(pmasks=pmasks, \n",
    "                                      sample_ids=sample_ids, \n",
    "                                      methods=methods, \n",
    "                                      weights=weights, \n",
    "                                      agree_ratio=agree_ratio,\n",
    "                                      filtered=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74f5e234",
   "metadata": {},
   "source": [
    "# Filter Probability Masks based off generated PGTs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "93a0a7f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_pmask(mask, pgt):\n",
    "    \"\"\"filter probability mask for a single method (mask) using the thresholded, averaged, proability mask\n",
    "    such that the new probability mask contains cell regions that only overlap with one cell region in the\n",
    "    pseudo-ground-truth\"\"\"\n",
    "    #copy \n",
    "    filtered = mask.copy()\n",
    "    mask_lab = label(mask)\n",
    "    rps = regionprops(mask_lab)\n",
    "    \n",
    "    pgt = label(pgt)\n",
    "    \n",
    "    for rp in rps: #for each cell region\n",
    "        #get cell center coordinates from single-method mask\n",
    "        coords = rp.coords\n",
    "        #use those coords to get values of averaged mask \n",
    "        vals = pgt[coords[:,0], coords[:,1]]\n",
    "        uniq, counts = np.unique(vals, return_counts=True)\n",
    "        \n",
    "        #ignore background\n",
    "        if uniq[0] == 0:\n",
    "            uniq = uniq[1:]\n",
    "            counts = counts[1:]\n",
    "            \n",
    "        n_unique = len(uniq)\n",
    "        \n",
    "        #if more than 1 value, zero out pixels in mask that do not equal the most common value\n",
    "        if n_unique > 1:\n",
    "            amax = np.argmax(counts)\n",
    "            top_val = uniq[amax]\n",
    "            idxs = np.where(vals != top_val)\n",
    "            to_zero = coords[idxs,:][0]\n",
    "            filtered[to_zero[:,0], to_zero[:,1]] = False\n",
    "            \n",
    "    return filtered"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04db53ce",
   "metadata": {},
   "source": [
    "filter probability masks so that each cell region in the probability mask can only vote for one cell label in the pseudo-ground truth mask. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "90b8e203",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "f_pmasks = {}\n",
    "\n",
    "for method_out in methods:\n",
    "    f_pmasks[method_out] = {}\n",
    "    \n",
    "    for sid in sample_ids:\n",
    "        f_pmasks[method_out][sid] = {}\n",
    "\n",
    "        for method_in in methods:\n",
    "            if method_in == method_out: continue \n",
    "\n",
    "            f_pmasks[method_out][sid][method_in] = filter_pmask(mask=pmasks[sid][method_in], \n",
    "                                                                pgt=pseudo_gts[sid][method_out])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "493abb89",
   "metadata": {},
   "source": [
    "# Regenerate PGTs with filtered pmasks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a0b08a",
   "metadata": {},
   "source": [
    "Now we need to regenerate these pseudo-ground-truth masks, this time using filtered probability masks where each probability mask is filtered down to where one pseudo-cell-label in the  probability mask can only vote for one cell label in the pseudo-ground truth mask. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "245e8b1d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generating PGTs for SID:Scene 002\n",
      "Generating PGT with Mesmer left out\n",
      "dict_keys(['Stardist', 'Cellpose', 'UnMicst'])\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "dict_keys(['Mesmer', 'Cellpose', 'UnMicst'])\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'UnMicst'])\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'Cellpose'])\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 003\n",
      "Generating PGT with Mesmer left out\n",
      "dict_keys(['Stardist', 'Cellpose', 'UnMicst'])\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "dict_keys(['Mesmer', 'Cellpose', 'UnMicst'])\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'UnMicst'])\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'Cellpose'])\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 017\n",
      "Generating PGT with Mesmer left out\n",
      "dict_keys(['Stardist', 'Cellpose', 'UnMicst'])\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "dict_keys(['Mesmer', 'Cellpose', 'UnMicst'])\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'UnMicst'])\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'Cellpose'])\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 049\n",
      "Generating PGT with Mesmer left out\n",
      "dict_keys(['Stardist', 'Cellpose', 'UnMicst'])\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "dict_keys(['Mesmer', 'Cellpose', 'UnMicst'])\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'UnMicst'])\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'Cellpose'])\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n",
      "\n",
      "Generating PGTs for SID:Scene 059\n",
      "Generating PGT with Mesmer left out\n",
      "dict_keys(['Stardist', 'Cellpose', 'UnMicst'])\n",
      "using ['Stardist', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Stardist left out\n",
      "dict_keys(['Mesmer', 'Cellpose', 'UnMicst'])\n",
      "using ['Mesmer', 'Cellpose', 'UnMicst']\n",
      "\n",
      "Generating PGT with Cellpose left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'UnMicst'])\n",
      "using ['Mesmer', 'Stardist', 'UnMicst']\n",
      "\n",
      "Generating PGT with UnMicst left out\n",
      "dict_keys(['Mesmer', 'Stardist', 'Cellpose'])\n",
      "using ['Mesmer', 'Stardist', 'Cellpose']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f_pseudo_gts =  get_pseudo_ground_truths(pmasks=f_pmasks, \n",
    "                                         sample_ids=sample_ids,\n",
    "                                         methods=methods, \n",
    "                                         weights=weights, \n",
    "                                         agree_ratio=agree_ratio, \n",
    "                                         filtered=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8fe9636",
   "metadata": {},
   "source": [
    "# Generate F1 table"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0ed243d",
   "metadata": {},
   "source": [
    "Now we need to generate f1 table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "37b6b52d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_score(gt, method):\n",
    "    \"\"\"returns precision and recall for a pair of masks\"\"\"\n",
    "    \n",
    "    ### PRECISION ###\n",
    "    m_labs = label(method)\n",
    "    rps = regionprops(m_labs)\n",
    "    coords = list(map(lambda x : x.coords, rps))\n",
    "    correct = 0\n",
    "    print(f'number of regions: {len(rps)}')\n",
    "    \n",
    "    for c in coords:\n",
    "        correct += (gt[c[:,0], c[:,1]]).max()\n",
    "    print(f'num correct for precision: {correct}')\n",
    "    precision = correct / len(rps)\n",
    "    \n",
    "    ### RECALL ###\n",
    "    gt_labs = label(gt)\n",
    "    gt_rps = regionprops(gt_labs)\n",
    "    coords = list(map(lambda x : x.coords, gt_rps))\n",
    "    correct = 0\n",
    "    print(f'number of gt regions: {len(gt_rps)}')\n",
    "    \n",
    "    for c in coords:\n",
    "        correct += (method[c[:,0], c[:,1]]).max() > 0\n",
    "    print(f'num correct for recall: {correct}')\n",
    "    recall = correct / len(gt_rps)\n",
    "    \n",
    "    assert precision <= 1\n",
    "    assert precision >= 0\n",
    "    assert recall <= 1\n",
    "    assert recall >= 0\n",
    "    \n",
    "    f1 = lambda p,r: 2 * (p * r) / (p + r)\n",
    "    #f1 = lambda p,r: (p + r)/2\n",
    "    return precision, recall, f1(precision, recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "b618f989",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "method_to_eval='Mesmer' method_left_out='Stardist'\n",
      "number of regions: 10578\n",
      "num correct for precision: 10264\n",
      "number of gt regions: 9687\n",
      "num correct for recall: 9628\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Cellpose'\n",
      "number of regions: 10575\n",
      "num correct for precision: 10179\n",
      "number of gt regions: 9428\n",
      "num correct for recall: 9369\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='UnMicst'\n",
      "number of regions: 10584\n",
      "num correct for precision: 10377\n",
      "number of gt regions: 9946\n",
      "num correct for recall: 9705\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Mesmer'\n",
      "number of regions: 9345\n",
      "num correct for precision: 9068\n",
      "number of gt regions: 8919\n",
      "num correct for recall: 8534\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Cellpose'\n",
      "number of regions: 9337\n",
      "num correct for precision: 9133\n",
      "number of gt regions: 9428\n",
      "num correct for recall: 8974\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='UnMicst'\n",
      "number of regions: 9340\n",
      "num correct for precision: 9173\n",
      "number of gt regions: 9946\n",
      "num correct for recall: 8691\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Mesmer'\n",
      "number of regions: 9603\n",
      "num correct for precision: 9374\n",
      "number of gt regions: 8919\n",
      "num correct for recall: 8798\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Stardist'\n",
      "number of regions: 9600\n",
      "num correct for precision: 9502\n",
      "number of gt regions: 9687\n",
      "num correct for recall: 9476\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='UnMicst'\n",
      "number of regions: 9609\n",
      "num correct for precision: 9567\n",
      "number of gt regions: 9946\n",
      "num correct for recall: 9288\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Mesmer'\n",
      "number of regions: 5688\n",
      "num correct for precision: 5659\n",
      "number of gt regions: 8919\n",
      "num correct for recall: 5485\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Stardist'\n",
      "number of regions: 5653\n",
      "num correct for precision: 5627\n",
      "number of gt regions: 9687\n",
      "num correct for recall: 5523\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Cellpose'\n",
      "number of regions: 5662\n",
      "num correct for precision: 5642\n",
      "number of gt regions: 9428\n",
      "num correct for recall: 5567\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Stardist'\n",
      "number of regions: 7678\n",
      "num correct for precision: 6231\n",
      "number of gt regions: 5936\n",
      "num correct for recall: 5920\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Cellpose'\n",
      "number of regions: 7680\n",
      "num correct for precision: 6821\n",
      "number of gt regions: 6523\n",
      "num correct for recall: 6490\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='UnMicst'\n",
      "number of regions: 7677\n",
      "num correct for precision: 6630\n",
      "number of gt regions: 6503\n",
      "num correct for recall: 6422\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Mesmer'\n",
      "number of regions: 6913\n",
      "num correct for precision: 5849\n",
      "number of gt regions: 5681\n",
      "num correct for recall: 5574\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Cellpose'\n",
      "number of regions: 6917\n",
      "num correct for precision: 6403\n",
      "number of gt regions: 6523\n",
      "num correct for recall: 6264\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='UnMicst'\n",
      "number of regions: 6924\n",
      "num correct for precision: 6376\n",
      "number of gt regions: 6503\n",
      "num correct for recall: 6199\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Mesmer'\n",
      "number of regions: 5635\n",
      "num correct for precision: 5562\n",
      "number of gt regions: 5681\n",
      "num correct for recall: 5352\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Stardist'\n",
      "number of regions: 5635\n",
      "num correct for precision: 5602\n",
      "number of gt regions: 5936\n",
      "num correct for recall: 5479\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='UnMicst'\n",
      "number of regions: 5636\n",
      "num correct for precision: 5616\n",
      "number of gt regions: 6503\n",
      "num correct for recall: 5528\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Mesmer'\n",
      "number of regions: 4479\n",
      "num correct for precision: 4295\n",
      "number of gt regions: 5681\n",
      "num correct for recall: 4235\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Stardist'\n",
      "number of regions: 4476\n",
      "num correct for precision: 4409\n",
      "number of gt regions: 5936\n",
      "num correct for recall: 4361\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Cellpose'\n",
      "number of regions: 4478\n",
      "num correct for precision: 4432\n",
      "number of gt regions: 6523\n",
      "num correct for recall: 4382\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Stardist'\n",
      "number of regions: 5710\n",
      "num correct for precision: 5502\n",
      "number of gt regions: 5023\n",
      "num correct for recall: 4950\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Cellpose'\n",
      "number of regions: 5710\n",
      "num correct for precision: 5528\n",
      "number of gt regions: 5033\n",
      "num correct for recall: 4982\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='UnMicst'\n",
      "number of regions: 5713\n",
      "num correct for precision: 5592\n",
      "number of gt regions: 5207\n",
      "num correct for recall: 5097\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Mesmer'\n",
      "number of regions: 4937\n",
      "num correct for precision: 4767\n",
      "number of gt regions: 4732\n",
      "num correct for recall: 4402\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Cellpose'\n",
      "number of regions: 4941\n",
      "num correct for precision: 4825\n",
      "number of gt regions: 5033\n",
      "num correct for recall: 4591\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='UnMicst'\n",
      "number of regions: 4940\n",
      "num correct for precision: 4853\n",
      "number of gt regions: 5207\n",
      "num correct for recall: 4612\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Mesmer'\n",
      "number of regions: 5047\n",
      "num correct for precision: 4954\n",
      "number of gt regions: 4732\n",
      "num correct for recall: 4604\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Stardist'\n",
      "number of regions: 5047\n",
      "num correct for precision: 4997\n",
      "number of gt regions: 5023\n",
      "num correct for recall: 4804\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='UnMicst'\n",
      "number of regions: 5049\n",
      "num correct for precision: 5025\n",
      "number of gt regions: 5207\n",
      "num correct for recall: 4831\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Mesmer'\n",
      "number of regions: 4031\n",
      "num correct for precision: 3993\n",
      "number of gt regions: 4732\n",
      "num correct for recall: 3792\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Stardist'\n",
      "number of regions: 4030\n",
      "num correct for precision: 4007\n",
      "number of gt regions: 5023\n",
      "num correct for recall: 3873\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Cellpose'\n",
      "number of regions: 4030\n",
      "num correct for precision: 4007\n",
      "number of gt regions: 5033\n",
      "num correct for recall: 3895\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Stardist'\n",
      "number of regions: 4586\n",
      "num correct for precision: 4409\n",
      "number of gt regions: 4296\n",
      "num correct for recall: 4268\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Cellpose'\n",
      "number of regions: 4586\n",
      "num correct for precision: 4409\n",
      "number of gt regions: 4307\n",
      "num correct for recall: 4283\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='UnMicst'\n",
      "number of regions: 4585\n",
      "num correct for precision: 4484\n",
      "number of gt regions: 4442\n",
      "num correct for recall: 4344\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Mesmer'\n",
      "number of regions: 4500\n",
      "num correct for precision: 4260\n",
      "number of gt regions: 4243\n",
      "num correct for recall: 4115\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Cellpose'\n",
      "number of regions: 4501\n",
      "num correct for precision: 4263\n",
      "number of gt regions: 4307\n",
      "num correct for recall: 4158\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='UnMicst'\n",
      "number of regions: 4501\n",
      "num correct for precision: 4339\n",
      "number of gt regions: 4442\n",
      "num correct for recall: 4226\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Mesmer'\n",
      "number of regions: 4401\n",
      "num correct for precision: 4290\n",
      "number of gt regions: 4243\n",
      "num correct for recall: 4180\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Stardist'\n",
      "number of regions: 4402\n",
      "num correct for precision: 4297\n",
      "number of gt regions: 4296\n",
      "num correct for recall: 4205\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='UnMicst'\n",
      "number of regions: 4401\n",
      "num correct for precision: 4383\n",
      "number of gt regions: 4442\n",
      "num correct for recall: 4273\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Mesmer'\n",
      "number of regions: 3432\n",
      "num correct for precision: 3425\n",
      "number of gt regions: 4243\n",
      "num correct for recall: 3348\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Stardist'\n",
      "number of regions: 3431\n",
      "num correct for precision: 3428\n",
      "number of gt regions: 4296\n",
      "num correct for recall: 3372\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Cellpose'\n",
      "number of regions: 3434\n",
      "num correct for precision: 3434\n",
      "number of gt regions: 4307\n",
      "num correct for recall: 3370\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Stardist'\n",
      "number of regions: 4141\n",
      "num correct for precision: 4061\n",
      "number of gt regions: 3910\n",
      "num correct for recall: 3872\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='Cellpose'\n",
      "number of regions: 4139\n",
      "num correct for precision: 4021\n",
      "number of gt regions: 3770\n",
      "num correct for recall: 3747\n",
      "\n",
      "method_to_eval='Mesmer' method_left_out='UnMicst'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of regions: 4139\n",
      "num correct for precision: 4095\n",
      "number of gt regions: 4047\n",
      "num correct for recall: 3915\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Mesmer'\n",
      "number of regions: 3841\n",
      "num correct for precision: 3696\n",
      "number of gt regions: 3800\n",
      "num correct for recall: 3580\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='Cellpose'\n",
      "number of regions: 3842\n",
      "num correct for precision: 3645\n",
      "number of gt regions: 3770\n",
      "num correct for recall: 3543\n",
      "\n",
      "method_to_eval='Stardist' method_left_out='UnMicst'\n",
      "number of regions: 3845\n",
      "num correct for precision: 3743\n",
      "number of gt regions: 4047\n",
      "num correct for recall: 3639\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Mesmer'\n",
      "number of regions: 4084\n",
      "num correct for precision: 3985\n",
      "number of gt regions: 3800\n",
      "num correct for recall: 3759\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='Stardist'\n",
      "number of regions: 4085\n",
      "num correct for precision: 3957\n",
      "number of gt regions: 3910\n",
      "num correct for recall: 3830\n",
      "\n",
      "method_to_eval='Cellpose' method_left_out='UnMicst'\n",
      "number of regions: 4085\n",
      "num correct for precision: 4055\n",
      "number of gt regions: 4047\n",
      "num correct for recall: 3937\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Mesmer'\n",
      "number of regions: 2902\n",
      "num correct for precision: 2887\n",
      "number of gt regions: 3800\n",
      "num correct for recall: 2821\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Stardist'\n",
      "number of regions: 2904\n",
      "num correct for precision: 2896\n",
      "number of gt regions: 3910\n",
      "num correct for recall: 2835\n",
      "\n",
      "method_to_eval='UnMicst' method_left_out='Cellpose'\n",
      "number of regions: 2904\n",
      "num correct for precision: 2894\n",
      "number of gt regions: 3770\n",
      "num correct for recall: 2836\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f1_table = np.zeros((len(sample_ids), len(methods), len(methods)))\n",
    "p_table = np.zeros((len(sample_ids), len(methods), len(methods)))\n",
    "r_table = np.zeros((len(sample_ids), len(methods), len(methods)))\n",
    "\n",
    "for s,sid in enumerate(sample_ids):\n",
    "    for i,method_to_eval in enumerate(methods):\n",
    "        for j,method_left_out in enumerate(methods):\n",
    "            if method_left_out == method_to_eval: continue\n",
    "            print(f'{method_to_eval=} {method_left_out=}')\n",
    "            p_table[s][i][j],r_table[s][i][j],f1_table[s][i][j] = f1_score(gt=f_pseudo_gts[sid][method_left_out], \n",
    "                                                                           method=f_pmasks[method_left_out][sid][method_to_eval])\n",
    "            print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f13cd3d",
   "metadata": {},
   "source": [
    "We assign a weight to a method by quantifying that method's impact on the overall agreeance between the pseudo-ground-truth (PGT) and the other methods. For example, in order to find out how the presence of Stardist in the PGT label effects the correspondence between the PGT and the Mesmer segmentation, we first calculate the F1 scores between the Mesmer segmentation and each of the PGTs comprised of Mesmer and two of the three other methods i.e. PGT({Mesmer,Cellpose,UnMicst}), PGT({Mesmer, Stardist, UnMicst}), and PGT({Mesmer, Stardist, Cellpose}). If the F1 score between Mesmer and the PGT with Stardist left out i.e. PGT({Mesmer,Cellpose,UnMicst}) is high relative to the F1 scores between Mesmer and the other two PGTs containing Stardist, that indicates that Stardist is a bad contributor to the PGT, with respect to Mesmer. If the same holds true for Cellpose, and UnMicst, then it follows that the contribution of Stardist to the PGT should be discounted by assigning a lower weight to its vote."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "fba36161",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = [f'PGT({[m for m in methods if m != m_out]})' for m_out in methods]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6140d818",
   "metadata": {},
   "source": [
    "precision = 'How many predictions are in the ground-truth?'\n",
    "recall = 'how many ground-truths are in the prediction?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "5da4fc65",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PGT(['Stardist', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'Cellpose'])</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Mesmer</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.981971</td>\n",
       "      <td>0.977899</td>\n",
       "      <td>0.978100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stardist</th>\n",
       "      <td>0.963549</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.964819</td>\n",
       "      <td>0.924809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cellpose</th>\n",
       "      <td>0.981266</td>\n",
       "      <td>0.983971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.963747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UnMicst</th>\n",
       "      <td>0.760111</td>\n",
       "      <td>0.725016</td>\n",
       "      <td>0.741538</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          PGT(['Stardist', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                    0.000000   \n",
       "Stardist                                  0.963549   \n",
       "Cellpose                                  0.981266   \n",
       "UnMicst                                   0.760111   \n",
       "\n",
       "          PGT(['Mesmer', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                  0.981971   \n",
       "Stardist                                0.000000   \n",
       "Cellpose                                0.983971   \n",
       "UnMicst                                 0.725016   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'UnMicst'])  \\\n",
       "Mesmer                                  0.977899   \n",
       "Stardist                                0.964819   \n",
       "Cellpose                                0.000000   \n",
       "UnMicst                                 0.741538   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'Cellpose'])  \n",
       "Mesmer                                   0.978100  \n",
       "Stardist                                 0.924809  \n",
       "Cellpose                                 0.963747  \n",
       "UnMicst                                  0.000000  "
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with unfiltered pmask\n",
    "#pd.DataFrame(f1_table[0], index=methods, columns=[f'{m}-left-out' for m in methods])\n",
    "pd.DataFrame(f1_table[0], index=methods, columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "67e608d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PGT(['Stardist', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'Cellpose'])</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Mesmer</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.970316</td>\n",
       "      <td>0.962553</td>\n",
       "      <td>0.980442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stardist</th>\n",
       "      <td>0.970358</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.978151</td>\n",
       "      <td>0.982120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cellpose</th>\n",
       "      <td>0.976153</td>\n",
       "      <td>0.989792</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.995629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UnMicst</th>\n",
       "      <td>0.994902</td>\n",
       "      <td>0.995401</td>\n",
       "      <td>0.996468</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          PGT(['Stardist', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                    0.000000   \n",
       "Stardist                                  0.970358   \n",
       "Cellpose                                  0.976153   \n",
       "UnMicst                                   0.994902   \n",
       "\n",
       "          PGT(['Mesmer', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                  0.970316   \n",
       "Stardist                                0.000000   \n",
       "Cellpose                                0.989792   \n",
       "UnMicst                                 0.995401   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'UnMicst'])  \\\n",
       "Mesmer                                  0.962553   \n",
       "Stardist                                0.978151   \n",
       "Cellpose                                0.000000   \n",
       "UnMicst                                 0.996468   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'Cellpose'])  \n",
       "Mesmer                                   0.980442  \n",
       "Stardist                                 0.982120  \n",
       "Cellpose                                 0.995629  \n",
       "UnMicst                                  0.000000  "
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### with unfiltered pmask\n",
    "#pd.DataFrame(p_table[0], index=methods, columns=[f'{m}-left-out' for m in methods])\n",
    "pd.DataFrame(p_table[0], index=methods, columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "5b43a0dd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PGT(['Stardist', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Cellpose', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'UnMicst'])</th>\n",
       "      <th>PGT(['Mesmer', 'Stardist', 'Cellpose'])</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Mesmer</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.993909</td>\n",
       "      <td>0.993742</td>\n",
       "      <td>0.975769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stardist</th>\n",
       "      <td>0.956834</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.951846</td>\n",
       "      <td>0.873819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cellpose</th>\n",
       "      <td>0.986433</td>\n",
       "      <td>0.978218</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.933843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UnMicst</th>\n",
       "      <td>0.614979</td>\n",
       "      <td>0.570146</td>\n",
       "      <td>0.590475</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          PGT(['Stardist', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                    0.000000   \n",
       "Stardist                                  0.956834   \n",
       "Cellpose                                  0.986433   \n",
       "UnMicst                                   0.614979   \n",
       "\n",
       "          PGT(['Mesmer', 'Cellpose', 'UnMicst'])  \\\n",
       "Mesmer                                  0.993909   \n",
       "Stardist                                0.000000   \n",
       "Cellpose                                0.978218   \n",
       "UnMicst                                 0.570146   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'UnMicst'])  \\\n",
       "Mesmer                                  0.993742   \n",
       "Stardist                                0.951846   \n",
       "Cellpose                                0.000000   \n",
       "UnMicst                                 0.590475   \n",
       "\n",
       "          PGT(['Mesmer', 'Stardist', 'Cellpose'])  \n",
       "Mesmer                                   0.975769  \n",
       "Stardist                                 0.873819  \n",
       "Cellpose                                 0.933843  \n",
       "UnMicst                                  0.000000  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with unfiltered pmask\n",
    "#pd.DataFrame(r_table[0], index=methods, columns=[f'{m}-left-out' for m in methods])\n",
    "pd.DataFrame(r_table[0], index=methods, columns=column_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bae69d75",
   "metadata": {},
   "source": [
    "Mesmer, Stardist, and Cellpose have a higher f1 on average with the PGT({Mesmer, Stardist, Cellpose}) relative to PGT({Mesmer, Stardist, UnMicst}), PGT({Mesmer, Cellpose, UnMicst}), and PGT({Stardist, Cellpose, UnMicst}) implying that UnMicst displays poor contribution, when the agreeance ratio is 3/3.\n",
    "\n",
    "However, when the agreeance ratio is 2/3, then Mesmer, Stardist, and Cellpose have a lower f1 on average with the PGT({Mesmer, Stardist, Cellpose}), implying that UnMicst displays poor contribution."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba9a0e09",
   "metadata": {},
   "source": [
    "Now we need to convert to a delta table representing the relative change in performance caused by dropping a single method from the ensemble used for creating the PGT. We obtain this delta by averaging the non-zero F1 scores in each row, and subtracting that mean from each non-zero value in the row and dividing by the average, which gives us a normalized delta value. Finally, we multiply each value by 100 to obtain a percentage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "b2d3d972",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.25, 0.25, 0.25, 0.24])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.0116, 0.0167, 0.0192, -0.0475]).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "295fce04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Scene 002'"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_ids[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "14e48562",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[       nan, 0.98197086, 0.977899  , 0.97810008],\n",
       "       [0.96354865,        nan, 0.96481923, 0.92480936],\n",
       "       [0.98126645, 0.98397092,        nan, 0.96374665],\n",
       "       [0.7601107 , 0.7250163 , 0.74153827,        nan]])"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_table[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "4691875f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.97932331 0.95105908 0.97632801 0.74222176]\n",
      "[ 0.0735451   0.04256154  0.07026161 -0.18636826]\n",
      "[0.26757057 0.2594074  0.26669345 0.20632859]\n"
     ]
    }
   ],
   "source": [
    "f1_t = f1_table[0]\n",
    "f1_t[f1_t == 0] = np.nan\n",
    "mF1 = np.nanmean(f1_t, axis=1)\n",
    "print(mF1)\n",
    "mean_mF1 = np.mean(mF1)\n",
    "z = (mF1 - mean_mF1) / mean_mF1\n",
    "print(z)\n",
    "print(softmax(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "ec2f1b1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9016419318063336\n",
      "0.8969860262529866\n",
      "0.8947521665203345\n",
      "0.9555520329117567\n"
     ]
    }
   ],
   "source": [
    "f1_t = f1_table[0]\n",
    "f1_delta_table = np.zeros(f1_t.shape)\n",
    "f1_t[f1_t==0] = np.nan\n",
    "for m,method in enumerate(methods):\n",
    "    mean_col = np.nanmean(f1_t[:,m])\n",
    "    print(mean_col)\n",
    "    for i,f1 in enumerate(f1_t[:,m]):\n",
    "        if np.isnan(f1):\n",
    "            f1_delta_table[i,m] = np.nan\n",
    "        else:   \n",
    "            f1_delta_table[i,m] = 100 * (f1 - mean_col) / mean_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "6a542e74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[        nan, -9.47448771, -9.29272214, -2.35968843],\n",
       "       [-6.86599766,         nan, -7.83089049,  3.21726783],\n",
       "       [-8.83105724, -9.69746344,         nan, -0.8575794 ],\n",
       "       [15.6970549 , 19.17195115, 17.12361262,         nan]])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_delta_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "36b9d278",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.0735451   0.04256154  0.07026161 -0.18636826]\n",
      "[ 0.01444228  0.02668536  0.04534674 -0.08647438]\n",
      "[ 3.88435097e-02 -3.01122100e-05  3.11952143e-02 -7.00086118e-02]\n",
      "[ 0.03092616  0.00954085  0.03208051 -0.07254752]\n",
      "[ 0.04627907  0.00539881  0.04374742 -0.0954253 ]\n"
     ]
    }
   ],
   "source": [
    "weights  = np.zeros((len(sample_ids),len(methods))) \n",
    "for s,sid in enumerate(sample_ids):\n",
    "    f1_t = f1_table[s]\n",
    "    f1_t[f1_t == 0] = np.nan\n",
    "    mF1 = np.nanmean(f1_t, axis=1)\n",
    "    mean_mF1 = np.mean(mF1)\n",
    "    z = (mF1 - mean_mF1) / mean_mF1\n",
    "    print(z)\n",
    "    weights[s] = softmax(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6f7c8815",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_delta_table = np.zeros((len(sample_ids), len(methods), len(methods)))\n",
    "\n",
    "for s,_ in enumerate(f1_table):#get table for each sample\n",
    "    for i,f1_i in enumerate(f1_table[s]):#get row for each method\n",
    "        mean_f1 = np.mean([f1 for f1 in f1_i if f1 != 0])\n",
    "\n",
    "        for j,f1 in enumerate(f1_i):\n",
    "            if i == j: continue\n",
    "            f1_delta_table[s][i][j] = 100 * (f1 - mean_f1) / mean_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "04d62d45",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Mesmer dropped</th>\n",
       "      <th>Stardist dropped</th>\n",
       "      <th>Cellpose dropped</th>\n",
       "      <th>UnMicst dropped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Mesmer</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.270344</td>\n",
       "      <td>-0.145439</td>\n",
       "      <td>-0.124906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stardist</th>\n",
       "      <td>1.313227</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.446824</td>\n",
       "      <td>-2.760051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cellpose</th>\n",
       "      <td>0.505818</td>\n",
       "      <td>0.782822</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.288640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UnMicst</th>\n",
       "      <td>2.410189</td>\n",
       "      <td>-2.318102</td>\n",
       "      <td>-0.092087</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Mesmer dropped  Stardist dropped  Cellpose dropped  UnMicst dropped\n",
       "Mesmer          0.000000          0.270344         -0.145439        -0.124906\n",
       "Stardist        1.313227          0.000000          1.446824        -2.760051\n",
       "Cellpose        0.505818          0.782822          0.000000        -1.288640\n",
       "UnMicst         2.410189         -2.318102         -0.092087         0.000000"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(f1_delta_table[0], index=methods, columns=[f'{m} dropped' for m in methods])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d607f723",
   "metadata": {},
   "source": [
    "Finally, calculate weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5010be51",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights  = np.zeros((len(sample_ids),len(methods)))\n",
    "avg_deltas  = np.zeros((len(sample_ids),len(methods)))\n",
    "\n",
    "for s,sid in enumerate(sample_ids):\n",
    "    delta_table = f1_delta_table[s]\n",
    "    for m, method in enumerate(methods):\n",
    "        avg_deltas[s][m] = -np.mean([f1_d for f1_d in delta_table[:,m] if f1_d != 0])\n",
    "\n",
    "    weights[s] = softmax(avg_deltas[s])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e3cb04ef",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.03782273, 0.23611048, 0.10349831, 0.62256848],\n",
       "       [0.16076133, 0.20273457, 0.20088619, 0.43561791],\n",
       "       [0.08858254, 0.28050836, 0.22673281, 0.40417629],\n",
       "       [0.21646338, 0.26615496, 0.2739616 , 0.24342005],\n",
       "       [0.14993142, 0.36212145, 0.1616214 , 0.32632573]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9212cba5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:vit-mesmer] *",
   "language": "python",
   "name": "conda-env-vit-mesmer-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}