paper_counts.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "paper_counts.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "TB_pSQss0WmS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -U numpy\n",
        "!pip install pandas==0.24.2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zK9Bgo42KfTb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHLx-i92BR-P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import Counter\n",
        "from pathlib import Path\n",
        "\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "\n",
        "FONT_SIZE = 20\n",
        "matplotlib.rc('font', size=FONT_SIZE)\n",
        "matplotlib.rc('axes', labelsize=28)\n",
        "sns.set_palette('colorblind')\n",
        "\n",
        "NOTEBOOK_DIR = ('drive/My Drive/Cell Profiling/Challenges/'\n",
        "                'HPA challenge paper-Ouyang et al 2019/notebooks')\n",
        "TRUTH_PATH = Path(NOTEBOOK_DIR) / 'kaggle_derived_solution.csv'\n",
        "MASTER_TEST_PATH = Path(NOTEBOOK_DIR) / 'kaggle_master_test.csv'\n",
        "MASTER_TRAIN_PATH = Path(NOTEBOOK_DIR) / 'kaggle_master_training.csv'\n",
        "TRAIN_PATH = Path(NOTEBOOK_DIR) / 'kaggle_train.csv'\n",
        "HPA_PATH = Path(NOTEBOOK_DIR) / 'hpa_v18_1_all_no_dups_merged.csv'\n",
        "\n",
        "COL_CELL_TYPE = 'cell_type'\n",
        "COL_ID = 'id'\n",
        "COL_EXPECTED = 'expected'\n",
        "COL_PREDICTED = 'predicted'\n",
        "COL_TEAM = 'team'\n",
        "\n",
        "ABB_LABEL_INDEX = {\n",
        "    \"0\": \"Nucleoplasm\",\n",
        "    \"1\": \"N. membrane\",\n",
        "    \"2\": \"Nucleoli\",\n",
        "    \"3\": \"N. fibrillar c.\",\n",
        "    \"4\": \"N. speckles\",\n",
        "    \"5\": \"N. bodies\",\n",
        "    \"6\": \"ER\",\n",
        "    \"7\": \"Golgi app.\",\n",
        "    \"8\": \"Peroxisomes\",\n",
        "    \"9\": \"Endosomes\",\n",
        "    \"10\": \"Lysosomes\",\n",
        "    \"11\": \"Int. fil.\",\n",
        "    \"12\": \"Actin fil.\",\n",
        "    \"13\": \"F. a. sites\",\n",
        "    \"14\": \"Microtubules\",\n",
        "    \"15\": \"M. ends\",\n",
        "    \"16\": \"Cyt. bridge\",\n",
        "    \"17\": \"Mitotic spindle\",\n",
        "    \"18\": \"MTOC\",\n",
        "    \"19\": \"Centrosome\",\n",
        "    \"20\": \"Lipid droplets\",\n",
        "    \"21\": \"PM\",\n",
        "    \"22\": \"C. Junctions\",\n",
        "    \"23\": \"Mitochondria\",\n",
        "    \"24\": \"Aggresome\",\n",
        "    \"25\": \"Cytosol\",\n",
        "    \"26\": \"C. bodies\",\n",
        "    \"27\": \"Rods & Rings\"\n",
        "}\n",
        "\n",
        "\n",
        "def load_data(path):\n",
        "    \"\"\"Return data from csv file.\"\"\"\n",
        "    try:\n",
        "        data = pd.read_csv(path, compression=None, header=0, dtype=str)\n",
        "    except UnicodeError:\n",
        "        data = pd.read_csv(path, compression='gzip', header=0, dtype=str)\n",
        "    return data\n",
        "\n",
        "\n",
        "def tidy_cols(data):\n",
        "    _data = data.copy()\n",
        "    _data.columns = (_data.columns\n",
        "                    .str.strip()\n",
        "                    .str.lower()\n",
        "                    .str.replace(' ', '_')\n",
        "                    .str.replace('(', '')\n",
        "                    .str.replace(')', ''))\n",
        "    return _data\n",
        "\n",
        "\n",
        "class Count:\n",
        "    \"\"\"Represent a count to be calculated.\"\"\"\n",
        "    \n",
        "    def __init__(\n",
        "            self, hpa, kaggle_test, kaggle_train, master_test, master_train):\n",
        "        \"\"\"Set up instance.\"\"\"\n",
        "        self.hpa = hpa\n",
        "        self.kaggle_test = kaggle_test\n",
        "        self.kaggle_train = kaggle_train\n",
        "        self.master_test = master_test\n",
        "        self.master_train = master_train\n",
        "    \n",
        "    @classmethod\n",
        "    def create(\n",
        "            cls, hpa_path, test_path, train_path, master_test_path,\n",
        "            master_train_path):\n",
        "        hpa = (load_data(hpa_path)\n",
        "               .drop(columns='Unnamed: 0')\n",
        "               .rename(columns={'location': 'hpa'})\n",
        "               .loc[lambda df: ~df.hpa.str.contains('-1')])\n",
        "        hpa = cls.parse_data(hpa, 'hpa')\n",
        "        test = load_data(test_path)\n",
        "        test = cls.parse_data(test, 'expected')\n",
        "        train = load_data(train_path)\n",
        "        train = cls.parse_data(train, 'target')\n",
        "        master_test = (load_data(master_test_path)\n",
        "                        .pipe(tidy_cols))\n",
        "        master_train = (load_data(master_train_path)\n",
        "                        .pipe(tidy_cols))\n",
        "        return cls(hpa, test, train, master_test, master_train)\n",
        "    \n",
        "    @staticmethod\n",
        "    def parse_data(data, data_column):\n",
        "        \"\"\"Parse and return tidy data.\"\"\"\n",
        "        return (data\n",
        "                .pipe(tidy_cols)\n",
        "                .sort_values(by=[COL_ID])\n",
        "                .reset_index(drop=True)\n",
        "                .assign(**{data_column: lambda df: (df\n",
        "                                             .loc[:, data_column]\n",
        "                                             .str.split(' ')\n",
        "                                             .apply(pd.to_numeric))}))\n",
        "    \n",
        "    @staticmethod\n",
        "    def _set_plot_defaults(figsize=(16.5, 11.7), **kwargs):\n",
        "        \"\"\"Set and return plot defaults.\"\"\"\n",
        "        fig, ax = plt.subplots(figsize=figsize, **kwargs)\n",
        "        return fig, ax\n",
        "    \n",
        "    @staticmethod\n",
        "    def _save_fig(path):\n",
        "        \"\"\"Save the current figure to path.\"\"\"\n",
        "        plt.savefig(path, dpi=300, bbox_inches='tight')\n",
        "    \n",
        "    @staticmethod\n",
        "    def calc_count(data, data_type):\n",
        "        \"\"\"Calculate label count for a pandas Series data set.\"\"\"\n",
        "        count = data.apply(Counter).sum()\n",
        "        count = (pd.DataFrame(dict(count), index=[data_type])\n",
        "         .T\n",
        "         .sort_values(data_type, ascending=False))\n",
        "        return count\n",
        "    \n",
        "    @staticmethod\n",
        "    def calc_label_per_image(data, data_type):\n",
        "        \"\"\"Calculate count of amount of labels per image for Series data.\"\"\"\n",
        "        count = data.apply(len).value_counts()\n",
        "        count = pd.DataFrame(dict(count), index=[data_type]).T\n",
        "        return count\n",
        "    \n",
        "    @staticmethod\n",
        "    def rename_labels(data):\n",
        "        \"\"\"Rename labels.\"\"\"\n",
        "        data = data.rename(\n",
        "            index={int(idx): label for idx, label in ABB_LABEL_INDEX.items()})\n",
        "        return data\n",
        "    \n",
        "    def calc_test(self, row_slice=None, data_type='test'):\n",
        "        \"\"\"Calculate and return counts for kaggle test set.\"\"\"\n",
        "        if row_slice is None:\n",
        "            row_slice = slice(None)\n",
        "        test = self.kaggle_test.loc[row_slice]\n",
        "        return self.rename_labels(self.calc_count(test.expected, data_type))\n",
        "    \n",
        "    def calc_train(self, row_slice=None, method='calc_count', rename=True):\n",
        "        \"\"\"Calculate and return counts for kaggle training set.\"\"\"\n",
        "        if row_slice is None:\n",
        "            row_slice = slice(None)\n",
        "        train = self.kaggle_train.loc[row_slice]\n",
        "        count = getattr(self, method)(train.target, 'training')\n",
        "        if not rename:\n",
        "            return count\n",
        "        return self.rename_labels(count)\n",
        "    \n",
        "    def calc_test_cell_type(self, row_slice=None):\n",
        "        \"\"\"Calculate and return cell type counts for kaggle test set.\"\"\"\n",
        "        if row_slice is None:\n",
        "            row_slice = slice(None)\n",
        "        test = self.master_test.loc[row_slice]\n",
        "        return self.calc_count(\n",
        "            test.atlas_name.str.split(','), 'test cell type')\n",
        "    \n",
        "    def calc_train_cell_type(self, row_slice=None):\n",
        "        \"\"\"Calculate and return cell type counts for kaggle training set.\"\"\"\n",
        "        if row_slice is None:\n",
        "            row_slice = slice(None)\n",
        "        train = self.master_train.loc[row_slice]\n",
        "        return self.calc_count(\n",
        "            train.atlas_name.str.split(','), 'train cell type')\n",
        "    \n",
        "    def calc_private_lb(self):\n",
        "        \"\"\"Calculate and return counts for kaggle private leaderboard.\"\"\"\n",
        "        private_slice = self.kaggle_test.usage == 'Private'\n",
        "        return self.rename_labels(\n",
        "            self.calc_test(row_slice=private_slice, data_type='test_private'))\n",
        "    \n",
        "    def calc_public_lb(self):\n",
        "        \"\"\"Calculate and return counts for kaggle public leaderboard.\"\"\"\n",
        "        public_slice = self.kaggle_test.usage == 'Public'\n",
        "        return self.rename_labels(\n",
        "            self.calc_test(\n",
        "                row_slice=public_slice, data_type='validation_public'))\n",
        "    \n",
        "    def calc_ignored_lb(self):\n",
        "        \"\"\"Calculate and return counts for kaggle ignored samples.\"\"\"\n",
        "        ignored_slice = self.kaggle_test.usage == 'Ignored'\n",
        "        return self.rename_labels(\n",
        "            self.calc_test(row_slice=ignored_slice, data_type='ignored'))\n",
        "    \n",
        "    def calc_hpa(self, row_slice=None, method='calc_count', rename=True):\n",
        "        \"\"\"Calculate and return counts for hpa training set.\"\"\"\n",
        "        if row_slice is None:\n",
        "            row_slice = slice(None)\n",
        "        hpa = self.hpa.loc[row_slice]\n",
        "        count = getattr(self, method)(hpa.hpa, 'HPAv18')\n",
        "        if not rename:\n",
        "            return count\n",
        "        return self.rename_labels(count)\n",
        "    \n",
        "    def calc_hpa_image_per_ab(self):\n",
        "        \"\"\"Calculate how many images per antibody id there is in HPAv18.\"\"\"\n",
        "        count = (self.hpa\n",
        "                 .id.str.split('_')\n",
        "                 .apply(lambda x: x[0])\n",
        "                 .value_counts()\n",
        "                )\n",
        "        return count\n",
        "    \n",
        "    def plot_fig2a_class_distribution(self, save_path=None):\n",
        "        \"\"\"Return the plot for figure 1 class distribution.\"\"\"\n",
        "        return self.plot_kaggle(\n",
        "            orient='v', scale='linear', hpa=True, save_path=save_path)\n",
        "    \n",
        "    def plot_kaggle(\n",
        "            self, orient='h', scale='linear', hpa=True, save_path=None):\n",
        "        \"\"\"Plot a bar plot with kaggle counts.\"\"\"\n",
        "        private = self.calc_private_lb()\n",
        "        public = self.calc_public_lb()\n",
        "        train = self.calc_train()\n",
        "        if hpa:\n",
        "            hpa_set = self.calc_hpa()\n",
        "            all_sets = [hpa_set, train, public, private]\n",
        "            all_sets = (pd\n",
        "                        .concat(all_sets, axis=1, sort=False)\n",
        "                        .assign(**{'hpa_train': lambda df: (df\n",
        "                                            .loc[:, ['HPAv18', 'training']]\n",
        "                                            .sum(axis=1))\n",
        "                                  })\n",
        "                        .sort_values(by='hpa_train', ascending=False)\n",
        "                       )\n",
        "            hpa_train = all_sets.pop('hpa_train')\n",
        "        else:\n",
        "            all_sets = [train, public, private]\n",
        "            all_sets = pd.concat(all_sets, axis=1, sort=False)\n",
        "        \n",
        "        a3_dims = (16.5, 11.7)\n",
        "        fig, ax = self._set_plot_defaults(figsize=a3_dims)\n",
        "        ax.set_ylabel(\"Images\")\n",
        "        ax.set_xlabel(\"Location class\")\n",
        "        if hpa:\n",
        "            hpa_train.plot(\n",
        "                ax=ax, marker='+', linestyle='None',\n",
        "                color='black', label='HPAv18 + Training sets', legend=True)\n",
        "        if orient == 'h':\n",
        "            if scale == 'log':\n",
        "                ax.set_xscale('log')\n",
        "            bar = all_sets.plot.barh(\n",
        "                ax=ax, stacked=True,\n",
        "                color=sns.color_palette(n_colors=len(all_sets.columns)))\n",
        "        else:\n",
        "            if scale == 'log':\n",
        "                ax.set_yscale('log')\n",
        "            bar = all_sets.plot.bar(\n",
        "                ax=ax, stacked=True,\n",
        "                color=sns.color_palette(n_colors=len(all_sets.columns)))\n",
        "            bar.set_xticklabels(\n",
        "                labels=bar.get_xticklabels(), rotation=70, ha='right')\n",
        "        if save_path:\n",
        "            self._save_fig(Path(save_path) / 'fig2a_class_distribution_all.svg')\n",
        "        return bar\n",
        "    \n",
        "    def plot_hpa(self, orient='h', scale='log', save_path=None):\n",
        "        \"\"\"Plot a bar plot with hpa counts.\"\"\"\n",
        "        hpa_set = self.calc_hpa()\n",
        "        a3_dims = (16.5, 11.7)\n",
        "        fig, ax = self._set_plot_defaults(figsize=a3_dims)\n",
        "        if orient == 'h':\n",
        "            if scale == 'log':\n",
        "                ax.set_xscale('log')\n",
        "            bar = hpa_set.plot.barh(ax=ax)\n",
        "        else:\n",
        "            if scale == 'log':\n",
        "                ax.set_yscale('log')\n",
        "            bar = hpa_set.plot.bar(ax=ax)\n",
        "            bar.set_xticklabels(labels=bar.get_xticklabels(), rotation=70)\n",
        "        if save_path:\n",
        "            self._save_fig(Path(save_path) / 'class_distribution_hpa.svg')\n",
        "        return bar\n",
        "    \n",
        "    def plot_pie_per_class(self, save_path=None):\n",
        "        \"\"\"Plot a pie chart of training data per class.\"\"\"\n",
        "        train = self.calc_train().rename(columns={'training': 'Images'})\n",
        "        hpa = self.calc_hpa().rename(columns={'HPAv18': 'Images'})\n",
        "        data = train + hpa\n",
        "        data = data.sort_values('Images', ascending=False)\n",
        "        stats = data\n",
        "        print(stats)\n",
        "        fig, ax = self._set_plot_defaults()\n",
        "        show_labels = ('Nucleoplasm', 'Cytosol', 'Rods & Rings')\n",
        "        labels = [\n",
        "            label if label in show_labels else ''\n",
        "            for idx, label in enumerate(data.index)]\n",
        "        pie = data.plot.pie(y='Images', ax=ax, labels=labels)\n",
        "        ax.get_legend().remove()\n",
        "        if save_path:\n",
        "            fig_name = 'fig1_class_distribution_pie'\n",
        "            stats.to_csv(Path(save_path) / f'{fig_name}.csv')\n",
        "            self._save_fig(Path(save_path) / f'{fig_name}.svg')\n",
        "        return pie\n",
        "    \n",
        "    def plot_pie_per_label_count(self, save_path=None):\n",
        "        \"\"\"Plot a pie chart of training data per label count.\"\"\"\n",
        "        train = (self.calc_train(method='calc_label_per_image', rename=False)\n",
        "                 .rename(columns={'training': 'Labels'}))\n",
        "        hpa = (self.calc_hpa(method='calc_label_per_image', rename=False)\n",
        "               .rename(columns={'HPAv18': 'Labels'}))\n",
        "        data = train.add(hpa, fill_value=0)\n",
        "        data = data.sort_values('Labels', ascending=False)\n",
        "        stats = data\n",
        "        print(stats)\n",
        "        fig, ax = self._set_plot_defaults()\n",
        "        show_labels = (1, 2, 3, 4)\n",
        "        labels = [\n",
        "            label if label in show_labels else ''\n",
        "            for idx, label in enumerate(data.index)]\n",
        "        pie = data.plot.pie(y='Labels', ax=ax, labels=labels)\n",
        "        ax.get_legend().remove()\n",
        "        if save_path:\n",
        "            fig_name = 'fig1_label_distribution_pie'\n",
        "            stats.to_csv(Path(save_path) / f'{fig_name}.csv')\n",
        "            self._save_fig(Path(save_path) / f'{fig_name}.svg')\n",
        "        return pie\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHZuAHW2Bfwu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count = Count.create(\n",
        "    HPA_PATH, TRUTH_PATH, TRAIN_PATH, MASTER_TEST_PATH, MASTER_TRAIN_PATH)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_UzvBGaRinz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count.plot_pie_per_label_count(save_path=Path(NOTEBOOK_DIR) / 'figures')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKdLASrl5ELX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count.plot_pie_per_class(save_path=Path(NOTEBOOK_DIR) / 'figures')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJqIWzzgHMLJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count.plot_fig2a_class_distribution(save_path=Path(NOTEBOOK_DIR) / 'figures')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVYIc08ZXPaf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count.calc_hpa()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}