notebooks/2_PII_EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fake PII data: Exploratory data analysis\n",
    "\n",
    "This notebook is used to verify the different fake entities before and after the creation of a synthetic dataset / augmented dataset. First part looks at the generation details and stats, second part evaluates the created synthetic dataset after it has been generated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install matplotlib\n",
    "#!pip install wordcloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from presidio_evaluator import InputSample\n",
    "from presidio_evaluator.data_generator import PresidioDataGenerator\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Evaluate generation logic and the fake PII bank used during generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pii_df = pd.read_csv(\n",
    "    \"../presidio_evaluator/data_generator/raw_data/FakeNameGenerator.com_3000.csv\",\n",
    "    encoding=\"utf-8\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PresidioDataGenerator.update_fake_name_generator_df(pii_df)\n",
    "pii_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (name, series) in pii_df.iteritems():\n",
    "    print(name)\n",
    "    print(\"Unique values: {}\".format(len(series.unique())))\n",
    "    print(series.value_counts())\n",
    "    print(\"\\n**************\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "\n",
    "\n",
    "def series_to_wordcloud(series):\n",
    "    freqs = series.value_counts()\n",
    "    wordcloud = WordCloud(\n",
    "        background_color=\"white\", width=800, height=400\n",
    "    ).generate_from_frequencies(freqs)\n",
    "    fig = plt.figure(figsize=(16, 8))\n",
    "    plt.suptitle(\"{} word cloud\".format(series.name))\n",
    "    plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.first_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.last_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.country_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.company)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.city)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Evaluate different entities in the synthetic dataset after creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "synth = InputSample.read_dataset_json(\"../data/synth_dataset_v2.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity_values_from_sample(sample, entity_types):\n",
    "    name_entities = [\n",
    "        span.entity_value for span in sample.spans if span.entity_type in entity_types\n",
    "    ]\n",
    "    return name_entities\n",
    "\n",
    "\n",
    "names = [\n",
    "    get_entity_values_from_sample(sample, [\"PERSON\", \"FIRST_NAME\", \"LAST_NAME\"])\n",
    "    for sample in synth\n",
    "]\n",
    "names = [item for sublist in names for item in sublist]\n",
    "series_to_wordcloud(pd.Series(names, name=\"PERSON, FIRST_NAME, LAST_NAME\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries = [get_entity_values_from_sample(sample, [\"LOCATION\"]) for sample in synth]\n",
    "countries = [item for sublist in countries for item in sublist]\n",
    "series_to_wordcloud(pd.Series(countries, name=\"LOCATION\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "orgs = [get_entity_values_from_sample(sample, [\"ORGANIZATION\"]) for sample in synth]\n",
    "orgs = [item for sublist in orgs for item in sublist]\n",
    "series_to_wordcloud(pd.Series(orgs, name=\"ORGANIZATION\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio",
   "language": "python",
   "name": "presidio"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}