Normalizzazione e features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, re, string\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "def pulizia(s):\n",
    "    s = s.replace(\"\\n\", \"\")\n",
    "    s = s.replace(\"\\’\", \" \")\n",
    "    s = s.replace(\"a'\", \"à\")\n",
    "    s = s.replace(\"e'\", \"è\")\n",
    "    s = s.replace(\"i'\", \"ì\")\n",
    "    s = s.replace(\"o'\", \"ò\")\n",
    "    s = s.replace(\"u'\", \"ù\")\n",
    "    s = re.sub(r'(?<=[.,?!\\'])(?=[^\\s])', r' ', s)\n",
    "    s = s.translate(str.maketrans('', '', string.punctuation))\n",
    "    return s\n",
    "\n",
    "def capitalizza(testo):\n",
    "    testo = \" \".join([s.replace(\"\\n\", \"\").capitalize() for s in re.split(r'( *[\\.\\?!][\\'\"\\)\\]]* *)', testo)])\n",
    "    testo = testo.replace(\" . \", \". \")\n",
    "    testo = testo.replace(\" ? \", \"? \")\n",
    "    testo = testo.replace(\" ! \", \"! \")\n",
    "    return testo\n",
    "\n",
    "d = {}\n",
    "d2 = defaultdict(list)\n",
    "\n",
    "for filename in os.listdir('./Corpus/'):\n",
    "    if filename.endswith('.txt'):\n",
    "        with open(os.path.join('./Corpus/', filename)) as f:\n",
    "            \n",
    "            content = f.read()\n",
    "            if content.isupper():\n",
    "                content = capitalizza(content)\n",
    "            normalized = content.lower()\n",
    "            normalized = pulizia(normalized)\n",
    "            \n",
    "            filename = filename.split(\".\")[0]\n",
    "            year, pres = (filename.split(\"-\"))\n",
    "            year = int(year)\n",
    "            \n",
    "            \n",
    "        d[year] = {\"presidente\":pres, \"nb_token\":len(content.split()), \"discorso_norm\":normalized, \"discorso_orig\":content}\n",
    "        d2[pres].append(normalized)\n",
    "        \n",
    "d3 = {}\n",
    "for pres,los in d2.items():\n",
    "    d3[pres] = {\"discorso_norm\":\" \".join(los)}\n",
    "d2 = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "ignored_words = nltk.corpus.stopwords.words('italian')\n",
    "\n",
    "\n",
    "def tokenize_without_sw(diz, ignored_words=ignored_words):\n",
    "    return [w for w in diz[\"discorso_norm\"].split() if w not in ignored_words]\n",
    "\n",
    "for year, diz in d.items():\n",
    "    tokens = tokenize_without_sw(diz)\n",
    "    d[year][\"piu_frequenti\"] = {k:v for k,v in Counter(tokens).most_common(30)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections, functools, operator\n",
    "\n",
    "pri_rep = []\n",
    "sec_rep = []\n",
    "\n",
    "for year in d:\n",
    "    if year<1994:\n",
    "        pri_rep.append(d[year][\"piu_frequenti\"])\n",
    "    else:\n",
    "        sec_rep.append(d[year][\"piu_frequenti\"])\n",
    "\n",
    "priRep = dict(functools.reduce(operator.add, map(collections.Counter, pri_rep)))\n",
    "secRep = dict(functools.reduce(operator.add, map(collections.Counter, sec_rep)))\n",
    "\n",
    "\n",
    "respub = []\n",
    "for i in priRep:respub = []\n",
    "for i in priRep:\n",
    "    if i in secRep:\n",
    "        respub.append([i, priRep[i], secRep[i]])\n",
    "    else:\n",
    "        respub.append([i, priRep[i], 1])\n",
    "    if i in secRep:\n",
    "        respub.append([i, priRep[i], secRep[i]])\n",
    "    else:\n",
    "        respub.append([i, priRep[i], 1])\n",
    "\n",
    "for i in secRep:\n",
    "    if i not in priRep:\n",
    "        respub.append([i, 1, secRep[i]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "header = ['token', \"freq. 1° Repubblica\", \"freq. 2° Repubblica\"]\n",
    "\n",
    "with open('respublica.csv', 'w', encoding='UTF8', newline='') as f:\n",
    "    writer = csv.writer(f)\n",
    "\n",
    "    # write the header\n",
    "    writer.writerow(header)\n",
    "\n",
    "    # write multiple rows\n",
    "    writer.writerows(respub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.metrics.association import BigramAssocMeasures\n",
    "from nltk.collocations import BigramCollocationFinder\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "def multiwords(tokenized_text):      \n",
    "    bigram_measures = nltk.collocations.BigramAssocMeasures()                       \n",
    "        ## oggetto: misure di associazione per bigrammi     \n",
    "    finder2 = BigramCollocationFinder.from_words(tokenized_text, window_size=2)     \n",
    "        ## oggetto: cercare collocazioni in finestre di taglia due     \n",
    "    #ignored_words = nltk.corpus.stopwords.words('italian')                          \n",
    "        ## (se aumenti la finestra cerchi degli skipgrams)\n",
    "    #finder2.apply_word_filter(lambda w:w.lower() in ignored_words)                  \n",
    "        ## applico filtro per collocazioni con stopwords (lower and uppercase)     \n",
    "    scored2 = finder2.score_ngrams(bigram_measures.student_t)                       \n",
    "        ## crea una lista di collocazioni in ordine di score di associazione \n",
    "        ##(raw_freq, student_t, likelihood_ratio, poisson_stirling...)     \n",
    "    #return [(indice, collocazione) for indice, collocazione in enumerate(scored2)]\n",
    "    return {ix:str(v) for ix,v in enumerate(scored2[:30])}\n",
    "\n",
    "#multiwords(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import SnowballStemmer\n",
    "\n",
    "stemmer_snowball = SnowballStemmer('italian')    \n",
    "    \n",
    "for pres, diz in d3.items():\n",
    "    tokens = tokenize_without_sw(diz)\n",
    "    tokens_stem = [stemmer_snowball.stem(w) for w in tokens]\n",
    "    d3[pres][\"piu_frequenti_stem\"] = {k:v for k,v in Counter(tokens_stem).most_common(30)}\n",
    "    d3[pres][\"30collocazioni\"] = multiwords(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('it_core_news_sm')\n",
    "\n",
    "def gulp(n_chars: int, n_words: int, n_sents: int) -> float:\n",
    "    return (300 * n_sents / n_words) - (10 * n_chars / n_words) + 89\n",
    "\n",
    "for year, diz in d.items():\n",
    "    \n",
    "    n_chars = len(diz[\"discorso_orig\"])\n",
    "    speech = nlp(diz['discorso_orig'])\n",
    "    n_words = len(speech)\n",
    "    n_sents = 0\n",
    "    for sent in speech.sents:\n",
    "        n_sents += 1\n",
    "    \n",
    "    tmp = []\n",
    "    \n",
    "    for token in speech:\n",
    "        tmp.append(token.pos_)\n",
    "    \n",
    "    counter = dict(Counter(tmp)).items()\n",
    "    len_per = {k:(v/d[year][\"nb_token\"])*100 for k,v in counter}\n",
    "    d[year][\"PoS_percentage\"] = {k:v for k,v in sorted(len_per.items(),key=lambda i:i[1], reverse=True)}\n",
    "    d[year][\"indice_gulpease\"] = gulp(n_chars, n_words, n_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('corpus_xdata.json', 'w') as f:\n",
    "    json.dump(d, f)\n",
    "    \n",
    "with open('corpus_xpresidente.json', 'w') as f:\n",
    "    json.dump(d3, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# todos\n",
    "\n",
    "# bugfixing generico sulla normalizzazione\n",
    "\n",
    "## lower / capitalize speeches in capslock\n",
    "## trattini e punteggiatura speciale: verificare#\n",
    "## rapp lunghezza, punteggiatura"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my_venv",
   "language": "python",
   "name": "my_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}