3_ulmfit_lm_finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = 'de'\n",
    "backwards = False\n",
    "bs=128\n",
    "vocab_sz = 15000\n",
    "num_workers=18\n",
    "step = 'ft' # fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "direction = 'fwd' if backwards == False else 'bwd'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = f'{lang}wiki'\n",
    "base_path = Path('data')\n",
    "path = base_path/wiki\n",
    "model_base_path = path/'model'\n",
    "model_path = model_base_path/step/direction\n",
    "spm_path = model_base_path/step/'spm'\n",
    "lm_fns = [(model_base_path/'lm'/direction/f'{lang}_wikitext_model').absolute(), \n",
    "          (model_base_path/'lm'/direction/f'{lang}_wikitext_vocab').absolute()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = path/'germeval'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([Path('/data/projects/git/fastai_ulmfit/data/dewiki/model/lm/bwd/de_wikitext_model'),\n",
       "  Path('/data/projects/git/fastai_ulmfit/data/dewiki/model/lm/bwd/de_wikitext_vocab')],\n",
       " Path('data/dewiki/model/ft/spm'),\n",
       " 'bwd',\n",
       " True)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_fns, spm_path, direction, backwards"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare corpus for fine tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare a .csv-file with a 'text' column to use this notebook for the language model fine-tuning. Otherwise you'll have to adjust the DataBlock / Dataloaders creation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_path/'data_lm.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Meine Mutter hat mir erzählt, dass mein Vater einen Wahlkreiskandidaten nicht gewählt hat, weil der gegen die Homo-Ehe ist ☺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Meine Reaktion; Nicht jeder Moslem ist ein Terrorist. Aber jeder Moslem glaubt an Überlieferungen, die Gewalt und Terror begünstigen.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>#Merkel rollt dem Emir von #Katar, der islamistischen Terror unterstützt, den roten Teppich aus.Wir brauchen einen sofortigen #Waffenstopp!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>„Merle ist kein junges unschuldiges Mädchen“ Kch....... 😱 #tatort</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Asylantenflut bringt eben nur negatives für Deutschland. Drum Asylanenstop und Rückführung der Mehrzahl.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512531</th>\n",
       "      <td>🇪🇺 Keine Lust auf #Europa? Zurück zu nationalen Grenzen? Drei junge #Europäer wollten dabei nicht länger zusehen. Und gründeten #Volt. Eine Partei, die Europa einen soll. ⏩ #Europawahl2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512532</th>\n",
       "      <td>Morgen um 11 Uhr geht die große Sprayer-Aktion an der Bahnunterführung in #Langen los. Wir sind total gespannt auf die Kunstwerke, die dort entstehen und verabschieden uns schon mal von den alten Schmierereien. Mehr Infos: #Graffiti #StreetArt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512533</th>\n",
       "      <td>Neuigkeit: Gesundheitsgefahr: Krebshilfe fordert Werbeverbot für E-Zigaretten #news</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512534</th>\n",
       "      <td>Hundeanhänger 925 Silber und recyceltes Holz sabina #Tchotchke #kreativerTchotchke #handgefertigt Cane ciondolo in argento 925 e sabina legno riciclato #gioielli #gioiellicreativi #fattoamano</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512535</th>\n",
       "      <td>Nein! Nichts hat was mit Irgendwas zu tun! Zu den Attentaten in Christchurch und SriLanka hören wir bei #maybritillner von friedliebenden Radikal-Muslimen - bis Wolfgang Bosbach die Blutgrätsche macht.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>512536 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                       text\n",
       "0                                                                                                                              Meine Mutter hat mir erzählt, dass mein Vater einen Wahlkreiskandidaten nicht gewählt hat, weil der gegen die Homo-Ehe ist ☺\n",
       "1                                                                                                                     Meine Reaktion; Nicht jeder Moslem ist ein Terrorist. Aber jeder Moslem glaubt an Überlieferungen, die Gewalt und Terror begünstigen.\n",
       "2                                                                                                               #Merkel rollt dem Emir von #Katar, der islamistischen Terror unterstützt, den roten Teppich aus.Wir brauchen einen sofortigen #Waffenstopp!\n",
       "3                                                                                                                                                                                         „Merle ist kein junges unschuldiges Mädchen“ Kch....... 😱 #tatort\n",
       "4                                                                                                                                                  Asylantenflut bringt eben nur negatives für Deutschland. Drum Asylanenstop und Rückführung der Mehrzahl.\n",
       "...                                                                                                                                                                                                                                                     ...\n",
       "512531                                                         🇪🇺 Keine Lust auf #Europa? Zurück zu nationalen Grenzen? Drei junge #Europäer wollten dabei nicht länger zusehen. Und gründeten #Volt. Eine Partei, die Europa einen soll. ⏩ #Europawahl2019\n",
       "512532  Morgen um 11 Uhr geht die große Sprayer-Aktion an der Bahnunterführung in #Langen los. Wir sind total gespannt auf die Kunstwerke, die dort entstehen und verabschieden uns schon mal von den alten Schmierereien. Mehr Infos: #Graffiti #StreetArt\n",
       "512533                                                                                                                                                                  Neuigkeit: Gesundheitsgefahr: Krebshilfe fordert Werbeverbot für E-Zigaretten #news\n",
       "512534                                                      Hundeanhänger 925 Silber und recyceltes Holz sabina #Tchotchke #kreativerTchotchke #handgefertigt Cane ciondolo in argento 925 e sabina legno riciclato #gioielli #gioiellicreativi #fattoamano\n",
       "512535                                            Nein! Nichts hat was mit Irgendwas zu tun! Zu den Attentaten in Christchurch und SriLanka hören wir bei #maybritillner von friedliebenden Radikal-Muslimen - bis Wolfgang Bosbach die Blutgrätsche macht.\n",
       "\n",
       "[512536 rows x 1 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine tune model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We **don't reuse** the SentencePiece model trained on the Wikipedia corpus during fine-tuning. That might lead to slightly different tokenization, but fast.ai's language model learner takes care about adjusting the old / new vocabs. The language model fine-tuning will learn meaningful embeddings for the new tokens.\n",
    "\n",
    "For the **forward / backard model** we'll use the same SentencePiece model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SentencePiece Model exists - using data/dewiki/model/ft/spm\n"
     ]
    }
   ],
   "source": [
    "if spm_path.exists():\n",
    "    tok = SentencePieceTokenizer(lang=lang, max_vocab_sz=vocab_sz, sp_model=spm_path/'spm.model')\n",
    "    print(f'SentencePiece Model exists - using {spm_path}')\n",
    "else:\n",
    "    tok = SentencePieceTokenizer(lang=lang, max_vocab_sz=vocab_sz, cache_dir=spm_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/florian/miniconda3/envs/fastai/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "dblocks = DataBlock(blocks=(TextBlock.from_df('text', tok=tok, is_lm=True, backwards=backwards)),\n",
    "                    get_x=ColReader('text'), \n",
    "                    splitter=RandomSplitter(valid_pct=0.1, seed=42))\n",
    "dls = dblocks.dataloaders(df, bs=bs, num_workers=num_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id str a ▁help ▁# mix my ▁match ▁# : ▁hier ▁dazu ▁infos ▁xxmaj ▁mehr ▁xxmaj ). ▁ländern ▁xxmaj ischen n afrika ▁west ▁anderen ▁aus ▁auch ▁aber ne ger ▁( nigeria ▁ ▁xxmaj ▁aus ▁einem ▁und ▁europa ▁xxmaj ▁aus teil ▁eltern ▁xxmaj ▁einem ▁je ▁mit erweise ▁ideal ▁xxmaj ! in * r spende n zelle stamm ▁ ▁xxmaj r * deutsche ro af ▁ e * ▁ein ▁wird ▁gesucht ▁xxmaj ️ !!</td>\n",
       "      <td>str a ▁help ▁# mix my ▁match ▁# : ▁hier ▁dazu ▁infos ▁xxmaj ▁mehr ▁xxmaj ). ▁ländern ▁xxmaj ischen n afrika ▁west ▁anderen ▁aus ▁auch ▁aber ne ger ▁( nigeria ▁ ▁xxmaj ▁aus ▁einem ▁und ▁europa ▁xxmaj ▁aus teil ▁eltern ▁xxmaj ▁einem ▁je ▁mit erweise ▁ideal ▁xxmaj ! in * r spende n zelle stamm ▁ ▁xxmaj r * deutsche ro af ▁ e * ▁ein ▁wird ▁gesucht ▁xxmaj ️ !! ▁</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>▁xxmaj , kommen ▁an ▁xxmaj : ▁ich ▁finde ▁wichtig ▁xxmaj . ▁unnötig ▁für ▁ich ▁halte ziel ▁zeit ▁xxmaj ▁ein ▁durch ▁druck ▁xxmaj er ▁zusätzlich ▁xxmaj . ▁erfahrungen ▁xxmaj ▁und ▁eindrücke ▁xxmaj ▁neue ▁viele ▁so ▁es ▁gibt kampf ett ▁w ▁xxmaj ▁ersten ▁beim ▁gerade ▁xxmaj ▁xxbos ▁fort ▁griechenland ▁xxmaj ▁in a . ▁u ▁sich e ▁setzt ▁und ▁an ▁araber ▁xxmaj ▁die ▁durch ung ober er ▁ ▁xxmaj ▁der ▁nach tern p gy ä</td>\n",
       "      <td>, kommen ▁an ▁xxmaj : ▁ich ▁finde ▁wichtig ▁xxmaj . ▁unnötig ▁für ▁ich ▁halte ziel ▁zeit ▁xxmaj ▁ein ▁durch ▁druck ▁xxmaj er ▁zusätzlich ▁xxmaj . ▁erfahrungen ▁xxmaj ▁und ▁eindrücke ▁xxmaj ▁neue ▁viele ▁so ▁es ▁gibt kampf ett ▁w ▁xxmaj ▁ersten ▁beim ▁gerade ▁xxmaj ▁xxbos ▁fort ▁griechenland ▁xxmaj ▁in a . ▁u ▁sich e ▁setzt ▁und ▁an ▁araber ▁xxmaj ▁die ▁durch ung ober er ▁ ▁xxmaj ▁der ▁nach tern p gy ä ▁</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>▁schon ▁geht ▁xxmaj . gs son ▁lieblings ▁xxmaj ▁disney ▁xxmaj ▁meiner ▁einer ▁ist ▁world ▁new ▁whole ▁a ▁xxbos ▁stellen es ▁fall ▁xxmaj ▁des en ation lik p ▁im ▁xxmaj rechtlichen ▁menschen ▁den ▁aber ▁sich ▁muss r m eg ▁ ▁xxup ▁der ▁wurde en tritt s ▁be a vis ▁ ▁xxup humanitäre ▁ ▁für ständigkeit ▁zu ▁xxmaj ▁die erweise ios ▁kur ▁indem , gezogen ▁ affäre ▁ ▁xxmaj ▁der ▁aus ▁2017 ▁sich ▁hat</td>\n",
       "      <td>▁geht ▁xxmaj . gs son ▁lieblings ▁xxmaj ▁disney ▁xxmaj ▁meiner ▁einer ▁ist ▁world ▁new ▁whole ▁a ▁xxbos ▁stellen es ▁fall ▁xxmaj ▁des en ation lik p ▁im ▁xxmaj rechtlichen ▁menschen ▁den ▁aber ▁sich ▁muss r m eg ▁ ▁xxup ▁der ▁wurde en tritt s ▁be a vis ▁ ▁xxup humanitäre ▁ ▁für ständigkeit ▁zu ▁xxmaj ▁die erweise ios ▁kur ▁indem , gezogen ▁ affäre ▁ ▁xxmaj ▁der ▁aus ▁2017 ▁sich ▁hat ugh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>▁anderer ▁mit ▁menschen ▁xxmaj ▁wenn ▁xxmaj . ▁klar ▁is , ▁ne ▁ja ▁xxmaj . ▁genauso ▁leuten ▁xxmaj ▁vielen ▁bei ▁ist ▁das ▁und ▁xxmaj . ▁so ▁ist ▁das ▁xxmaj . ▁schlecht ▁oder ▁gut ▁nicht ▁ist ▁das ▁xxmaj . ▁weiße ▁xxmaj ▁prozent ▁xxmaj ▁90 ▁über ▁zu ▁sitzen ▁bahn ▁xxmaj ▁meiner ▁in ▁xxmaj ▁xxbos rbl ▁hsv ▁xxup ▁# ▁dfbpokal ▁# ▁müssen ▁zu ▁machen ticket platz ▁steh ▁xxmaj ▁sein ▁um ▁sorgen ▁xxmaj ▁keine ▁sich ▁und ▁sein</td>\n",
       "      <td>▁mit ▁menschen ▁xxmaj ▁wenn ▁xxmaj . ▁klar ▁is , ▁ne ▁ja ▁xxmaj . ▁genauso ▁leuten ▁xxmaj ▁vielen ▁bei ▁ist ▁das ▁und ▁xxmaj . ▁so ▁ist ▁das ▁xxmaj . ▁schlecht ▁oder ▁gut ▁nicht ▁ist ▁das ▁xxmaj . ▁weiße ▁xxmaj ▁prozent ▁xxmaj ▁90 ▁über ▁zu ▁sitzen ▁bahn ▁xxmaj ▁meiner ▁in ▁xxmaj ▁xxbos rbl ▁hsv ▁xxup ▁# ▁dfbpokal ▁# ▁müssen ▁zu ▁machen ticket platz ▁steh ▁xxmaj ▁sein ▁um ▁sorgen ▁xxmaj ▁keine ▁sich ▁und ▁sein ▁zu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>▁xxup ▁wieso ▁xxup ▁xxbos ▁... ▁zeitung ▁xxmaj ▁keine ▁auch ▁vermutlich ▁lesen ▁die ▁aber ▁xxmaj . ▁nötig ▁bitter s tipp s ▁alltag ▁xxmaj ▁solche ▁hätten menschen ▁mit ▁xxmaj ▁viele ▁xxmaj : räumen ▁ein ▁man ▁muss ▁andererseits ▁xxmaj ▁xxbos monster zel t ▁kri ▁xxmaj ▁# . t leuchte ▁ ▁zunge ▁xxmaj ▁zu ▁schön ▁so ▁immer ▁danach ▁weil ▁xxmaj . cken mü os mo ▁leucht ▁xxmaj ▁liebsten ▁am st ▁is ▁und ge bir ▁ge ▁xxmaj</td>\n",
       "      <td>▁wieso ▁xxup ▁xxbos ▁... ▁zeitung ▁xxmaj ▁keine ▁auch ▁vermutlich ▁lesen ▁die ▁aber ▁xxmaj . ▁nötig ▁bitter s tipp s ▁alltag ▁xxmaj ▁solche ▁hätten menschen ▁mit ▁xxmaj ▁viele ▁xxmaj : räumen ▁ein ▁man ▁muss ▁andererseits ▁xxmaj ▁xxbos monster zel t ▁kri ▁xxmaj ▁# . t leuchte ▁ ▁zunge ▁xxmaj ▁zu ▁schön ▁so ▁immer ▁danach ▁weil ▁xxmaj . cken mü os mo ▁leucht ▁xxmaj ▁liebsten ▁am st ▁is ▁und ge bir ▁ge ▁xxmaj ▁im</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>s ziehung er ▁ ▁ohne zeit . z er ▁ . ▁wg ▁nicht ▁frauen ▁xxmaj ▁recht ▁erst , en ▁bild ▁nicht ▁xxup ell i finanz ) ▁b lich zeit ) ▁a rente ▁ ▁xxmaj ▁für . gen ▁ ▁xxmaj ▁meine ▁konnte ▁/ ▁kann lagen ▁rück ▁xxmaj . l er ford er ) ▁3 ▁xxbos ▁de , ▁berlin ▁xxmaj - ▁ 27 - ▁24 straße ▁ritter ▁xxmaj - ▁ berg kreuz - ▁berlin</td>\n",
       "      <td>ziehung er ▁ ▁ohne zeit . z er ▁ . ▁wg ▁nicht ▁frauen ▁xxmaj ▁recht ▁erst , en ▁bild ▁nicht ▁xxup ell i finanz ) ▁b lich zeit ) ▁a rente ▁ ▁xxmaj ▁für . gen ▁ ▁xxmaj ▁meine ▁konnte ▁/ ▁kann lagen ▁rück ▁xxmaj . l er ford er ) ▁3 ▁xxbos ▁de , ▁berlin ▁xxmaj - ▁ 27 - ▁24 straße ▁ritter ▁xxmaj - ▁ berg kreuz - ▁berlin space</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>▁köln ▁xxmaj ▁düsseldorf ▁xxmaj ▁dortmund ▁xxmaj ▁bielefeld ▁xxmaj ▁hannover ▁xxmaj ▁bremen ▁xxmaj ▁dresden ▁xxmaj ▁leipzig ▁xxmaj ▁nürnberg ▁xxmaj ▁regensburg ▁xxmaj : ▁melden ▁gerne ▁sich ▁können ▁da ▁von ▁spieler ▁xxmaj e ▁aktiv ▁städten ▁xxmaj n ▁folgende ▁in ions connect ▁ ▁xxmaj ▁suche ▁ich da ▁eu ▁xxbos ch f ▁fcb ▁xxup ▁# ? ▁werden ▁sitzen ▁bank ▁xxmaj ▁der ▁auf ▁bayern ▁xxmaj ▁bei ▁saison ▁xxmaj ▁nächsten ▁der ▁in zel t gla ▁ ▁xxmaj ▁und er</td>\n",
       "      <td>▁xxmaj ▁düsseldorf ▁xxmaj ▁dortmund ▁xxmaj ▁bielefeld ▁xxmaj ▁hannover ▁xxmaj ▁bremen ▁xxmaj ▁dresden ▁xxmaj ▁leipzig ▁xxmaj ▁nürnberg ▁xxmaj ▁regensburg ▁xxmaj : ▁melden ▁gerne ▁sich ▁können ▁da ▁von ▁spieler ▁xxmaj e ▁aktiv ▁städten ▁xxmaj n ▁folgende ▁in ions connect ▁ ▁xxmaj ▁suche ▁ich da ▁eu ▁xxbos ch f ▁fcb ▁xxup ▁# ? ▁werden ▁sitzen ▁bank ▁xxmaj ▁der ▁auf ▁bayern ▁xxmaj ▁bei ▁saison ▁xxmaj ▁nächsten ▁der ▁in zel t gla ▁ ▁xxmaj ▁und er atter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>▁fordert ling ster ▁ ▁xxmaj em he ra ▁ ▁xxmaj spieler ▁national ▁xxmaj s ▁england ▁xxmaj ▁xxbos ▁... ▁eigentlich ▁aber , elt äch bel ▁ ▁nett treffen s vorstand - ▁spd ▁nächsten ▁im ▁wieder ▁nicht ▁und ▁werden ▁zu ▁gehört ▁geben ▁gefühl ▁xxmaj ▁das ▁generation ▁xxmaj ▁unserer ▁würde ung hör ▁an ▁xxmaj ▁eine ▁xxmaj . ▁generation ▁xxmaj ▁meiner ▁viele ▁für ▁vorbild ▁xxmaj ▁ein ▁aber , abgeordnete ▁ ▁xxmaj ▁keine ▁ist ▁sie , ▁nein</td>\n",
       "      <td>ling ster ▁ ▁xxmaj em he ra ▁ ▁xxmaj spieler ▁national ▁xxmaj s ▁england ▁xxmaj ▁xxbos ▁... ▁eigentlich ▁aber , elt äch bel ▁ ▁nett treffen s vorstand - ▁spd ▁nächsten ▁im ▁wieder ▁nicht ▁und ▁werden ▁zu ▁gehört ▁geben ▁gefühl ▁xxmaj ▁das ▁generation ▁xxmaj ▁unserer ▁würde ung hör ▁an ▁xxmaj ▁eine ▁xxmaj . ▁generation ▁xxmaj ▁meiner ▁viele ▁für ▁vorbild ▁xxmaj ▁ein ▁aber , abgeordnete ▁ ▁xxmaj ▁keine ▁ist ▁sie , ▁nein ▁xxmaj</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>▁min ▁xxup ▁# s pier m ▁po ▁# s our sec de s ▁service ▁# dis cg ▁ ▁xxup ▁# ▁sinn et ▁ ▁soll ou es - ▁ n anne ab ▁mat e un io reflex ’ d ▁ ▁an ▁aktiv rain ter ▁ ▁xxmaj um ▁v t lei ▁ ▁xxmaj 👍 ! n hun ▁ schafft ge ▁mat gruppe s echt b ar ▁a ▁xxmaj ▁den ▁an ▁intensiv ▁+ ach ma</td>\n",
       "      <td>▁xxup ▁# s pier m ▁po ▁# s our sec de s ▁service ▁# dis cg ▁ ▁xxup ▁# ▁sinn et ▁ ▁soll ou es - ▁ n anne ab ▁mat e un io reflex ’ d ▁ ▁an ▁aktiv rain ter ▁ ▁xxmaj um ▁v t lei ▁ ▁xxmaj 👍 ! n hun ▁ schafft ge ▁mat gruppe s echt b ar ▁a ▁xxmaj ▁den ▁an ▁intensiv ▁+ ach ma ge</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(dls, AWD_LSTM, drop_mult=0.5, pretrained=True, pretrained_fnames=lm_fns, \n",
    "                               metrics=[accuracy, Perplexity()]).to_fp16()\n",
    "learn.path = model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=0.03019951581954956, lr_steep=0.04786301031708717)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkzUlEQVR4nO3de5RcZZnv8e9TVX2/pZN0bp2EEEACBEJCgADCoIByh0FxQB2F48gw49FRj3NGl0uZmXVm6cxR54jOyDBeUEdRYZBBBQZvQBS5dCCBhATIjSSddNKddHf1rbrr8pw/andomk7SIdlVu7t+n7Vqddfeu2r/ugn19Pu+e7+vuTsiIlK6YsUOICIixaVCICJS4lQIRERKnAqBiEiJUyEQESlxKgQiIiUuUewAh2v69Om+YMGCYscQEZlQVq1a1eHuTWPtm3CFYMGCBbS0tBQ7hojIhGJmrx5on7qGRERKnAqBiEiJUyEQESlxKgQiIiUu1EJgZn9lZmvNbJ2ZfXyM/WZmt5vZRjN73syWhZlHRETeKLRCYGaLgQ8DZwFLgCvN7PhRh10GnBA8bgG+EVYeEREZW5gtgpOAp9y9390zwGPAdaOOuQb4nuc9CUwxs9khZhIRmZB++eJuNu7pCeW9wywEa4HzzWyamVUDlwPzRh3TDGwf8XxHsO11zOwWM2sxs5b29vbQAouIRJG785c/WMV/PtsayvuHVgjcfT3wj8AjwMPAaiD7Jt/rTndf7u7Lm5rGvDFORGTSGkhnSWed+sqyUN4/1MFid/+Wu5/h7hcAncDLow5p5fWthLnBNhERCSQHMgA0VE3AQmBmM4Kv88mPD/xw1CEPAB8Irh5aAXS7+64wM4mITDTdA2kgvEIQ9lxD/2lm04A08BF37zKzWwHc/Q7gQfJjBxuBfuDmkPOIiEw4E7oQuPv5Y2y7Y8T3DnwkzAwiIhNdMigE9VXhfGTrzmIRkYgLu0WgQiAiEnEqBCIiJW64ENRNxMtHRUTkyCVTaeoqEsRjFsr7qxCIiERc90Ca+pC6hUCFQEQk8pIqBCIipS05kKEhpEtHQYVARCTyugfSoV0xBCoEIiKR1z2QDm3COVAhEBGJvGRKLQIRkZKVzuboH8qqEIiIlKru/fMMqRCIiJSkZMjTS4AKgYhIpIU9zxCoEIiIRFp3yFNQgwqBiEikJVPhLlMJKgQiIpGmwWIRkRK3f3Uy3VAmIlKakgNpKhIxKsvioZ1DhUBEJMLCnmcIVAhERCIt7LUIQIVARCTSwp5nCFQIREQiTV1DIiIlLj8FdXg3k4EKgYhIpOVXJ5vALQIz+4SZrTOztWZ2t5lVjtp/k5m1m9nq4PFnYeYREZlIcjmf2GMEZtYMfAxY7u6LgThwwxiH/tjdTw8e3wwrj4jIRNMzmME93LuKIfyuoQRQZWYJoBrYGfL5REQmjWQBppeAEAuBu7cCXwK2AbuAbnd/ZIxD32Vmz5vZvWY2b6z3MrNbzKzFzFra29vDiiwiEimFmIIawu0aagSuAY4F5gA1Zvb+UYf9DFjg7qcBvwS+O9Z7ufud7r7c3Zc3NTWFFVlEJFIKMc8QhNs1dDGwxd3b3T0N3AecO/IAd9/r7oPB028CZ4SYR0RkQpnwLQLyXUIrzKzazAy4CFg/8gAzmz3i6dWj94uIlLJkKigE1eEWgtDuUnD3p8zsXuBZIAM8B9xpZn8PtLj7A8DHzOzqYP8+4Kaw8oiITDT71yII+YayUN/d3W8Dbhu1+fMj9n8G+EyYGUREJqrugTTxmFFboTuLRURKUnIgQ31lgnzvenhUCEREIqoQU1CDCoGISGQVYuZRUCEQEYmsQswzBCoEIiKRpa4hEZESlxxIh35XMagQiIhEkrsXZC0CUCEQEYmkVDrHUDanQiAiUqr231VcFe7NZKBCICISSfvnGVKLQESkNHX1qxCIiJS0fX1DADRWl4d+LhUCEZEI6urPF4KpNSoEIiIlaV+/WgQiIiWtqz9NZVmMqvJ46OdSIRARiaB9fUNMLUBrAFQIREQiqbNviCkqBCIipauzf6ggA8WgQiAiEkmd/WmmhLxo/TAVAhGRCFKLQESkhGWyOboH0hojEBEpVd0DadxhqrqGRERKU2cwz1CjuoZEREpTZwHvKgYVAhGRyOnsK9w8QxByITCzT5jZOjNba2Z3m1nlqP0VZvZjM9toZk+Z2YIw84iITATDLYIJf/momTUDHwOWu/tiIA7cMOqwDwGd7n488M/AP4aVR0RkotjXlx8jmBQtAiABVJlZAqgGdo7afw3w3eD7e4GLzMxCziQiEmld/UOUJ2JUlYU/4RyEWAjcvRX4ErAN2AV0u/sjow5rBrYHx2eAbmDa6Pcys1vMrMXMWtrb28OKLCISCcMTzhXq7+Iwu4Yayf/FfywwB6gxs/e/mfdy9zvdfbm7L29qajqaMUVEIqezP12wS0ch3K6hi4Et7t7u7mngPuDcUce0AvMAgu6jBmBviJlERCKvs3+IxgINFEO4hWAbsMLMqoN+/4uA9aOOeQD4YPD9u4HfuLuHmElEJPI6+4YmR4vA3Z8iPwD8LPBCcK47zezvzezq4LBvAdPMbCPwSeDTYeUREZkoCt0iSIT55u5+G3DbqM2fH7E/BVwfZgYRkYkkm3O6BtIFW50MdGexiEikJIMJ5yZF15CIiBy+fQWeZwhUCEREImV4niG1CEREStT+KagnyeWjIiJymPa3CNQ1JCJSmvavRaCuIRGR0rSvf4jyeIya8sJMOAcqBCIikdLVl6axpqxgE86BCoGISKTs6x8q6PgAqBCIiERKZ58KgYhISevsH6KxpnCXjoIKgYhIpHT2p9UiEBEpVbmc06UxAhGR0pVMpckVeMI5UCEQEYmMfcFdxVM1RiAiUpqG5xmaoq4hEZHSNDzPUCEXpYFxFgIzqzGzWPD9W8zsajMrbNtFRGSS6yzCWgQw/hbB40ClmTUDjwB/CtwVVigRkVK0b/9aBNEcIzB37weuA/7V3a8HTgkvlohI6WlLpqgpj1NXGdFCYGbnAO8DfhFsK9zUeCIiJaCtO8XMhsqCn3e8heDjwGeAn7r7OjNbCPw2tFQiIiWoLZlidhEKQWI8B7n7Y8BjAMGgcYe7fyzMYCIipWZ3d4oVx00r+HnHe9XQD82s3sxqgLXAi2b21+FGExEpHdmcs7tnsCgtgvF2DZ3s7kngWuAh4FjyVw4dkJmdaGarRzySZvbxUcdcaGbdI475/Jv4GUREJry9vYNkc86s+oh2DQFlwX0D1wJfd/e0mfnBXuDuLwGnA5hZHGgFfjrGoSvd/cpxJxYRmYR2dacAmFmEQjDeFsG/AVuBGuBxMzsGSB7GeS4CNrn7q4cXT0SkNLQl84VgdkNVwc89rkLg7re7e7O7X+55rwJvO4zz3ADcfYB955jZGjN7yMx0b4KIlKTdQSGY2VBR8HOPd7C4wcy+YmYtwePL5FsH43ltOXA1cM8Yu58FjnH3JcDXgPsP8B63DJ+7vb19PKcVEZlQdnWnSMSM6TURLQTAt4Ee4D3BIwl8Z5yvvQx41t13j97h7kl37w2+f5D8WMT0MY67092Xu/vypqamcZ5WRGTi2N2dYmZ9JbGYFfzc4x0sPs7d3zXi+d+Z2epxvvZGDtAtZGazgN3u7mZ2FvnCtHec7ysiMmm0JVPMrC98awDG3yIYMLO3Dj8xs/OAgUO9KLjv4BLgvhHbbjWzW4On7wbWmtka4HbgBnc/6NVIIiKTUVt3qigDxTD+FsGtwPfMrCF43gl88FAvcvc+YNqobXeM+P7rwNfHmUFEZFJyd9qSKS48cUZRzj/eKSbWAEvMrD54Pnxz2PMhZhMRKQk9gxn6h7LMKsIVQ3CYK5QFg7vD9w98MoQ8IiIlpy24mWxWkbqGjmSpysIPbYuITEL7C0ER7iqGIysEGtQVETkKhu8qLlYhOOgYgZn1MPYHvgHFacOIiEwywy2CGUW6fPSghcDd6woVRESkVLUlU0ytKaeyrDgLPx5J15CIiBwFbd2ponULgQqBiEjRtXWnmFWEBWmGqRCIiBTZ7mSqKOsQDFMhEBEposFMlr19Q0VZonKYCoGISBHtSQ4Cxbt0FFQIRESKqm3/gjQqBCIiJWl4rWJ1DYmIlKjdRVy0fpgKgYhIEbUlU1SXx6mvHO+qAEefCoGISBEN30xmVrx5PFUIRESKaOvePuZPqy5qBhUCEZEicXe2dPRx7PSaouZQIRARKZI9PYP0D2VZqEIgIlKaNrf3AXDs9Nqi5lAhEBEpki0dQSFoUotARKQkbenopSIRY3YR7yEAFQIRkaIZHiiOxYq7BLwKgYhIkWyOwBVDoEIgIlIUmWyObXv7J3chMLMTzWz1iEfSzD4+6hgzs9vNbKOZPW9my8LKIyISJTs6B8jkPBKFILTJLdz9JeB0ADOLA63AT0cddhlwQvA4G/hG8FVEZFIbvmJoYZGvGILCdQ1dBGxy91dHbb8G+J7nPQlMMbPZBcokIlI0mzuicQ8BFK4Q3ADcPcb2ZmD7iOc7gm0iIpPalo5eGqrKaKwuK3aU8AuBmZUDVwP3HMF73GJmLWbW0t7efvTCiYgUyfClo8WcdXRYIVoElwHPuvvuMfa1AvNGPJ8bbHsdd7/T3Ze7+/KmpqaQYoqIFM6W9r6izzE0rBCF4EbG7hYCeAD4QHD10Aqg2913FSCTiEjRDAxl2dmdisQVQxDiVUMAZlYDXAL8+YhttwK4+x3Ag8DlwEagH7g5zDwiIlGwdW805hgaFmohcPc+YNqobXeM+N6Bj4SZQUQkavZPNheRFoHuLBYRKbDhQrBgmgqBiEhJ2tzex6z6Smoqirdg/UgqBCIiBbalozcy3UKgQiAiUlDuzqb2vsgMFIMKgYhIQbV2DdA9kOak2fXFjrKfCoGISAGtbU0CsHiOCoGISElat7ObeMzUIhARKVVrW7s5vqmWyrJ4saPsp0IgIlJAa3cmOaU5Oq0BUCEQESmYPckU7T2DLJ7TUOwor6NCICJSIOt2BgPFzSoEIiIlaW1rNwAnR+iKIVAhEBEpmLU7u1k4vYbaiEwtMUyFQESkQNa2JiPXGgAVAhGRgujsG6K1ayBy4wOgQiAiUhD7B4ojdsUQqBCIiBTEup35geJT1DVUXBv39BY7goiUqLU7kzRPqaKxprzYUd6gZArBPS3bueSfH+OFHd3FjiIiJWhdazeLI3ZH8bCSKQTvXDyLaTXl/N3P1pFfKllEpDB6Umk2d/RFcnwASqgQ1FeW8al3nEjLq538/PldxY4jIiVkzfZ8T8Spc1UIiu765fM4eXY9X3xoAwND2WLHEZES8eTmvcRjxvIFU4sdZUwlVQjiMeO2q06mtWuAOx/fXOw4IlIinty8l1ObGyJ3R/GwkioEAGcvnMYVp87mjsc28czWfRovEJFQDQxlWbOjixULpxU7ygGVXCEA+PRliyhPxLj+jj9w/j/9li88tJ7t+/qLHUtEJqFnt3WSzjorFkazWwhCLgRmNsXM7jWzDWa23szOGbX/QjPrNrPVwePzYeYZNm9qNSv/5m186folHD+jlm+t3MK1//L7/TMDiogcLVEfH4DwWwRfBR5290XAEmD9GMesdPfTg8ffh5xnv/rKMt59xlzuuvksHvnEBVSWxbnxzid5esu+QkUQkRIQ9fEBgNCSmVkDcAFwE4C7DwFDYZ3vSCxsquWeW8/h/d96ig98+yk+e8XJNNVWEDOorUiwdH4jVeX59UXdnWe2dvKjZ7aRCKr8mQumsmBaNWZW5J9ERKJkYCjL6u1dfOitC4sd5aDCLFHHAu3Ad8xsCbAK+Ct37xt13DlmtgbYCXzK3deNfiMzuwW4BWD+/PmhhJ0zpYp7/vwcPvDtp/nc/Wtft688EePsY6eydN4UHnlxNxvaeqirTBAz4yctOwA4fkYtt111Muef0BRKPhGZeIbHB86O8PgAgIV11YyZLQeeBM5z96fM7KtA0t0/N+KYeiDn7r1mdjnwVXc/4WDvu3z5cm9paQklM0A6m2NTey/ZnOMOHb2DrHylg0df2sOm9j5Oml3PB885hmtOb6YiEWNTey9Pb93Hvz++ma17+7ls8Sw+e8VJzG2sDi3jZJBMpdm4p5ft+/rZ2ZViV/cAA0NZaioSVJXHKYvHGMxkGUznGMzkcHdy7uQcKsti1FaU7S/GQ5kcQ9ks7lBTkaCmPE51RYKKRCx4xGmsKWdmfQXTaysoi5fkNRJSBF9+5CX+9dFNrP78JdRVlhU1i5mtcvflY+4LsRDMAp509wXB8/OBT7v7FQd5zVZgubt3HOiYsAvBwXQPpKmvTIzZBZRKZ/nmys18/bcbSaVzNFaX0dxYxdwp1SydP4VzjpvGKXMaiMeOrPvI3ensT9PWnWIgnWFqTQXTa8uprcjnyuWcTM4pi9uYObM5Z2/vIHt6BunsH8IwYjEoi8eYUVfB7IYqyhOH/0Hp7vQMZmjvGdz/2NMzyJ5kin19Q6QyOQbTWXoHM2zp6GNXd+p1r6+vTFBdnqB/KEP/UJZMzikf8UEej0HMDANSmRw9qTTp7Gv/dmMGZkY2d/B/z2ZQU54vEpVlceoqE8yZUkXzlCqm11aQymTpG8xnqK8sY0Z9BU21FdRUJEjEjHjcaKwuZ9GsOirL4of9e5LScv0dTzCUdf7rI+cVO8pBC0FoXUPu3mZm283sRHd/CbgIeHFUsFnAbnd3MzuL/OD13rAyHamGqgNX9MqyOP/z7Sdw7dJmfrZmFzs6+2ntGmBDW5KH17UBUFeZ4IxjGlk2v5Gl86ewdH7juAaQtu/r5xcv7OLBF3axYVcPQ9ncG46Jx4yc51sxABWJGNNqyplaW45hJFNpkgNpugfSHOyzMmYwq76SxppyqsriVJXHcc8Xwa6BIXpSGdKZHOmck8nmGH6rA/09UR6PMbWmnKryOBWJGFXlcVYsnMYJM2s5YUYdx0yrZnZD5Rv+WnL3Q465pNL5VkBZ3EjEY7g7g5kc/UP5D/PBTI6hTI5UJsu+3qF8YepJ0ZPKkEpnGczk6OpPs7NrgFWvdtI9kKYsbvlWSVmc5ECavgPcgZ6IGYtm13F8Uy1D2Ry9g1n6BzM4YOSLVnVFnIaqMqZUldFQVUZ9VRn1lWVMCf5ImDe1mvoi/5Uo4RkeH/gfbz222FEOKexh7I8CPzCzcmAzcLOZ3Qrg7ncA7wb+wswywABwg0/wO7zmNlbzFxce97ptu5Mpnty8lz9s2suz2zp57OV23PMfkiuOm8YlJ81g2TGNDAxl6R5Is69viG37+tm6t59Xdvewoa0HgCVzG7j5vAXMaqhkZn0lVeVxOvuG6OgdpKs/TTxmJGIx4jHoSWXo6B1iX98gAAubavZ/CM2oq6CprpJptfnpcDNZJ53N0ZZMsaNzgB2d/XT3pxkI/oIHmF5bzvEzaqmrTFAWj5GIG2WxGCMbOHWVZTTVVex/zKiroKGq7E0Noo/nNaP/IjczKsviVJbFmfompvrNZHMkRnUb9Q1m2NMzyMBQlmzOyeRy7E6mWLOjmxd2dPPM1k6qyuP7u6RiZkEXlrOvb4jN7X10D6RJptJjFsu6igQVZXHK4kZZPEZlWYyq8qB7qzxBXWWC2or817rKMuqrEjRUlTGrvpI5U6qYUVfxhswSDa/dPxDdG8mGhdY1FJZidg0dLclUmtXbulj5Sju/Wr+HLR2jx8/zf5nPbaxmwfQazj0ufzf0vKkad5iocjmndyhDd3++0Ld25Qvuzq4Ug5kcmWyOTM5JpbP0BS2avsEMvcHXZCozZrdXPGbMa6zi+Bm1HDejlpNn13PGMY00T6nSVWxF9rcPrOOHT2/j2c9dEolLR4syRhCWyVAIRtvU3sv6XUnqKvNdCI3VZW+6r14mJ3dnIJ0lOZCha2CIXd0pdnWlaO3qZ0tHHxv39LK1o39/t+HM+gqWzX+tG3Jxc4PGNAook82x4gu/4cwFjXzj/WcUOw5QpDECGb/jmmo5rqm22DEkwsyM6vL8gPqshkoWzXrjAieZbI6Xdvfw7KudrHq1k1XbOnlobX58KhEzTphZx+I59SxubmDZ/EZOml2nbqWQ/GHzXjp6B7nm9DnFjjIuKgQik0QiHuOUOQ2cMqeBPz1nAQDtPYM8t62T1du7WLszya837OGeVfl7X2orEiw7ppGzFjSy7JhGTp83hepyfSQcDQ+s3kldRYILT5xR7Cjjov/qIpNYU10F7zhlFu84ZRaQ72La2Z2iZes+ntm6j2e2dPKlR14G8uMNi5sbuGjRDC4+aSYnza7TOMObkEpneXhtG+9cPGvCdMepEIiUEDOjeUoVzac3c83pzQB096d5dnsnq7Z28ruNHXzlly/zlV++zOyGyv2XO59xTCOnzKlXV9I4PPpSOz2DGa5eMjG6hUCFQKTkNVSX8bYTZ/C2E2fwqXeeyJ6eFL/dsIfHX+5g1YilXRuqynjrCdO58C1NvH3RDKbVVhQ5eTQ9sKaV6bXlnHtc9C8bHaZCICKvM6Oukj85cz5/cmZ+Xq9d3QM8s7WTlS+389jL7fzi+V0kYsYfvaWJP17WzMUnzZwwXSBh60ml+dX6Pbz3rPkTqvWkQiAiBzW7oYqrl1Rx9ZI5uDvrdib52Zqd3L+6lV9v2ENdZYIrT5vNHy+dy5kLGkt6XOGRdbsZyuS4agJ1C4EKgYgcBrP8gPLi5gb+96WL+MOmvdz33A7+a/VO7n56O/OmVnH9GfO4fvlcZjdUFTtuQbk733/yVeZPrWbZ/CnFjnNYdEOZiByxvsEM/72ujXtadvCHzXuJGVzwlibee9Z83r5oxoTqJnmzfrthDzff9QxfuO5UbjwrnOnyj4TuLBaRgnl1bx/3tOzgnlXb2Z0cZHZDJTeeNZ8bzpzHjPrKYscLhbtzzb/8ns7+IX7zvy6M5FTnKgQiUnDpbI5fr9/Nfzy5jd9t7CARMy4+aSbvPXs+bz1+OrEjnJI9Sn714m7+7Hst/NO7T+M9y+cVO86YNMWEiBRcWTzGpYtnc+ni2Wzp6OPup7dxT8t2Hl7XxtzGKt61bC7vWjaX+dMm9mSK7s5Xfvkyx0yr5rqlzcWO86aoRSAiBZNKZ/nvdW38pGU7T2zaizucfexU3rfiGC49ZdaEnGjx4bVt3Pofq/jy9Ut41xlzix3ngNQ1JCKR09o1wP3PtfLjZ7azbV8/02vLec/yeVy1ZA6LZk2M6S1S6SxXfe13ZHPOI5+4INKD4ioEIhJZuZyzcmMH3//Dq/xmw25yDvOmVnHJSbO45OSZLF/QGMnBV8ivOXDXE1v5zk1n8rZF0Z5gTmMEIhJZseAu5T96SxPtPYP8ev1uHnlxN//x1Kt8+/dbqK/Mz+J56eJZvH3RjMjcxfybDbu564mt3HTugsgXgUNRi0BEIqlvMMPKVzr49frd/GbDHvb2DVFfmeCK0+Zw3bJmlh9TvLuY9yRTXPrVlcyoq+D+j5wXmeJ0MGoRiMiEU1OR4NLFs7h08SyyOeeJTR389NlW7n+ulbuf3sb8qdVcu7SZP17azLHTawqWa2Aoyyd/sob+oQxfu3HFhCgCh6IWgYhMKH2DGR5a28ZPn9ux/8qj0+Y2cNVpc7jitNnMmRLe1Ba/3bCHz/3XWnZ0DvDF607lhgjeQXwgGiwWkUlpV/cAD6zeyc+f38ULrd0ALJnbwNsXzeSik2Zwypz6I+4+cnfWtib5xmMbefCFNo6fUcs/XLuYsxdOnGmmQYVARErA1o4+fvHCLn61fjert3fhnl9DYXFzfp3mU5sbWDSrjgXTag55mWdX/xCv7Onl9xs7eGD1TjZ39FGRiPHRtx/PLRccNyHvd1AhEJGS0tE7yKMvtbPq1U5eaO3ipbYe0tn8Z115IsbC6TU0VJVRW5GguiLBUCZL32CW3sEMOzoH6OgdBMAMVhw7jWtOn8Nli2fTUF1WzB/riKgQiEhJG8xkeWV3Ly+19bChLcmWjj6SqQy9qQz9QxnKEzFqKxLUVCSYVV/JW2bWcfzMWk6ZXT9pJsrTVUMiUtIqEvH96yjIG4Xa0WVmU8zsXjPbYGbrzeycUfvNzG43s41m9ryZLQszj4iIvFHYLYKvAg+7+7vNrBwYPc3gZcAJweNs4BvBVxERKZDQWgRm1gBcAHwLwN2H3L1r1GHXAN/zvCeBKWY2O6xMIiLyRmF2DR0LtAPfMbPnzOybZjb69r9mYPuI5zuCba9jZreYWYuZtbS3t4eXWESkBIVZCBLAMuAb7r4U6AM+/WbeyN3vdPfl7r68qanpaGYUESl5YRaCHcAOd38qeH4v+cIwUiswcl23ucE2EREpkNAKgbu3AdvN7MRg00XAi6MOewD4QHD10Aqg2913hZVJRETeKOyrhj4K/CC4YmgzcLOZ3Qrg7ncADwKXAxuBfuDmkPOIiMgoE+7OYjNrB14NnjYA3Qf5fvS2MqDjME858j3Gs2/0tvFmHP46/TAzFirf8Db9DqOVbyJkjHq+I8l4sG1R+x0e4+5jD7K6+4R9AHce7PvR24CWIznHePaN3jbejCO+HlbGQuXT7zCa+SZCxqjnO5KMh8gaqd/hwR4Tbwq91/vZIb4/0P43e47x7Bu9bbwZo57vUOc6GP0OD32egznU66KeMer5DrR/PBkPte1whP07PKAJ1zV0JMysxQ8w6VJURD1j1PNB9DNGPR9EP2PU88HEyDhsorcIDtedxQ4wDlHPGPV8EP2MUc8H0c8Y9XwwMTICJdYiEBGRNyq1FoGIiIyiQiAiUuJUCERESpwKQcDMzjezO4JZUp8odp6xmFnMzP7BzL5mZh8sdp7RzOxCM1sZ/B4vLHaesZhZTTCT7ZXFzjIWMzsp+P3da2Z/Uew8YzGza83s383sx2b2jmLnGc3MFprZt8zs3mJnGRb8u/tu8Ht7X7HzjDYpCoGZfdvM9pjZ2lHbLzWzl4IV0A4686m7r3T3W4GfA9+NYkby6zfMBdLkJ/WLWj4HeoHKiOYD+BvgJ0cz29HM6O7rg3+H7wHOi2jG+939w8CtwJ9EMN9md//Q0cw1lsPMeh1wb/B7uzrsbIftcO58i+qD/AI4y4C1I7bFgU3AQqAcWAOcDJxK/sN+5GPGiNf9BKiLYkby03j/efDaeyOYLxa8bibwgwjmuwS4AbgJuDKK/42D11wNPAS8N6oZg9d9GVgW4XxH9f+RI8z6GeD04JgfhpnrzTwmxeL17v64mS0YtfksYKO7bwYwsx8B17j7F4AxuwXMbD75GVB7opjRzHYAQ8HTbNTyjdAJVEQtX9BdVUP+f8wBM3vQ3XNRyhi8zwPAA2b2C+CHRyvf0cpoZgZ8EXjI3Z+NWr5COZys5FvIc4HVRLAnZlIUggMYa/WzQ62H/CHgO6EleqPDzXgf8DUzOx94PMxggcPKZ2bXAe8EpgBfDzVZ3mHlc/fPApjZTUDH0SwCB3G4v8MLyXcjVJCfnbcQDvff4UeBi4EGMzve8zMJh+lwf4fTgH8AlprZZ4KCUSgHyno78HUzu4I3PwVFaCZzIThs7n5bsTMcjLv3ky9WkeTu95EvVpHm7ncVO8OBuPujwKNFjnFQ7n47+Q+2SHL3veTHLyLD3fuI8DT7kWuiHEUTYfWzqGdUviOnjEcu6vlGmkhZ95vMheAZ4AQzO9byC+PcQH5FtCiJekblO3LKeOSinm+kiZT1NcUerT4aD+BuYBevXVb5oWD75cDL5EfxP6uMyqeM0c4Y9XwTNeuhHpp0TkSkxE3mriERERkHFQIRkRKnQiAiUuJUCERESpwKgYhIiVMhEBEpcSoEMimYWW+Bz3dU1qyw/BoO3Wa22sw2mNmXxvGaa83s5KNxfhFQIRAZk5kddB4udz/3KJ5upbufDiwFrjSzQ61DcC35GVRFjgoVApm0zOw4M3vYzFZZfuW0RcH2q8zsKTN7zsx+ZWYzg+1/a2bfN7PfA98Pnn/bzB41s81m9rER790bfL0w2H9v8Bf9D4JpmjGzy4Ntq8zsdjP7+cHyuvsA+WmKm4PXf9jMnjGzNWb2n2ZWbWbnkl+v4P8GrYjjDvRzioyXCoFMZncCH3X3M4BPAf8abP8dsMLdlwI/Av73iNecDFzs7jcGzxeRn1r7LOA2Mysb4zxLgY8Hr10InGdmlcC/AZcF5286VFgzawRO4LUpxu9z9zPdfQmwnvwUBk+Qn7vmr939dHffdJCfU2RcNA21TEpmVgucC9wT/IEOry2WMxf4sZnNJr+K1JYRL30g+Mt82C/cfRAYNLM95FdfG70M59PuviM472pgAfklOze7+/B73w3ccoC455vZGvJF4P+5e1uwfbGZ/R/y6zvUAv99mD+nyLioEMhkFQO6gr730b4GfMXdHwgWgvnbEfv6Rh07OOL7LGP/PzOeYw5mpbtfaWbHAk+a2U/cfTVwF3Ctu68JFtO5cIzXHuznFBkXdQ3JpOTuSWCLmV0P+eUVzWxJsLuB1+aI/2BIEV4CFo5YyvCQi7wHrYcvAn8TbKoDdgXdUe8bcWhPsO9QP6fIuKgQyGRRbWY7Rjw+Sf7D80NBt8s68mvHQr4FcI+ZrQI6wggTdC/9JfBwcJ4eoHscL70DuCAoIJ8DngJ+D2wYccyPgL8OBruP48A/p8i4aBpqkZCYWa279wZXEf0L8Iq7/3Oxc4mMphaBSHg+HAweryPfHfVvxY0jMja1CERESpxaBCIiJU6FQESkxKkQiIiUOBUCEZESp0IgIlLiVAhERErc/wcqS5QcPDllogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 3e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.274698</td>\n",
       "      <td>4.093458</td>\n",
       "      <td>0.340817</td>\n",
       "      <td>59.946812</td>\n",
       "      <td>02:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, lr, moms=(0.8,0.7,0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.068278</td>\n",
       "      <td>3.932473</td>\n",
       "      <td>0.359489</td>\n",
       "      <td>51.033012</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.014465</td>\n",
       "      <td>3.883495</td>\n",
       "      <td>0.366970</td>\n",
       "      <td>48.593758</td>\n",
       "      <td>02:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.915109</td>\n",
       "      <td>3.793149</td>\n",
       "      <td>0.379014</td>\n",
       "      <td>44.395962</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.824833</td>\n",
       "      <td>3.725914</td>\n",
       "      <td>0.387506</td>\n",
       "      <td>41.509136</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.774954</td>\n",
       "      <td>3.664786</td>\n",
       "      <td>0.394962</td>\n",
       "      <td>39.047775</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.695723</td>\n",
       "      <td>3.600970</td>\n",
       "      <td>0.402666</td>\n",
       "      <td>36.633743</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.624130</td>\n",
       "      <td>3.542982</td>\n",
       "      <td>0.409899</td>\n",
       "      <td>34.569855</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.544222</td>\n",
       "      <td>3.495561</td>\n",
       "      <td>0.416163</td>\n",
       "      <td>32.968765</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.493506</td>\n",
       "      <td>3.467265</td>\n",
       "      <td>0.420178</td>\n",
       "      <td>32.048973</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.435243</td>\n",
       "      <td>3.462677</td>\n",
       "      <td>0.421081</td>\n",
       "      <td>31.902250</td>\n",
       "      <td>02:42</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, slice(lr/100,lr), moms=(0.8,0.7,0.8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving fine tuned model, encoder and vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm_ft_fns = [(model_path/f'{lang}_{step}_model').absolute(), \n",
    "             (model_path/f'{lang}_{step}_vocab.pkl').absolute()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai.text.learner.LMLearner at 0x7f67502beca0>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.to_fp32()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('/data/projects/git/fastai_ulmfit/data/dewiki/model/ft/bwd/de_ft_model.pth')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.save(lm_ft_fns[0], with_opt=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder((model_path/f'{lang}_{step}_encoder').absolute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(lm_ft_fns[1], 'wb') as f:\n",
    "      pickle.dump(learn.dls.vocab, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fastai",
   "language": "python",
   "name": "fastai"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}