notebooks/data_preparation/test/test-final-image-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "358cd4a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:38:55.369880Z",
     "iopub.status.busy": "2022-02-09T16:38:55.368511Z",
     "iopub.status.idle": "2022-02-09T16:39:18.727096Z",
     "shell.execute_reply": "2022-02-09T16:39:18.726380Z",
     "shell.execute_reply.started": "2022-02-09T16:37:16.743554Z"
    },
    "papermill": {
     "duration": 23.382581,
     "end_time": "2022-02-09T16:39:18.727297",
     "exception": false,
     "start_time": "2022-02-09T16:38:55.344716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transliterate\r\n",
      "  Downloading transliterate-1.10.2-py2.py3-none-any.whl (45 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 45 kB 816 kB/s \r\n",
      "\u001b[?25hRequirement already satisfied: six>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from transliterate) (1.15.0)\r\n",
      "Installing collected packages: transliterate\r\n",
      "Successfully installed transliterate-1.10.2\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install rapidfuzz -qq\n",
    "!pip install transliterate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f09037aa",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:18.772858Z",
     "iopub.status.busy": "2022-02-09T16:39:18.771162Z",
     "iopub.status.idle": "2022-02-09T16:39:18.786606Z",
     "shell.execute_reply": "2022-02-09T16:39:18.785989Z",
     "shell.execute_reply.started": "2022-02-09T16:37:39.863759Z"
    },
    "papermill": {
     "duration": 0.040925,
     "end_time": "2022-02-09T16:39:18.786812",
     "exception": false,
     "start_time": "2022-02-09T16:39:18.745887",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from rapidfuzz import fuzz\n",
    "\n",
    "from collections import defaultdict\n",
    "import cloudpickle\n",
    "\n",
    "from dateutil.parser import parse\n",
    "\n",
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "12bca2ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:18.827166Z",
     "iopub.status.busy": "2022-02-09T16:39:18.826196Z",
     "iopub.status.idle": "2022-02-09T16:39:20.263053Z",
     "shell.execute_reply": "2022-02-09T16:39:20.262425Z",
     "shell.execute_reply.started": "2022-02-09T16:37:39.885237Z"
    },
    "papermill": {
     "duration": 1.458255,
     "end_time": "2022-02-09T16:39:20.263254",
     "exception": false,
     "start_time": "2022-02-09T16:39:18.804999",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 895 ms, sys: 132 ms, total: 1.03 s\n",
      "Wall time: 1.43 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "images = pd.read_csv('../input/test-dataset/test-images.csv', keep_default_na=False)\n",
    "images['id'] = images.index\n",
    "\n",
    "part1 = pd.read_csv('../input/test-trans-filename/translated_filenames.csv', keep_default_na=False)\n",
    "part1.rename(columns={'filename_translation': 'undigit_filename_translation'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bf8775b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:20.317374Z",
     "iopub.status.busy": "2022-02-09T16:39:20.310054Z",
     "iopub.status.idle": "2022-02-09T16:39:20.406844Z",
     "shell.execute_reply": "2022-02-09T16:39:20.407354Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.116060Z"
    },
    "papermill": {
     "duration": 0.124145,
     "end_time": "2022-02-09T16:39:20.407563",
     "exception": false,
     "start_time": "2022-02-09T16:39:20.283418",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_url</th>\n",
       "      <th>count</th>\n",
       "      <th>filename</th>\n",
       "      <th>pured_filename</th>\n",
       "      <th>spaced_filename</th>\n",
       "      <th>undigit_filename</th>\n",
       "      <th>undigit_filename_empty</th>\n",
       "      <th>filename_lang</th>\n",
       "      <th>filename_lang_p</th>\n",
       "      <th>filename_en</th>\n",
       "      <th>section</th>\n",
       "      <th>commons</th>\n",
       "      <th>lang_ok</th>\n",
       "      <th>spaced_filename_translit</th>\n",
       "      <th>spaced_filename_translited</th>\n",
       "      <th>ext</th>\n",
       "      <th>filename_contains_digit</th>\n",
       "      <th>id</th>\n",
       "      <th>undigit_filename_translation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Billingsgate Cartographer; Blome, RichardSurve...</td>\n",
       "      <td>Billingsgate Cartographer Blome RichardSurveyo...</td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.413867</td>\n",
       "      <td>True</td>\n",
       "      <td>commons</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Fairbanks, fur market (02 cut)</td>\n",
       "      <td>Fairbanks fur market 02 cut</td>\n",
       "      <td>Fairbanks fur market 02 cut</td>\n",
       "      <td>Fairbanks fur market cut</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.868023</td>\n",
       "      <td>True</td>\n",
       "      <td>commons</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio</td>\n",
       "      <td>False</td>\n",
       "      <td>nl</td>\n",
       "      <td>0.310636</td>\n",
       "      <td>False</td>\n",
       "      <td>commons</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>Blenio Media Tour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>2</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>False</td>\n",
       "      <td>nl</td>\n",
       "      <td>0.365448</td>\n",
       "      <td>False</td>\n",
       "      <td>commons</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Iron Maiden's Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.354466</td>\n",
       "      <td>True</td>\n",
       "      <td>commons</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44757</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/2/2f...</td>\n",
       "      <td>1</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.396447</td>\n",
       "      <td>True</td>\n",
       "      <td>vi</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>44757</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44758</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/5/52...</td>\n",
       "      <td>1</td>\n",
       "      <td>Thanhongdem1</td>\n",
       "      <td>Thanhongdem1</td>\n",
       "      <td>Thanhongdem 1</td>\n",
       "      <td>Thanhongdem</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.400808</td>\n",
       "      <td>True</td>\n",
       "      <td>vi</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44758</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44759</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/8/84...</td>\n",
       "      <td>1</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>0.999809</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Chua một cột 1896</td>\n",
       "      <td>True</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44759</td>\n",
       "      <td>One Pillar Pagoda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44760</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/8/89...</td>\n",
       "      <td>1</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>0.975191</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Giao hoang Gioan Phao lo II chụp cung HY Giuse...</td>\n",
       "      <td>True</td>\n",
       "      <td>png</td>\n",
       "      <td>False</td>\n",
       "      <td>44760</td>\n",
       "      <td>Pope John Paul II poses with Cardinal Joseph T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44761</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/b/b4...</td>\n",
       "      <td>3</td>\n",
       "      <td>NamKy (BasseCochichine)1878</td>\n",
       "      <td>NamKy BasseCochichine 1878</td>\n",
       "      <td>Nam Ky Basse Cochichine 1878</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "      <td>False</td>\n",
       "      <td>ceb</td>\n",
       "      <td>0.226009</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44761</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44762 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               image_url  count  \\\n",
       "0      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "1      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "2      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "3      http://upload.wikimedia.org/wikipedia/commons/...      2   \n",
       "4      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "...                                                  ...    ...   \n",
       "44757  https://upload.wikimedia.org/wikipedia/vi/2/2f...      1   \n",
       "44758  https://upload.wikimedia.org/wikipedia/vi/5/52...      1   \n",
       "44759  https://upload.wikimedia.org/wikipedia/vi/8/84...      1   \n",
       "44760  https://upload.wikimedia.org/wikipedia/vi/8/89...      1   \n",
       "44761  https://upload.wikimedia.org/wikipedia/vi/b/b4...      3   \n",
       "\n",
       "                                                filename  \\\n",
       "0      Billingsgate Cartographer; Blome, RichardSurve...   \n",
       "1                         Fairbanks, fur market (02 cut)   \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden's Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                       Thanhongdem1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                        NamKy (BasseCochichine)1878   \n",
       "\n",
       "                                          pured_filename  \\\n",
       "0      Billingsgate Cartographer Blome RichardSurveyo...   \n",
       "1                           Fairbanks fur market 02 cut    \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                       Thanhongdem1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                         NamKy BasseCochichine 1878   \n",
       "\n",
       "                                         spaced_filename  \\\n",
       "0      Billingsgate Cartographer Blome Richard Survey...   \n",
       "1                           Fairbanks fur market 02 cut    \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                      Thanhongdem 1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                       Nam Ky Basse Cochichine 1878   \n",
       "\n",
       "                                        undigit_filename  \\\n",
       "0      Billingsgate Cartographer Blome Richard Survey...   \n",
       "1                               Fairbanks fur market cut   \n",
       "2                                      Giro Media Blenio   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                        Thanhongdem   \n",
       "44759                                       Chùa một cột   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                            Nam Ky Basse Cochichine   \n",
       "\n",
       "       undigit_filename_empty filename_lang  filename_lang_p  filename_en  \\\n",
       "0                       False            en         0.413867         True   \n",
       "1                       False            en         0.868023         True   \n",
       "2                       False            nl         0.310636        False   \n",
       "3                       False            nl         0.365448        False   \n",
       "4                       False            en         0.354466         True   \n",
       "...                       ...           ...              ...          ...   \n",
       "44757                   False            en         0.396447         True   \n",
       "44758                   False            en         0.400808         True   \n",
       "44759                   False            vi         0.999809        False   \n",
       "44760                   False            vi         0.975191        False   \n",
       "44761                   False           ceb         0.226009        False   \n",
       "\n",
       "       section  commons  lang_ok  \\\n",
       "0      commons     True    False   \n",
       "1      commons     True    False   \n",
       "2      commons     True    False   \n",
       "3      commons     True    False   \n",
       "4      commons     True    False   \n",
       "...        ...      ...      ...   \n",
       "44757       vi    False    False   \n",
       "44758       vi    False    False   \n",
       "44759       vi    False     True   \n",
       "44760       vi    False     True   \n",
       "44761       vi    False    False   \n",
       "\n",
       "                                spaced_filename_translit  \\\n",
       "0                                                          \n",
       "1                                                          \n",
       "2                                                          \n",
       "3                                                          \n",
       "4                                                          \n",
       "...                                                  ...   \n",
       "44757                                                      \n",
       "44758                                                      \n",
       "44759                                  Chua một cột 1896   \n",
       "44760  Giao hoang Gioan Phao lo II chụp cung HY Giuse...   \n",
       "44761                                                      \n",
       "\n",
       "       spaced_filename_translited  ext  filename_contains_digit     id  \\\n",
       "0                           False  jpg                     True      0   \n",
       "1                           False  jpg                     True      1   \n",
       "2                           False  jpg                     True      2   \n",
       "3                           False  jpg                    False      3   \n",
       "4                           False  jpg                    False      4   \n",
       "...                           ...  ...                      ...    ...   \n",
       "44757                       False  jpg                    False  44757   \n",
       "44758                       False  jpg                     True  44758   \n",
       "44759                        True  jpg                     True  44759   \n",
       "44760                        True  png                    False  44760   \n",
       "44761                       False  jpg                     True  44761   \n",
       "\n",
       "                            undigit_filename_translation  \n",
       "0                                                         \n",
       "1                                                         \n",
       "2                                      Blenio Media Tour  \n",
       "3                       Guetenbachtor Lainzer Tiergarten  \n",
       "4                                                         \n",
       "...                                                  ...  \n",
       "44757                                                     \n",
       "44758                                                     \n",
       "44759                                  One Pillar Pagoda  \n",
       "44760  Pope John Paul II poses with Cardinal Joseph T...  \n",
       "44761                            Nam Ky Basse Cochichine  \n",
       "\n",
       "[44762 rows x 19 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parts = [part1]\n",
    "\n",
    "images['undigit_filename_translation'] = ''\n",
    "\n",
    "for p in parts:\n",
    "    p['undigit_filename_translation'] = p['undigit_filename_translation'].str.strip()\n",
    "    indices = p[p['undigit_filename_translation'] != ''].index\n",
    "    images.loc[indices, 'undigit_filename_translation'] = p.loc[indices, 'undigit_filename_translation']\n",
    "    \n",
    "images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0a5f3e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:20.451805Z",
     "iopub.status.busy": "2022-02-09T16:39:20.450786Z",
     "iopub.status.idle": "2022-02-09T16:39:20.537683Z",
     "shell.execute_reply": "2022-02-09T16:39:20.537015Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.219887Z"
    },
    "papermill": {
     "duration": 0.110512,
     "end_time": "2022-02-09T16:39:20.537880",
     "exception": false,
     "start_time": "2022-02-09T16:39:20.427368",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "images['final_filename'] = images['undigit_filename_translation']\n",
    "\n",
    "en_indices = images[(images['final_filename'] == '') & (images['filename_en'])].index\n",
    "images.loc[en_indices, 'final_filename'] = images.loc[en_indices, 'undigit_filename']\n",
    "\n",
    "translit_indices = images[(images['final_filename'] == '') & (images['spaced_filename_translit'] != '')].index\n",
    "images.loc[translit_indices, 'final_filename'] = images.loc[translit_indices, 'spaced_filename_translit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f1ea3d56",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:20.601995Z",
     "iopub.status.busy": "2022-02-09T16:39:20.600868Z",
     "iopub.status.idle": "2022-02-09T16:39:20.655249Z",
     "shell.execute_reply": "2022-02-09T16:39:20.654595Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.298264Z"
    },
    "papermill": {
     "duration": 0.097155,
     "end_time": "2022-02-09T16:39:20.655457",
     "exception": false,
     "start_time": "2022-02-09T16:39:20.558302",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "images['PREfinal_filename'] = images['undigit_filename_translation']\n",
    "empty_indices = images[(images['PREfinal_filename'] == '') & (images['undigit_filename'] != '')].index\n",
    "images.loc[empty_indices, 'PREfinal_filename'] = images.loc[empty_indices, 'undigit_filename']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4f82cd08",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:20.718166Z",
     "iopub.status.busy": "2022-02-09T16:39:20.709957Z",
     "iopub.status.idle": "2022-02-09T16:39:20.986570Z",
     "shell.execute_reply": "2022-02-09T16:39:20.985918Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.356883Z"
    },
    "papermill": {
     "duration": 0.310226,
     "end_time": "2022-02-09T16:39:20.986772",
     "exception": false,
     "start_time": "2022-02-09T16:39:20.676546",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for field in ['PREfinal_filename', 'final_filename', 'undigit_filename_translation']:\n",
    "    images[field] = images[field].str.replace(r'\\s{2,}', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e2fe7d65",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:21.042787Z",
     "iopub.status.busy": "2022-02-09T16:39:21.041509Z",
     "iopub.status.idle": "2022-02-09T16:39:21.152726Z",
     "shell.execute_reply": "2022-02-09T16:39:21.153270Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.729579Z"
    },
    "papermill": {
     "duration": 0.146925,
     "end_time": "2022-02-09T16:39:21.153457",
     "exception": false,
     "start_time": "2022-02-09T16:39:21.006532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_url</th>\n",
       "      <th>count</th>\n",
       "      <th>filename</th>\n",
       "      <th>pured_filename</th>\n",
       "      <th>spaced_filename</th>\n",
       "      <th>undigit_filename</th>\n",
       "      <th>undigit_filename_empty</th>\n",
       "      <th>filename_lang</th>\n",
       "      <th>filename_lang_p</th>\n",
       "      <th>filename_en</th>\n",
       "      <th>...</th>\n",
       "      <th>commons</th>\n",
       "      <th>lang_ok</th>\n",
       "      <th>spaced_filename_translit</th>\n",
       "      <th>spaced_filename_translited</th>\n",
       "      <th>ext</th>\n",
       "      <th>filename_contains_digit</th>\n",
       "      <th>id</th>\n",
       "      <th>undigit_filename_translation</th>\n",
       "      <th>final_filename</th>\n",
       "      <th>PREfinal_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Billingsgate Cartographer; Blome, RichardSurve...</td>\n",
       "      <td>Billingsgate Cartographer Blome RichardSurveyo...</td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.413867</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "      <td>Billingsgate Cartographer Blome Richard Survey...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Fairbanks, fur market (02 cut)</td>\n",
       "      <td>Fairbanks fur market 02 cut</td>\n",
       "      <td>Fairbanks fur market 02 cut</td>\n",
       "      <td>Fairbanks fur market cut</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.868023</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>Fairbanks fur market cut</td>\n",
       "      <td>Fairbanks fur market cut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio 1</td>\n",
       "      <td>Giro Media Blenio</td>\n",
       "      <td>False</td>\n",
       "      <td>nl</td>\n",
       "      <td>0.310636</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>Blenio Media Tour</td>\n",
       "      <td>Blenio Media Tour</td>\n",
       "      <td>Blenio Media Tour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>2</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>False</td>\n",
       "      <td>nl</td>\n",
       "      <td>0.365448</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "      <td>Guetenbachtor Lainzer Tiergarten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://upload.wikimedia.org/wikipedia/commons/...</td>\n",
       "      <td>1</td>\n",
       "      <td>Iron Maiden's Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.354466</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "      <td>Iron Maiden s Eddie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44757</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/2/2f...</td>\n",
       "      <td>1</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.396447</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>44757</td>\n",
       "      <td></td>\n",
       "      <td>Princess kaiulani</td>\n",
       "      <td>Princess kaiulani</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44758</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/5/52...</td>\n",
       "      <td>1</td>\n",
       "      <td>Thanhongdem1</td>\n",
       "      <td>Thanhongdem1</td>\n",
       "      <td>Thanhongdem 1</td>\n",
       "      <td>Thanhongdem</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.400808</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44758</td>\n",
       "      <td></td>\n",
       "      <td>Thanhongdem</td>\n",
       "      <td>Thanhongdem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44759</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/8/84...</td>\n",
       "      <td>1</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột 1896</td>\n",
       "      <td>Chùa một cột</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>0.999809</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Chua một cột 1896</td>\n",
       "      <td>True</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44759</td>\n",
       "      <td>One Pillar Pagoda</td>\n",
       "      <td>One Pillar Pagoda</td>\n",
       "      <td>One Pillar Pagoda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44760</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/8/89...</td>\n",
       "      <td>1</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...</td>\n",
       "      <td>False</td>\n",
       "      <td>vi</td>\n",
       "      <td>0.975191</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Giao hoang Gioan Phao lo II chụp cung HY Giuse...</td>\n",
       "      <td>True</td>\n",
       "      <td>png</td>\n",
       "      <td>False</td>\n",
       "      <td>44760</td>\n",
       "      <td>Pope John Paul II poses with Cardinal Joseph T...</td>\n",
       "      <td>Pope John Paul II poses with Cardinal Joseph T...</td>\n",
       "      <td>Pope John Paul II poses with Cardinal Joseph T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44761</th>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/vi/b/b4...</td>\n",
       "      <td>3</td>\n",
       "      <td>NamKy (BasseCochichine)1878</td>\n",
       "      <td>NamKy BasseCochichine 1878</td>\n",
       "      <td>Nam Ky Basse Cochichine 1878</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "      <td>False</td>\n",
       "      <td>ceb</td>\n",
       "      <td>0.226009</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>jpg</td>\n",
       "      <td>True</td>\n",
       "      <td>44761</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "      <td>Nam Ky Basse Cochichine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44754 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               image_url  count  \\\n",
       "0      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "1      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "2      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "3      http://upload.wikimedia.org/wikipedia/commons/...      2   \n",
       "4      http://upload.wikimedia.org/wikipedia/commons/...      1   \n",
       "...                                                  ...    ...   \n",
       "44757  https://upload.wikimedia.org/wikipedia/vi/2/2f...      1   \n",
       "44758  https://upload.wikimedia.org/wikipedia/vi/5/52...      1   \n",
       "44759  https://upload.wikimedia.org/wikipedia/vi/8/84...      1   \n",
       "44760  https://upload.wikimedia.org/wikipedia/vi/8/89...      1   \n",
       "44761  https://upload.wikimedia.org/wikipedia/vi/b/b4...      3   \n",
       "\n",
       "                                                filename  \\\n",
       "0      Billingsgate Cartographer; Blome, RichardSurve...   \n",
       "1                         Fairbanks, fur market (02 cut)   \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden's Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                       Thanhongdem1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                        NamKy (BasseCochichine)1878   \n",
       "\n",
       "                                          pured_filename  \\\n",
       "0      Billingsgate Cartographer Blome RichardSurveyo...   \n",
       "1                           Fairbanks fur market 02 cut    \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                       Thanhongdem1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                         NamKy BasseCochichine 1878   \n",
       "\n",
       "                                         spaced_filename  \\\n",
       "0      Billingsgate Cartographer Blome Richard Survey...   \n",
       "1                           Fairbanks fur market 02 cut    \n",
       "2                                    Giro Media Blenio 1   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                      Thanhongdem 1   \n",
       "44759                                  Chùa một cột 1896   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                       Nam Ky Basse Cochichine 1878   \n",
       "\n",
       "                                        undigit_filename  \\\n",
       "0      Billingsgate Cartographer Blome Richard Survey...   \n",
       "1                               Fairbanks fur market cut   \n",
       "2                                      Giro Media Blenio   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                        Thanhongdem   \n",
       "44759                                       Chùa một cột   \n",
       "44760  Giáo hoàng Gioan Phao lô II chụp cùng HY Giuse...   \n",
       "44761                            Nam Ky Basse Cochichine   \n",
       "\n",
       "       undigit_filename_empty filename_lang  filename_lang_p  filename_en  \\\n",
       "0                       False            en         0.413867         True   \n",
       "1                       False            en         0.868023         True   \n",
       "2                       False            nl         0.310636        False   \n",
       "3                       False            nl         0.365448        False   \n",
       "4                       False            en         0.354466         True   \n",
       "...                       ...           ...              ...          ...   \n",
       "44757                   False            en         0.396447         True   \n",
       "44758                   False            en         0.400808         True   \n",
       "44759                   False            vi         0.999809        False   \n",
       "44760                   False            vi         0.975191        False   \n",
       "44761                   False           ceb         0.226009        False   \n",
       "\n",
       "       ... commons  lang_ok  \\\n",
       "0      ...    True    False   \n",
       "1      ...    True    False   \n",
       "2      ...    True    False   \n",
       "3      ...    True    False   \n",
       "4      ...    True    False   \n",
       "...    ...     ...      ...   \n",
       "44757  ...   False    False   \n",
       "44758  ...   False    False   \n",
       "44759  ...   False     True   \n",
       "44760  ...   False     True   \n",
       "44761  ...   False    False   \n",
       "\n",
       "                                spaced_filename_translit  \\\n",
       "0                                                          \n",
       "1                                                          \n",
       "2                                                          \n",
       "3                                                          \n",
       "4                                                          \n",
       "...                                                  ...   \n",
       "44757                                                      \n",
       "44758                                                      \n",
       "44759                                  Chua một cột 1896   \n",
       "44760  Giao hoang Gioan Phao lo II chụp cung HY Giuse...   \n",
       "44761                                                      \n",
       "\n",
       "      spaced_filename_translited  ext filename_contains_digit     id  \\\n",
       "0                          False  jpg                    True      0   \n",
       "1                          False  jpg                    True      1   \n",
       "2                          False  jpg                    True      2   \n",
       "3                          False  jpg                   False      3   \n",
       "4                          False  jpg                   False      4   \n",
       "...                          ...  ...                     ...    ...   \n",
       "44757                      False  jpg                   False  44757   \n",
       "44758                      False  jpg                    True  44758   \n",
       "44759                       True  jpg                    True  44759   \n",
       "44760                       True  png                   False  44760   \n",
       "44761                      False  jpg                    True  44761   \n",
       "\n",
       "                            undigit_filename_translation  \\\n",
       "0                                                          \n",
       "1                                                          \n",
       "2                                      Blenio Media Tour   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                                          \n",
       "...                                                  ...   \n",
       "44757                                                      \n",
       "44758                                                      \n",
       "44759                                  One Pillar Pagoda   \n",
       "44760  Pope John Paul II poses with Cardinal Joseph T...   \n",
       "44761                            Nam Ky Basse Cochichine   \n",
       "\n",
       "                                          final_filename  \\\n",
       "0      Billingsgate Cartographer Blome Richard Survey...   \n",
       "1                               Fairbanks fur market cut   \n",
       "2                                      Blenio Media Tour   \n",
       "3                       Guetenbachtor Lainzer Tiergarten   \n",
       "4                                    Iron Maiden s Eddie   \n",
       "...                                                  ...   \n",
       "44757                                  Princess kaiulani   \n",
       "44758                                        Thanhongdem   \n",
       "44759                                  One Pillar Pagoda   \n",
       "44760  Pope John Paul II poses with Cardinal Joseph T...   \n",
       "44761                            Nam Ky Basse Cochichine   \n",
       "\n",
       "                                       PREfinal_filename  \n",
       "0      Billingsgate Cartographer Blome Richard Survey...  \n",
       "1                               Fairbanks fur market cut  \n",
       "2                                      Blenio Media Tour  \n",
       "3                       Guetenbachtor Lainzer Tiergarten  \n",
       "4                                    Iron Maiden s Eddie  \n",
       "...                                                  ...  \n",
       "44757                                  Princess kaiulani  \n",
       "44758                                        Thanhongdem  \n",
       "44759                                  One Pillar Pagoda  \n",
       "44760  Pope John Paul II poses with Cardinal Joseph T...  \n",
       "44761                            Nam Ky Basse Cochichine  \n",
       "\n",
       "[44754 rows x 21 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images[images['PREfinal_filename'] != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b547e668",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:21.207085Z",
     "iopub.status.busy": "2022-02-09T16:39:21.206348Z",
     "iopub.status.idle": "2022-02-09T16:39:22.031184Z",
     "shell.execute_reply": "2022-02-09T16:39:22.030454Z",
     "shell.execute_reply.started": "2022-02-09T16:37:41.839126Z"
    },
    "papermill": {
     "duration": 0.853898,
     "end_time": "2022-02-09T16:39:22.031342",
     "exception": false,
     "start_time": "2022-02-09T16:39:21.177444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: spaced_undigit_filename, dtype: object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images['spaced_undigit_filename'] = images['undigit_filename']. \\\n",
    "    str.replace(r'([a-z]*)([A-Z]{2,})([a-z]+)', r'\\1 \\2 \\3', regex=True). \\\n",
    "    str.replace(r'\\s{2,}', ' ', regex=True).str.strip()\n",
    "\n",
    "images.loc[images['undigit_filename'].str.contains('VEI'), 'spaced_undigit_filename']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3ef44d07",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:22.079166Z",
     "iopub.status.busy": "2022-02-09T16:39:22.078356Z",
     "iopub.status.idle": "2022-02-09T16:39:22.154476Z",
     "shell.execute_reply": "2022-02-09T16:39:22.153444Z",
     "shell.execute_reply.started": "2022-02-09T16:37:42.828984Z"
    },
    "papermill": {
     "duration": 0.101096,
     "end_time": "2022-02-09T16:39:22.154712",
     "exception": false,
     "start_time": "2022-02-09T16:39:22.053616",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images.isna().any().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a89e5bb5",
   "metadata": {
    "papermill": {
     "duration": 0.021874,
     "end_time": "2022-02-09T16:39:22.200398",
     "exception": false,
     "start_time": "2022-02-09T16:39:22.178524",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Предсохранение контейнеров"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "01f92c1b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:22.262952Z",
     "iopub.status.busy": "2022-02-09T16:39:22.254685Z",
     "iopub.status.idle": "2022-02-09T16:39:23.884585Z",
     "shell.execute_reply": "2022-02-09T16:39:23.883871Z",
     "shell.execute_reply.started": "2022-02-09T16:37:42.904662Z"
    },
    "papermill": {
     "duration": 1.660055,
     "end_time": "2022-02-09T16:39:23.884773",
     "exception": false,
     "start_time": "2022-02-09T16:39:22.224718",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'wikimatcher'...\r\n",
      "remote: Enumerating objects: 496, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (496/496), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (342/342), done.\u001b[K\r\n",
      "remote: Total 496 (delta 329), reused 315 (delta 151), pack-reused 0\u001b[K\r\n",
      "Receiving objects: 100% (496/496), 70.46 KiB | 2.71 MiB/s, done.\r\n",
      "Resolving deltas: 100% (329/329), done.\r\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "\n",
    "!git clone https://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@github.com/basic-go-ahead/wikimatcher.git\n",
    "sys.path.append('./wikimatcher')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "97a2cf91",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:23.938320Z",
     "iopub.status.busy": "2022-02-09T16:39:23.937580Z",
     "iopub.status.idle": "2022-02-09T16:39:26.224184Z",
     "shell.execute_reply": "2022-02-09T16:39:26.223506Z",
     "shell.execute_reply.started": "2022-02-09T16:37:44.606026Z"
    },
    "papermill": {
     "duration": 2.316465,
     "end_time": "2022-02-09T16:39:26.224378",
     "exception": false,
     "start_time": "2022-02-09T16:39:23.907913",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from wikimatcher.containers import WordContainer, ImageNumberContainer, ImageCapitalWordContainer\n",
    "from wikimatcher.preprocessing import WordBasicPreprocessor, ShortWordPreprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0d99eb93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:26.282276Z",
     "iopub.status.busy": "2022-02-09T16:39:26.281554Z",
     "iopub.status.idle": "2022-02-09T16:39:35.012183Z",
     "shell.execute_reply": "2022-02-09T16:39:35.011581Z",
     "shell.execute_reply.started": "2022-02-09T16:37:46.721036Z"
    },
    "papermill": {
     "duration": 8.763169,
     "end_time": "2022-02-09T16:39:35.012354",
     "exception": false,
     "start_time": "2022-02-09T16:39:26.249185",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir containers\n",
    "\n",
    "WordBasicPreprocessor(). \\\n",
    "    process(df=images, columns2process='final_filename', id_column='id'). \\\n",
    "    save('./containers/final_words.wc')\n",
    "\n",
    "ShortWordPreprocessor(). \\\n",
    "    process(df=images, columns2process='spaced_undigit_filename', id_column='id'). \\\n",
    "    save('./containers/short_words.wc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "82aaa077",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:35.073753Z",
     "iopub.status.busy": "2022-02-09T16:39:35.073016Z",
     "iopub.status.idle": "2022-02-09T16:39:37.736597Z",
     "shell.execute_reply": "2022-02-09T16:39:37.735458Z",
     "shell.execute_reply.started": "2022-02-09T16:37:56.023901Z"
    },
    "papermill": {
     "duration": 2.701231,
     "end_time": "2022-02-09T16:39:37.736813",
     "exception": false,
     "start_time": "2022-02-09T16:39:35.035582",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "image_numbers = ImageNumberContainer(images)\n",
    "image_numbers.save('./containers/numbers.inc')\n",
    "\n",
    "image_capitals = ImageCapitalWordContainer()\n",
    "image_capitals.process(images)\n",
    "image_capitals.save('./containers/capitals.icwc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "10d5ed1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:37.804159Z",
     "iopub.status.busy": "2022-02-09T16:39:37.793493Z",
     "iopub.status.idle": "2022-02-09T16:39:38.585237Z",
     "shell.execute_reply": "2022-02-09T16:39:38.586179Z",
     "shell.execute_reply.started": "2022-02-09T16:37:58.031028Z"
    },
    "papermill": {
     "duration": 0.82381,
     "end_time": "2022-02-09T16:39:38.586399",
     "exception": false,
     "start_time": "2022-02-09T16:39:37.762589",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm -r ./wikimatcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f6379a3a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:38.644216Z",
     "iopub.status.busy": "2022-02-09T16:39:38.643137Z",
     "iopub.status.idle": "2022-02-09T16:39:39.725828Z",
     "shell.execute_reply": "2022-02-09T16:39:39.724974Z",
     "shell.execute_reply.started": "2022-02-09T16:37:58.326458Z"
    },
    "papermill": {
     "duration": 1.113185,
     "end_time": "2022-02-09T16:39:39.726030",
     "exception": false,
     "start_time": "2022-02-09T16:39:38.612845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "images.drop(columns=['filename_lang_p', 'undigit_filename_empty', 'commons', 'lang_ok', 'spaced_filename_translited']). \\\n",
    "    to_csv('final_images.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ddb02e82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-09T16:39:39.794728Z",
     "iopub.status.busy": "2022-02-09T16:39:39.793966Z",
     "iopub.status.idle": "2022-02-09T16:39:39.802490Z",
     "shell.execute_reply": "2022-02-09T16:39:39.803201Z",
     "shell.execute_reply.started": "2022-02-09T16:37:59.107118Z"
    },
    "papermill": {
     "duration": 0.048859,
     "end_time": "2022-02-09T16:39:39.803414",
     "exception": false,
     "start_time": "2022-02-09T16:39:39.754555",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>image_url</th>\n",
       "      <td>image_url</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>count</td>\n",
       "      <td>int64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename</th>\n",
       "      <td>filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pured_filename</th>\n",
       "      <td>pured_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spaced_filename</th>\n",
       "      <td>spaced_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>undigit_filename</th>\n",
       "      <td>undigit_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>undigit_filename_empty</th>\n",
       "      <td>undigit_filename_empty</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename_lang</th>\n",
       "      <td>filename_lang</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename_lang_p</th>\n",
       "      <td>filename_lang_p</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename_en</th>\n",
       "      <td>filename_en</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>section</th>\n",
       "      <td>section</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>commons</th>\n",
       "      <td>commons</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lang_ok</th>\n",
       "      <td>lang_ok</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spaced_filename_translit</th>\n",
       "      <td>spaced_filename_translit</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spaced_filename_translited</th>\n",
       "      <td>spaced_filename_translited</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ext</th>\n",
       "      <td>ext</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filename_contains_digit</th>\n",
       "      <td>filename_contains_digit</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>id</td>\n",
       "      <td>int64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>undigit_filename_translation</th>\n",
       "      <td>undigit_filename_translation</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>final_filename</th>\n",
       "      <td>final_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PREfinal_filename</th>\n",
       "      <td>PREfinal_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spaced_undigit_filename</th>\n",
       "      <td>spaced_undigit_filename</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    column     type\n",
       "image_url                                        image_url   object\n",
       "count                                                count    int64\n",
       "filename                                          filename   object\n",
       "pured_filename                              pured_filename   object\n",
       "spaced_filename                            spaced_filename   object\n",
       "undigit_filename                          undigit_filename   object\n",
       "undigit_filename_empty              undigit_filename_empty     bool\n",
       "filename_lang                                filename_lang   object\n",
       "filename_lang_p                            filename_lang_p  float64\n",
       "filename_en                                    filename_en     bool\n",
       "section                                            section   object\n",
       "commons                                            commons     bool\n",
       "lang_ok                                            lang_ok     bool\n",
       "spaced_filename_translit          spaced_filename_translit   object\n",
       "spaced_filename_translited      spaced_filename_translited     bool\n",
       "ext                                                    ext   object\n",
       "filename_contains_digit            filename_contains_digit     bool\n",
       "id                                                      id    int64\n",
       "undigit_filename_translation  undigit_filename_translation   object\n",
       "final_filename                              final_filename   object\n",
       "PREfinal_filename                        PREfinal_filename   object\n",
       "spaced_undigit_filename            spaced_undigit_filename   object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({ 'column': images.columns, 'type': images.dtypes })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "901e87e2",
   "metadata": {
    "papermill": {
     "duration": 0.030075,
     "end_time": "2022-02-09T16:39:39.858431",
     "exception": false,
     "start_time": "2022-02-09T16:39:39.828356",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 56.042419,
   "end_time": "2022-02-09T16:39:41.464234",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-02-09T16:38:45.421815",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}