scripts/representation-bias-measurement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6459a8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f95a363",
   "metadata": {},
   "source": [
    "## Reading Grid Summary of All locations in GeoNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5fad1804",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_geonames_grids = os.path.dirname(os.getcwd())+'\\\\data\\\\grids-100sqkm-admin0-natural-earth\\\\grids_summary_GeoNames_poi.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e5056aef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>Join_Count</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>PageName</th>\n",
       "      <th>PageNumber</th>\n",
       "      <th>featurecla</th>\n",
       "      <th>scalerank</th>\n",
       "      <th>LABELRANK</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>SOV_A3</th>\n",
       "      <th>...</th>\n",
       "      <th>FCLASS_PL</th>\n",
       "      <th>FCLASS_GR</th>\n",
       "      <th>FCLASS_IT</th>\n",
       "      <th>FCLASS_NL</th>\n",
       "      <th>FCLASS_SE</th>\n",
       "      <th>FCLASS_BD</th>\n",
       "      <th>FCLASS_UA</th>\n",
       "      <th>Shape_Length</th>\n",
       "      <th>Shape_Area</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1456</td>\n",
       "      <td>A1456</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1457</td>\n",
       "      <td>A1457</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1458</td>\n",
       "      <td>A1458</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1459</td>\n",
       "      <td>A1459</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1461</td>\n",
       "      <td>A1461</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 169 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  Join_Count  TARGET_FID PageName  PageNumber featurecla  scalerank  \\\n",
       "0     1           0        1456    A1456           1        NaN        NaN   \n",
       "1     2           0        1457    A1457           2        NaN        NaN   \n",
       "2     3           0        1458    A1458           3        NaN        NaN   \n",
       "3     4           0        1459    A1459           4        NaN        NaN   \n",
       "4     5           0        1461    A1461           5        NaN        NaN   \n",
       "\n",
       "   LABELRANK SOVEREIGNT SOV_A3  ...  FCLASS_PL  FCLASS_GR FCLASS_IT FCLASS_NL  \\\n",
       "0        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "1        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "2        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "3        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "4        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "\n",
       "  FCLASS_SE  FCLASS_BD FCLASS_UA Shape_Length   Shape_Area Point_Count  \n",
       "0       NaN        NaN       NaN      40000.0  100000000.0           1  \n",
       "1       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "2       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "3       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "4       NaN        NaN       NaN      40000.0  100000000.0           1  \n",
       "\n",
       "[5 rows x 169 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geonames_grids = pd.read_csv(dir_geonames_grids)\n",
    "df_geonames_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "292ba359",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count\n",
       "0     1        1456        NaN            1\n",
       "1     2        1457        NaN            0\n",
       "2     3        1458        NaN            0\n",
       "3     4        1459        NaN            0\n",
       "4     5        1461        NaN            1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Only columns that are needed for measuring representation biases are kept.\n",
    "## The column Point_Count stands for the number of locations within the grid with a TARGET_FID.\n",
    "df_geonames_grids = df_geonames_grids[['OID_','TARGET_FID','SOVEREIGNT','Point_Count']]\n",
    "df_geonames_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bd398386",
   "metadata": {},
   "outputs": [],
   "source": [
    "## The same preprocessing is carried out for all the other grid summary datasets below."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15251c4d",
   "metadata": {},
   "source": [
    "## Measuring Spatial (Diversity) Misalignment between GeoWiki and GeoNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b0d1e83d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## all comments listed in this section can also be applied to all the other spatial (diversity) misalignment calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1e67d6b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_geowiki_grids = os.path.dirname(os.getcwd())+'\\\\data\\\\grids-100sqkm-admin0-natural-earth\\\\grids_summary_GeoWiki_poi.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "73efbd7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>Join_Count</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>PageName</th>\n",
       "      <th>PageNumber</th>\n",
       "      <th>featurecla</th>\n",
       "      <th>scalerank</th>\n",
       "      <th>LABELRANK</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>SOV_A3</th>\n",
       "      <th>...</th>\n",
       "      <th>FCLASS_PL</th>\n",
       "      <th>FCLASS_GR</th>\n",
       "      <th>FCLASS_IT</th>\n",
       "      <th>FCLASS_NL</th>\n",
       "      <th>FCLASS_SE</th>\n",
       "      <th>FCLASS_BD</th>\n",
       "      <th>FCLASS_UA</th>\n",
       "      <th>Shape_Length</th>\n",
       "      <th>Shape_Area</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1456</td>\n",
       "      <td>A1456</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1457</td>\n",
       "      <td>A1457</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1458</td>\n",
       "      <td>A1458</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1459</td>\n",
       "      <td>A1459</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1461</td>\n",
       "      <td>A1461</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 169 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  Join_Count  TARGET_FID PageName  PageNumber featurecla  scalerank  \\\n",
       "0     1           0        1456    A1456           1        NaN        NaN   \n",
       "1     2           0        1457    A1457           2        NaN        NaN   \n",
       "2     3           0        1458    A1458           3        NaN        NaN   \n",
       "3     4           0        1459    A1459           4        NaN        NaN   \n",
       "4     5           0        1461    A1461           5        NaN        NaN   \n",
       "\n",
       "   LABELRANK SOVEREIGNT SOV_A3  ...  FCLASS_PL  FCLASS_GR FCLASS_IT FCLASS_NL  \\\n",
       "0        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "1        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "2        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "3        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "4        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "\n",
       "  FCLASS_SE  FCLASS_BD FCLASS_UA Shape_Length   Shape_Area Point_Count  \n",
       "0       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "1       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "2       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "3       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "4       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "\n",
       "[5 rows x 169 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geowiki_grids = pd.read_csv(dir_geowiki_grids)\n",
    "df_geowiki_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7fe848eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count\n",
       "0     1        1456        NaN            0\n",
       "1     2        1457        NaN            0\n",
       "2     3        1458        NaN            0\n",
       "3     4        1459        NaN            0\n",
       "4     5        1461        NaN            0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geowiki_grids = df_geowiki_grids[['OID_','TARGET_FID','SOVEREIGNT','Point_Count']]\n",
    "df_geowiki_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6d449ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "## admin0_list is a list containing all Natural Earth Admin 0 regions that are included in the geowiki grid summary,\n",
    "admin0_list = list(df_geowiki_grids['SOVEREIGNT'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c89ae760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count_x</th>\n",
       "      <th>Point_Count_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count_x  Point_Count_y\n",
       "0     1        1456        NaN              1              0\n",
       "1     2        1457        NaN              0              0\n",
       "2     3        1458        NaN              0              0\n",
       "3     4        1459        NaN              0              0\n",
       "4     5        1461        NaN              1              0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## merge two grid summary datasets for spatial (diversity) misalignment calculation,\n",
    "df_geowiki_geonames_grids_merged = df_geonames_grids.merge(df_geowiki_grids, on=['OID_','SOVEREIGNT','TARGET_FID'])\n",
    "df_geowiki_geonames_grids_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1a87be95",
   "metadata": {},
   "outputs": [],
   "source": [
    "## calculate spatial misalignemnt\n",
    "sm_geowiki = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_geowiki_geonames_grids_merged_admin0 = df_geowiki_geonames_grids_merged[df_geowiki_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_geowiki_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_geowiki_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    if (np.isnan(1 - dot(gp, gc)/(norm(gp)*norm(gc))) == False):\n",
    "        sm_geowiki[admin0] = 1 - dot(gp, gc)/(norm(gp)*norm(gc))\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "sm_geowiki = sorted(sm_geowiki.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "efe3e77b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Barbados', 0.0),\n",
       " ('Federated States of Micronesia', 0.0),\n",
       " ('Saint Lucia', 0.0),\n",
       " ('Comoros', 0.009169831955701091),\n",
       " ('Luxembourg', 0.02008656540351006),\n",
       " ('Slovenia', 0.022833663200008014),\n",
       " ('Northern Cyprus', 0.028014686549853396),\n",
       " ('United Kingdom', 0.04773332815928455),\n",
       " ('Cyprus', 0.05395764881467724),\n",
       " ('Cabo Verde', 0.0618173443358514),\n",
       " ('Slovakia', 0.06615253967160784),\n",
       " ('Czechia', 0.07045340599556493),\n",
       " ('Estonia', 0.07732695663432188),\n",
       " ('São Tomé and Principe', 0.08051068557917795),\n",
       " ('Moldova', 0.09396088741919872),\n",
       " ('Poland', 0.10432917086463656),\n",
       " ('Mauritius', 0.1336966723668891),\n",
       " ('Nepal', 0.14045200096284582),\n",
       " ('Spain', 0.15661301694773155),\n",
       " ('Trinidad and Tobago', 0.15842393201886362),\n",
       " ('Italy', 0.16715368566494226),\n",
       " ('Costa Rica', 0.16783548792711478),\n",
       " ('Netherlands', 0.17349858130962958),\n",
       " ('Hungary', 0.1761168268096871),\n",
       " ('France', 0.19763911060240957),\n",
       " ('Lebanon', 0.20902390733990417),\n",
       " ('Azerbaijan', 0.2189315709170726),\n",
       " ('Jamaica', 0.22047117537497207),\n",
       " ('Iran', 0.22991405042650714),\n",
       " ('Lesotho', 0.23381374443785363),\n",
       " ('United States of America', 0.23736878209218715),\n",
       " ('eSwatini', 0.2458531308327806),\n",
       " ('Bhutan', 0.25168907056692635),\n",
       " ('Greece', 0.25925675500339773),\n",
       " ('Syria', 0.2625579627832453),\n",
       " ('Suriname', 0.26921131715942614),\n",
       " ('Albania', 0.2694395727303822),\n",
       " ('Israel', 0.2713468881860839),\n",
       " ('Bosnia and Herzegovina', 0.2789017875175578),\n",
       " ('Jordan', 0.28025147327450384),\n",
       " ('Portugal', 0.28539556237081876),\n",
       " ('Brunei', 0.285502241144279),\n",
       " ('East Timor', 0.2912661769231243),\n",
       " ('Republic of Serbia', 0.2950234364696517),\n",
       " ('Japan', 0.3005250649523148),\n",
       " ('Taiwan', 0.30927838632822513),\n",
       " ('Switzerland', 0.31468837527848925),\n",
       " ('Australia', 0.3172353862919177),\n",
       " ('Romania', 0.3419905518950168),\n",
       " ('Kenya', 0.347503023096565),\n",
       " ('Belgium', 0.3515387700178013),\n",
       " ('Samoa', 0.35210365512343367),\n",
       " ('Vietnam', 0.3544755579184464),\n",
       " ('Nicaragua', 0.3605724137103028),\n",
       " ('Kyrgyzstan', 0.3660743438773023),\n",
       " ('Oman', 0.3668958291629153),\n",
       " ('Bulgaria', 0.37201785616308003),\n",
       " ('Latvia', 0.3947767741770073),\n",
       " ('Egypt', 0.3950650368404959),\n",
       " ('United Arab Emirates', 0.39853617590506474),\n",
       " ('Colombia', 0.40455645842140875),\n",
       " ('Gambia', 0.4126249266099784),\n",
       " ('Kashmir', 0.42264973081037416),\n",
       " ('Iraq', 0.4246393252112963),\n",
       " ('Paraguay', 0.4252198890226857),\n",
       " ('Cambodia', 0.429478460669136),\n",
       " ('Namibia', 0.43132130797984025),\n",
       " ('Dominican Republic', 0.43185355101562406),\n",
       " ('Ireland', 0.4346069105288355),\n",
       " ('Kosovo', 0.4349272289943109),\n",
       " ('Burundi', 0.43934954116716585),\n",
       " ('Rwanda', 0.4431639098638426),\n",
       " ('Kuwait', 0.44519078518135247),\n",
       " ('Sri Lanka', 0.4480220666909177),\n",
       " ('Fiji', 0.44880938553832095),\n",
       " ('Guatemala', 0.4546029728836395),\n",
       " ('Saudi Arabia', 0.4567673859767968),\n",
       " ('Haiti', 0.45784588188115893),\n",
       " ('Argentina', 0.45868278490757886),\n",
       " ('Thailand', 0.45907759446655283),\n",
       " ('Benin', 0.4643099788461946),\n",
       " ('Germany', 0.4658641434765832),\n",
       " ('South Korea', 0.4689181486650629),\n",
       " ('Burkina Faso', 0.4725030847042765),\n",
       " ('Belize', 0.4725366684721457),\n",
       " ('El Salvador', 0.474380500544917),\n",
       " ('North Macedonia', 0.47469945409586956),\n",
       " ('Tunisia', 0.476397037687998),\n",
       " ('Chile', 0.47835776649624684),\n",
       " ('Honduras', 0.4861147674186781),\n",
       " ('Panama', 0.4946388566967461),\n",
       " ('Turkey', 0.49590701425310857),\n",
       " ('Denmark', 0.49745554354958965),\n",
       " ('Ecuador', 0.50433127638645),\n",
       " ('Armenia', 0.5115289402338612),\n",
       " ('Vanuatu', 0.5138431743730834),\n",
       " ('Ivory Coast', 0.5168716444596345),\n",
       " ('Sweden', 0.5181356296707006),\n",
       " ('Djibouti', 0.5211142993106883),\n",
       " ('Afghanistan', 0.5264843134894952),\n",
       " ('Cuba', 0.5307766521784582),\n",
       " ('Tajikistan', 0.5350068379035622),\n",
       " ('Georgia', 0.5442285188784886),\n",
       " ('Pakistan', 0.5492172624044573),\n",
       " ('Iceland', 0.5565426416239337),\n",
       " ('Austria', 0.5570924406725823),\n",
       " ('Malaysia', 0.56474053467198),\n",
       " ('Uganda', 0.5656537507348546),\n",
       " ('Ukraine', 0.567408017617824),\n",
       " ('Bir Tawil', 0.5735985672887791),\n",
       " ('Mali', 0.5764396474777601),\n",
       " ('Madagascar', 0.5766778824805492),\n",
       " ('Brazil', 0.5824596343688875),\n",
       " ('Qatar', 0.5834246161134984),\n",
       " ('Philippines', 0.5846595209672845),\n",
       " ('New Zealand', 0.5870535556777645),\n",
       " ('Morocco', 0.5886639049373763),\n",
       " ('Ghana', 0.592726875398941),\n",
       " ('Uruguay', 0.5934128389953692),\n",
       " ('Croatia', 0.597471950055087),\n",
       " ('Mongolia', 0.5983103451981469),\n",
       " ('Guinea-Bissau', 0.6011288018119552),\n",
       " ('Algeria', 0.6037248476579972),\n",
       " ('Venezuela', 0.6039714022524729),\n",
       " ('Liberia', 0.604485755104969),\n",
       " ('Mexico', 0.606286597478912),\n",
       " ('Libya', 0.6101149818854874),\n",
       " ('Uzbekistan', 0.6115928131030504),\n",
       " ('North Korea', 0.6121772838324926),\n",
       " ('Botswana', 0.6122286086990036),\n",
       " ('Togo', 0.6131561638058569),\n",
       " ('Montenegro', 0.618796768386674),\n",
       " ('Guinea', 0.6239083247293459),\n",
       " ('Bolivia', 0.624618143468993),\n",
       " ('Norway', 0.6275137766030624),\n",
       " ('United Republic of Tanzania', 0.6289099505391468),\n",
       " ('Zimbabwe', 0.6355404239919495),\n",
       " ('Eritrea', 0.6369920997411345),\n",
       " ('Lithuania', 0.6391036062728568),\n",
       " ('Gabon', 0.6444642149498485),\n",
       " ('Canada', 0.6514296005216055),\n",
       " ('The Bahamas', 0.6518446880886043),\n",
       " ('Antarctica', 0.6533947832401306),\n",
       " ('Bangladesh', 0.6540616225278261),\n",
       " ('Cameroon', 0.6564418746944144),\n",
       " ('Senegal', 0.6609034570504615),\n",
       " ('Finland', 0.6688109650817919),\n",
       " ('China', 0.6747710621391032),\n",
       " ('Equatorial Guinea', 0.6773180979960028),\n",
       " ('South Africa', 0.6797981406793643),\n",
       " ('Peru', 0.6812397699124221),\n",
       " ('Ethiopia', 0.6919286118389041),\n",
       " ('Papua New Guinea', 0.6965599164893954),\n",
       " ('Malawi', 0.6991398536242079),\n",
       " ('Myanmar', 0.7029004235855671),\n",
       " ('Guyana', 0.7172086175125696),\n",
       " ('Indonesia', 0.7186883654453743),\n",
       " ('Russia', 0.7214689452330677),\n",
       " ('Somalia', 0.7225465156948383),\n",
       " ('Somaliland', 0.7274286837299646),\n",
       " ('Laos', 0.7292730406716792),\n",
       " ('Sierra Leone', 0.740903720735232),\n",
       " ('Belarus', 0.7411862591938713),\n",
       " ('India', 0.742083912988768),\n",
       " ('Solomon Islands', 0.7475603235420777),\n",
       " ('Turkmenistan', 0.7498357143580092),\n",
       " ('Niger', 0.7520405747905685),\n",
       " ('Republic of the Congo', 0.7546376039613755),\n",
       " ('Nigeria', 0.7638226925640115),\n",
       " ('Kazakhstan', 0.7664058804565297),\n",
       " ('South Sudan', 0.7853758368697903),\n",
       " ('Mauritania', 0.7940621345124717),\n",
       " ('Yemen', 0.7944413765732171),\n",
       " ('Zambia', 0.7956472109009254),\n",
       " ('Mozambique', 0.8088590231701378),\n",
       " ('Angola', 0.8121648014989246),\n",
       " ('Democratic Republic of the Congo', 0.8219568435341607),\n",
       " ('Central African Republic', 0.833851023126543),\n",
       " ('Chad', 0.8416774248159262),\n",
       " ('Sudan', 0.8613396783076225),\n",
       " ('Western Sahara', 0.8651600275073515)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_geowiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1491e442",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    181.000000\n",
       "mean       0.471373\n",
       "std        0.218703\n",
       "min        0.000000\n",
       "25%        0.309278\n",
       "50%        0.494639\n",
       "75%        0.628910\n",
       "max        0.865160\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## get summary statistics of sm_geowiki\n",
    "pd.Series([sm[1] for sm in sm_geowiki]).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a2a5b910",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdm_geowiki = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_geowiki_geonames_grids_merged_admin0 = df_geowiki_geonames_grids_merged[df_geowiki_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_geowiki_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_geowiki_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    gc /= np.sum(gc)\n",
    "    gp /= np.sum(gp)\n",
    "    sdi_p = 0\n",
    "    for i in range(len(gp)):\n",
    "        if gp[i] != 0:\n",
    "            sdi_p -= gp[i]*np.log(gp[i])\n",
    "    sdi_c = 0\n",
    "    for i in range(len(gc)):\n",
    "        if gc[i] != 0:\n",
    "            sdi_c -= gc[i]*np.log(gc[i])\n",
    "    if (max(sdi_p, sdi_c) == 0):\n",
    "        continue\n",
    "    elif (np.isnan((sdi_p-sdi_c)/max(sdi_p, sdi_c)) == False):\n",
    "        sdm_geowiki[admin0] = (sdi_p-sdi_c)/max(sdi_p, sdi_c)\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "sdm_geowiki = sorted(sdm_geowiki.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2f537a6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Kashmir', -1.0),\n",
       " ('Kuwait', -0.6231406268051903),\n",
       " ('Western Sahara', -0.6116617632683333),\n",
       " ('The Bahamas', -0.5949032926407826),\n",
       " ('Mauritania', -0.5031359167936578),\n",
       " ('Chad', -0.4512511428131118),\n",
       " ('Republic of the Congo', -0.4489557868799401),\n",
       " ('Sudan', -0.44075278027908316),\n",
       " ('Turkmenistan', -0.42870762749452035),\n",
       " ('South Sudan', -0.4094576999747741),\n",
       " ('Mozambique', -0.4034487063079176),\n",
       " ('Oman', -0.39845617180118315),\n",
       " ('Madagascar', -0.39488095457185984),\n",
       " ('Angola', -0.3920090306960699),\n",
       " ('Kazakhstan', -0.3900324407987438),\n",
       " ('Sierra Leone', -0.3882137965493693),\n",
       " ('Democratic Republic of the Congo', -0.387402485571611),\n",
       " ('Laos', -0.3866074927622442),\n",
       " ('Somaliland', -0.38512340485237495),\n",
       " ('Saudi Arabia', -0.3824381245374625),\n",
       " ('Zambia', -0.38052525991574276),\n",
       " ('Venezuela', -0.3797528551252116),\n",
       " ('Central African Republic', -0.378800378798432),\n",
       " ('Guyana', -0.37785963757059743),\n",
       " ('Guinea-Bissau', -0.3754470124012201),\n",
       " ('Namibia', -0.37475705118626856),\n",
       " ('Gabon', -0.3688942582496375),\n",
       " ('Niger', -0.36862282878379005),\n",
       " ('Equatorial Guinea', -0.35890728027901536),\n",
       " ('United Arab Emirates', -0.35624109917155555),\n",
       " ('Solomon Islands', -0.3409978768234642),\n",
       " ('Egypt', -0.33995065222619114),\n",
       " ('North Korea', -0.3376422129972648),\n",
       " ('Eritrea', -0.33675650176371474),\n",
       " ('Malawi', -0.3343910808846368),\n",
       " ('Senegal', -0.3318428449806013),\n",
       " ('Qatar', -0.3270411043337966),\n",
       " ('Botswana', -0.32617597821488753),\n",
       " ('Libya', -0.3223679602051514),\n",
       " ('Zimbabwe', -0.31838874895590014),\n",
       " ('Somalia', -0.31684253298647974),\n",
       " ('Papua New Guinea', -0.3088128442751571),\n",
       " ('South Africa', -0.3077432808820052),\n",
       " ('Uruguay', -0.3066771246631398),\n",
       " ('Belarus', -0.30542693607218296),\n",
       " ('Suriname', -0.2960597449730987),\n",
       " ('Chile', -0.2921784962689075),\n",
       " ('Iraq', -0.29072090004195145),\n",
       " ('Cambodia', -0.2865319016233542),\n",
       " ('Malaysia', -0.28357117809015964),\n",
       " ('Greece', -0.2831385774858019),\n",
       " ('Paraguay', -0.2825064348749655),\n",
       " ('Djibouti', -0.27228124671526294),\n",
       " ('United Republic of Tanzania', -0.2713202268802658),\n",
       " ('Bir Tawil', -0.2705791120082078),\n",
       " ('Denmark', -0.27004930478690387),\n",
       " ('Cameroon', -0.2690503346107214),\n",
       " ('São Tomé and Principe', -0.26809427889637616),\n",
       " ('Nicaragua', -0.2666651632474122),\n",
       " ('Uzbekistan', -0.2628110508092099),\n",
       " ('Argentina', -0.26081893068820994),\n",
       " ('Mongolia', -0.2607034400564254),\n",
       " ('South Korea', -0.25789327881224),\n",
       " ('Liberia', -0.255652128825741),\n",
       " ('Montenegro', -0.25488796486969123),\n",
       " ('Russia', -0.25164399962570944),\n",
       " ('Togo', -0.24980890563070363),\n",
       " ('Fiji', -0.24884434723931473),\n",
       " ('Mali', -0.24775015462979333),\n",
       " ('Indonesia', -0.24518094609935054),\n",
       " ('Nigeria', -0.24516775940172317),\n",
       " ('Cuba', -0.2421106927941676),\n",
       " ('Uganda', -0.24198483260340672),\n",
       " ('Taiwan', -0.24097977578706078),\n",
       " ('Trinidad and Tobago', -0.2400549781902671),\n",
       " ('Antarctica', -0.23904627219594496),\n",
       " ('Belize', -0.2337081780328511),\n",
       " ('Mexico', -0.233297203481388),\n",
       " ('Mauritius', -0.23198755036833138),\n",
       " ('Austria', -0.23081860246487695),\n",
       " ('Myanmar', -0.2306902106053839),\n",
       " ('Thailand', -0.2297082330209401),\n",
       " ('Australia', -0.22784054911819795),\n",
       " ('China', -0.22772259471343118),\n",
       " ('Panama', -0.2274414944367626),\n",
       " ('Vanuatu', -0.2243656500467767),\n",
       " ('Ukraine', -0.22018339976655574),\n",
       " ('Iceland', -0.2188475963647648),\n",
       " ('Sri Lanka', -0.21680848662081206),\n",
       " ('Afghanistan', -0.21587485769681009),\n",
       " ('Colombia', -0.21567432350368174),\n",
       " ('Ethiopia', -0.2149563617363606),\n",
       " ('France', -0.21386387854677963),\n",
       " ('Rwanda', -0.21271816110438452),\n",
       " ('Costa Rica', -0.2126341971659556),\n",
       " ('Finland', -0.21162182996099155),\n",
       " ('Ecuador', -0.21152558577560224),\n",
       " ('Bolivia', -0.20960478937175764),\n",
       " ('Guinea', -0.20817294699374284),\n",
       " ('Brunei', -0.2072739581408916),\n",
       " ('Sweden', -0.20725465962893566),\n",
       " ('Tunisia', -0.20301149594485926),\n",
       " ('Tajikistan', -0.20281777220867486),\n",
       " ('Ghana', -0.20266562176357944),\n",
       " ('Georgia', -0.198696489696565),\n",
       " ('Jordan', -0.19750117756886554),\n",
       " ('Benin', -0.19675289513613795),\n",
       " ('Kyrgyzstan', -0.19569442832703127),\n",
       " ('Latvia', -0.19381010094132514),\n",
       " ('Dominican Republic', -0.192638968404465),\n",
       " ('Pakistan', -0.19227563933425068),\n",
       " ('Vietnam', -0.18778155468677388),\n",
       " ('Kenya', -0.18694200877723263),\n",
       " ('Canada', -0.1850186736875085),\n",
       " ('eSwatini', -0.18403140089764652),\n",
       " ('Honduras', -0.1812273931620349),\n",
       " ('Brazil', -0.17960763059865695),\n",
       " ('Guatemala', -0.17641398188500357),\n",
       " ('Morocco', -0.17637751456529607),\n",
       " ('Philippines', -0.17466303835577612),\n",
       " ('Belgium', -0.1725359797867555),\n",
       " ('Burkina Faso', -0.1716682115221569),\n",
       " ('Armenia', -0.1716245887304316),\n",
       " ('Burundi', -0.16998604137296966),\n",
       " ('Haiti', -0.16524628306963302),\n",
       " ('Peru', -0.16492225259129722),\n",
       " ('Lithuania', -0.1619717741865847),\n",
       " ('Bangladesh', -0.15707522052492595),\n",
       " ('Bhutan', -0.157001349238463),\n",
       " ('New Zealand', -0.15267712547958118),\n",
       " ('Algeria', -0.15056771559356902),\n",
       " ('North Macedonia', -0.14811332606399766),\n",
       " ('Italy', -0.14554768901896512),\n",
       " ('East Timor', -0.14288241341447197),\n",
       " ('Kosovo', -0.14248766055794346),\n",
       " ('Israel', -0.1421562119249627),\n",
       " ('Yemen', -0.14075462597271393),\n",
       " ('Syria', -0.136287876958622),\n",
       " ('Germany', -0.13460895848909693),\n",
       " ('Jamaica', -0.1320489948839202),\n",
       " ('Turkey', -0.12944227730126145),\n",
       " ('Norway', -0.12786003010621216),\n",
       " ('Ivory Coast', -0.12690256663349686),\n",
       " ('Gambia', -0.12059366137787542),\n",
       " ('Lebanon', -0.11839098361393778),\n",
       " ('Portugal', -0.11178372629211346),\n",
       " ('Lesotho', -0.1027943354078523),\n",
       " ('India', -0.09396601732157792),\n",
       " ('Albania', -0.0862420921534429),\n",
       " ('Bulgaria', -0.08295556618084987),\n",
       " ('Bosnia and Herzegovina', -0.07263660377962905),\n",
       " ('Ireland', -0.07046814510271501),\n",
       " ('United States of America', -0.06676193811946096),\n",
       " ('Romania', -0.05740001798077965),\n",
       " ('Azerbaijan', -0.05708789153459421),\n",
       " ('Hungary', -0.048463182424310164),\n",
       " ('Japan', -0.048427217546962935),\n",
       " ('United Kingdom', -0.04802659390236951),\n",
       " ('Iran', -0.047981734921604936),\n",
       " ('Croatia', -0.044358305536040483),\n",
       " ('Netherlands', -0.039402108312057205),\n",
       " ('Nepal', -0.03149202285832419),\n",
       " ('Cyprus', -0.03142391161870198),\n",
       " ('Cabo Verde', -0.03009805768546777),\n",
       " ('Luxembourg', -0.029286355920694115),\n",
       " ('Switzerland', -0.02927730412737657),\n",
       " ('Spain', -0.026972501020861593),\n",
       " ('Estonia', -0.021785784054904386),\n",
       " ('Slovenia', -0.015510987227792246),\n",
       " ('Slovakia', -0.015278275302927401),\n",
       " ('Czechia', -0.010387907018184324),\n",
       " ('Northern Cyprus', -0.009475312393347193),\n",
       " ('Moldova', -0.004462159651863369),\n",
       " ('Poland', 0.00155850839064273),\n",
       " ('El Salvador', 0.015928213608482587),\n",
       " ('Republic of Serbia', 0.017688573828060966),\n",
       " ('Samoa', 0.03267068898369207),\n",
       " ('Comoros', 0.42803862583577124)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdm_geowiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8db3535e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    178.000000\n",
       "mean      -0.223448\n",
       "std        0.146347\n",
       "min       -1.000000\n",
       "25%       -0.303085\n",
       "50%       -0.216342\n",
       "75%       -0.143549\n",
       "max        0.428039\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## get summary statistics of sdm_geowiki\n",
    "pd.Series([sdm[1] for sdm in sdm_geowiki]).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce1d932e",
   "metadata": {},
   "source": [
    "## Measuring Spatial (Diversity) Misalignment between WikToR and GeoNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17d134de",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_wiktor_grids = os.path.dirname(os.getcwd())+'\\\\data\\\\grids-100sqkm-admin0-natural-earth\\\\grids_summary_WikToR_poi.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4e158cd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>Join_Count</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>PageName</th>\n",
       "      <th>PageNumber</th>\n",
       "      <th>featurecla</th>\n",
       "      <th>scalerank</th>\n",
       "      <th>LABELRANK</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>SOV_A3</th>\n",
       "      <th>...</th>\n",
       "      <th>FCLASS_PL</th>\n",
       "      <th>FCLASS_GR</th>\n",
       "      <th>FCLASS_IT</th>\n",
       "      <th>FCLASS_NL</th>\n",
       "      <th>FCLASS_SE</th>\n",
       "      <th>FCLASS_BD</th>\n",
       "      <th>FCLASS_UA</th>\n",
       "      <th>Shape_Length</th>\n",
       "      <th>Shape_Area</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1456</td>\n",
       "      <td>A1456</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1457</td>\n",
       "      <td>A1457</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1458</td>\n",
       "      <td>A1458</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1459</td>\n",
       "      <td>A1459</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1461</td>\n",
       "      <td>A1461</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 169 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  Join_Count  TARGET_FID PageName  PageNumber featurecla  scalerank  \\\n",
       "0     1           0        1456    A1456           1        NaN        NaN   \n",
       "1     2           0        1457    A1457           2        NaN        NaN   \n",
       "2     3           0        1458    A1458           3        NaN        NaN   \n",
       "3     4           0        1459    A1459           4        NaN        NaN   \n",
       "4     5           0        1461    A1461           5        NaN        NaN   \n",
       "\n",
       "   LABELRANK SOVEREIGNT SOV_A3  ...  FCLASS_PL  FCLASS_GR FCLASS_IT FCLASS_NL  \\\n",
       "0        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "1        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "2        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "3        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "4        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "\n",
       "  FCLASS_SE  FCLASS_BD FCLASS_UA Shape_Length   Shape_Area Point_Count  \n",
       "0       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "1       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "2       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "3       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "4       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "\n",
       "[5 rows x 169 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_grids = pd.read_csv(dir_wiktor_grids)\n",
    "df_wiktor_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b02ba61d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count\n",
       "0     1        1456        NaN            0\n",
       "1     2        1457        NaN            0\n",
       "2     3        1458        NaN            0\n",
       "3     4        1459        NaN            0\n",
       "4     5        1461        NaN            0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_grids = df_wiktor_grids[['OID_','TARGET_FID','SOVEREIGNT','Point_Count']]\n",
    "df_wiktor_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d3afd835",
   "metadata": {},
   "outputs": [],
   "source": [
    "admin0_list = list(df_wiktor_grids['SOVEREIGNT'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "633c8cfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count_x</th>\n",
       "      <th>Point_Count_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count_x  Point_Count_y\n",
       "0     1        1456        NaN              1              0\n",
       "1     2        1457        NaN              0              0\n",
       "2     3        1458        NaN              0              0\n",
       "3     4        1459        NaN              0              0\n",
       "4     5        1461        NaN              1              0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wiktor_geonames_grids_merged = df_geonames_grids.merge(df_wiktor_grids, on=['OID_','SOVEREIGNT','TARGET_FID'])\n",
    "df_wiktor_geonames_grids_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a7bafc02",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_wiktor = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_wiktor_geonames_grids_merged_admin0 = df_wiktor_geonames_grids_merged[df_wiktor_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_wiktor_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_wiktor_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    if (np.isnan(1 - dot(gp, gc)/(norm(gp)*norm(gc))) == False):\n",
    "        sm_wiktor[admin0] = 1 - dot(gp, gc)/(norm(gp)*norm(gc))\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "sm_wiktor = sorted(sm_wiktor.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9971b9b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Australia', 0.4925566700743349),\n",
       " ('El Salvador', 0.4982565185722486),\n",
       " ('United Kingdom', 0.5079330828352608),\n",
       " ('Hungary', 0.6209303547660514),\n",
       " ('Philippines', 0.6520331630934006),\n",
       " ('United States of America', 0.6739461461903019),\n",
       " ('Israel', 0.7030762400040729),\n",
       " ('France', 0.7359596275426319),\n",
       " ('Switzerland', 0.7744039092126801),\n",
       " ('Argentina', 0.783506566178089),\n",
       " ('Lebanon', 0.7844801001497417),\n",
       " ('Ireland', 0.8000126012256825),\n",
       " ('Cyprus', 0.801960034836821),\n",
       " ('Egypt', 0.8034375539249907),\n",
       " ('Chile', 0.8035242039563254),\n",
       " ('Luxembourg', 0.8035749420295191),\n",
       " ('Afghanistan', 0.8080521811211696),\n",
       " ('Bhutan', 0.8101149346889834),\n",
       " ('Taiwan', 0.8112574017379999),\n",
       " ('Nicaragua', 0.8129342562577387),\n",
       " ('Uganda', 0.8198756923360152),\n",
       " ('Slovenia', 0.825112558123186),\n",
       " ('Cuba', 0.8263644925810846),\n",
       " ('Costa Rica', 0.8326663729814856),\n",
       " ('Iraq', 0.8334747302527087),\n",
       " ('Sri Lanka', 0.8340918735169158),\n",
       " ('Spain', 0.8382227444695993),\n",
       " ('Panama', 0.8421377777851042),\n",
       " ('Canada', 0.8468154579851834),\n",
       " ('Sierra Leone', 0.8631975550774477),\n",
       " ('Greece', 0.8663658173669035),\n",
       " ('Qatar', 0.8670453279697827),\n",
       " ('Malaysia', 0.8690276353672747),\n",
       " ('Croatia', 0.8695035828569908),\n",
       " ('New Zealand', 0.871235434250251),\n",
       " ('South Africa', 0.8758860908014292),\n",
       " ('East Timor', 0.8762325367567302),\n",
       " ('Belgium', 0.8768208885450163),\n",
       " ('Jamaica', 0.8768688938861214),\n",
       " ('Eritrea', 0.8779891459088475),\n",
       " ('Armenia', 0.8781229780017854),\n",
       " ('South Korea', 0.8787520512120659),\n",
       " ('Uruguay', 0.8796370398654056),\n",
       " ('Trinidad and Tobago', 0.8879310344827587),\n",
       " ('Pakistan', 0.888713644115729),\n",
       " ('Albania', 0.8926518098635181),\n",
       " ('Namibia', 0.8944160476581273),\n",
       " ('Lesotho', 0.8953782561703282),\n",
       " ('Netherlands', 0.8963424731884085),\n",
       " ('Montenegro', 0.8982144354777918),\n",
       " ('Nepal', 0.8982443142273014),\n",
       " ('Djibouti', 0.899359198940071),\n",
       " ('Bolivia', 0.8999484480136322),\n",
       " ('Guatemala', 0.9034685545299497),\n",
       " ('Georgia', 0.9050981116708903),\n",
       " ('Guyana', 0.9051539273759975),\n",
       " ('Saudi Arabia', 0.9071748565443098),\n",
       " ('Portugal', 0.9092596566140028),\n",
       " ('Norway', 0.9097575662576515),\n",
       " ('Benin', 0.9143788857642207),\n",
       " ('Myanmar', 0.9146180838877411),\n",
       " ('Kosovo', 0.9151518701892728),\n",
       " ('North Macedonia', 0.9175070154086025),\n",
       " ('Dominican Republic', 0.9187824167024519),\n",
       " ('Kenya', 0.920740607990933),\n",
       " ('Estonia', 0.9213169185515664),\n",
       " ('Poland', 0.9242647787855552),\n",
       " ('Honduras', 0.928279000890835),\n",
       " ('Thailand', 0.9288889838374823),\n",
       " ('Sweden', 0.9298992027622586),\n",
       " ('Syria', 0.9305287532909281),\n",
       " ('Mexico', 0.9337132461773158),\n",
       " ('Slovakia', 0.9352735768014567),\n",
       " ('Venezuela', 0.9364068866091473),\n",
       " ('China', 0.9381163759112865),\n",
       " ('Italy', 0.9383976845067241),\n",
       " ('United Republic of Tanzania', 0.939333348016862),\n",
       " ('Bulgaria', 0.9395847168898859),\n",
       " ('Japan', 0.9398265377098978),\n",
       " ('Germany', 0.9412731978489446),\n",
       " ('Sudan', 0.9431113304680433),\n",
       " ('Bosnia and Herzegovina', 0.9440369504256468),\n",
       " ('Turkey', 0.9456798789961444),\n",
       " ('Brazil', 0.9457916579318391),\n",
       " ('Peru', 0.9458977572520806),\n",
       " ('India', 0.9475502426675754),\n",
       " ('Mali', 0.9492633756997354),\n",
       " ('Malawi', 0.9500575926472575),\n",
       " ('Indonesia', 0.9525951035367912),\n",
       " ('Papua New Guinea', 0.954795552601173),\n",
       " ('Ethiopia', 0.9557221878753656),\n",
       " ('South Sudan', 0.9560185185185185),\n",
       " ('Colombia', 0.9561338726193245),\n",
       " ('Nigeria', 0.956607488033917),\n",
       " ('Russia', 0.9587372119753927),\n",
       " ('Cameroon', 0.959113880310435),\n",
       " ('Turkmenistan', 0.9595516852046058),\n",
       " ('Morocco', 0.9597286953556355),\n",
       " ('Togo', 0.960844783744096),\n",
       " ('Republic of the Congo', 0.9625273626004855),\n",
       " ('Paraguay', 0.9625534337530801),\n",
       " ('Liberia', 0.9645246671663716),\n",
       " ('Finland', 0.9657191892113832),\n",
       " ('Brunei', 0.9668594260963984),\n",
       " ('Zambia', 0.9712042216584486),\n",
       " ('Vanuatu', 0.9746214554057454),\n",
       " ('Ukraine', 0.9749299022828732),\n",
       " ('Angola', 0.9750003160261312),\n",
       " ('Ghana', 0.9753432754467911),\n",
       " ('Kazakhstan', 0.9764439448917926),\n",
       " ('Iran', 0.9765096308595146),\n",
       " ('Zimbabwe', 0.9768937767908262),\n",
       " ('Mozambique', 0.9797492179114462),\n",
       " ('Democratic Republic of the Congo', 0.9804963158962516),\n",
       " ('Romania', 0.9810408245322433),\n",
       " ('Yemen', 0.9825400515965365),\n",
       " ('Guinea', 0.9860641557936174),\n",
       " ('Belarus', 0.9906539807009637),\n",
       " ('Algeria', 0.9918298716493915),\n",
       " ('Republic of Serbia', 0.9918761208979939),\n",
       " ('Libya', 0.992358642674868),\n",
       " ('Somalia', 0.9963703973259204),\n",
       " ('Jordan', 0.9971668429898696),\n",
       " ('Gabon', 1.0)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_wiktor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8547a382",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    124.000000\n",
       "mean       0.892736\n",
       "std        0.096244\n",
       "min        0.492557\n",
       "25%        0.866875\n",
       "50%        0.916329\n",
       "75%        0.956252\n",
       "max        1.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([sm[1] for sm in sm_wiktor]).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0759b433",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdm_wiktor = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_wiktor_geonames_grids_merged_admin0 = df_wiktor_geonames_grids_merged[df_wiktor_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_wiktor_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_wiktor_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    gc /= np.sum(gc)\n",
    "    gp /= np.sum(gp)\n",
    "    sdi_p = 0\n",
    "    for i in range(len(gp)):\n",
    "        if gp[i] != 0:\n",
    "            sdi_p -= gp[i]*np.log(gp[i])\n",
    "    sdi_c = 0\n",
    "    for i in range(len(gc)):\n",
    "        if gc[i] != 0:\n",
    "            sdi_c -= gc[i]*np.log(gc[i])\n",
    "    if (max(sdi_p, sdi_c) == 0):\n",
    "        continue\n",
    "    elif (np.isnan((sdi_p-sdi_c)/max(sdi_p, sdi_c)) == False):\n",
    "        sdm_wiktor[admin0] = (sdi_p-sdi_c)/max(sdi_p, sdi_c)\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "sdm_wiktor = sorted(sdm_wiktor.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b24c3c5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Albania', -1.0),\n",
       " ('Algeria', -1.0),\n",
       " ('Belarus', -1.0),\n",
       " ('Benin', -1.0),\n",
       " ('Bhutan', -1.0),\n",
       " ('Bosnia and Herzegovina', -1.0),\n",
       " ('Brunei', -1.0),\n",
       " ('Croatia', -1.0),\n",
       " ('Cyprus', -1.0),\n",
       " ('Djibouti', -1.0),\n",
       " ('East Timor', -1.0),\n",
       " ('Ghana', -1.0),\n",
       " ('Guinea', -1.0),\n",
       " ('Guyana', -1.0),\n",
       " ('Hungary', -1.0),\n",
       " ('Kosovo', -1.0),\n",
       " ('Lesotho', -1.0),\n",
       " ('Liberia', -1.0),\n",
       " ('Libya', -1.0),\n",
       " ('Luxembourg', -1.0),\n",
       " ('Malawi', -1.0),\n",
       " ('Montenegro', -1.0),\n",
       " ('Mozambique', -1.0),\n",
       " ('Namibia', -1.0),\n",
       " ('North Macedonia', -1.0),\n",
       " ('Qatar', -1.0),\n",
       " ('Republic of the Congo', -1.0),\n",
       " ('Romania', -1.0),\n",
       " ('Slovakia', -1.0),\n",
       " ('Somalia', -1.0),\n",
       " ('Togo', -1.0),\n",
       " ('Trinidad and Tobago', -1.0),\n",
       " ('Turkmenistan', -1.0),\n",
       " ('Vanuatu', -1.0),\n",
       " ('Iran', -0.9242115430174368),\n",
       " ('Saudi Arabia', -0.9214006080091494),\n",
       " ('United Republic of Tanzania', -0.9161335217942945),\n",
       " ('Sweden', -0.9132667455903506),\n",
       " ('Zimbabwe', -0.9111344036324205),\n",
       " ('Paraguay', -0.9014280389669703),\n",
       " ('Gabon', -0.9011757216231875),\n",
       " ('Portugal', -0.8911651036723524),\n",
       " ('Eritrea', -0.8897278826683698),\n",
       " ('Georgia', -0.8873469774155247),\n",
       " ('Kazakhstan', -0.8864398498089834),\n",
       " ('Dominican Republic', -0.8822596687599),\n",
       " ('Estonia', -0.8790038118499944),\n",
       " ('Angola', -0.874502500980139),\n",
       " ('Sudan', -0.8718933021269741),\n",
       " ('Netherlands', -0.8714634012951233),\n",
       " ('Ukraine', -0.8694843492309158),\n",
       " ('Myanmar', -0.8688932985534571),\n",
       " ('Venezuela', -0.8668921380419734),\n",
       " ('Thailand', -0.8664244523284742),\n",
       " ('South Sudan', -0.8597496283822232),\n",
       " ('Jordan', -0.8566787923653284),\n",
       " ('Morocco', -0.8565059968596973),\n",
       " ('Papua New Guinea', -0.8540572437578776),\n",
       " ('Uruguay', -0.8457732926940914),\n",
       " ('Egypt', -0.8410427330350612),\n",
       " ('Jamaica', -0.8390514290489561),\n",
       " ('Yemen', -0.8355505463191264),\n",
       " ('Lebanon', -0.8316331875269343),\n",
       " ('South Korea', -0.8292322017693845),\n",
       " ('Democratic Republic of the Congo', -0.8265716423800814),\n",
       " ('Cameroon', -0.8211129341799565),\n",
       " ('Republic of Serbia', -0.8190710505466796),\n",
       " ('Finland', -0.817797010270627),\n",
       " ('Mali', -0.810643418286054),\n",
       " ('Peru', -0.8070882494752791),\n",
       " ('Poland', -0.7953775171847781),\n",
       " ('Bulgaria', -0.7902875652740617),\n",
       " ('Armenia', -0.7876607118429623),\n",
       " ('Bolivia', -0.7844809062337204),\n",
       " ('Afghanistan', -0.782207889233328),\n",
       " ('Nicaragua', -0.7799308047386599),\n",
       " ('Iraq', -0.7774452596095492),\n",
       " ('Nepal', -0.7653110858422846),\n",
       " ('Zambia', -0.7633474632514566),\n",
       " ('Norway', -0.7556035791203369),\n",
       " ('Guatemala', -0.7524933423610001),\n",
       " ('France', -0.7520468249230897),\n",
       " ('Kenya', -0.7463810184685071),\n",
       " ('Sierra Leone', -0.7435529660984423),\n",
       " ('Indonesia', -0.7420215654635445),\n",
       " ('Belgium', -0.7419672105003494),\n",
       " ('Sri Lanka', -0.7349327577887045),\n",
       " ('Panama', -0.7285400658200791),\n",
       " ('Uganda', -0.7285137270314647),\n",
       " ('Germany', -0.7242119182255997),\n",
       " ('Argentina', -0.7162437052246577),\n",
       " ('Costa Rica', -0.7042778239783608),\n",
       " ('Turkey', -0.7038383639165195),\n",
       " ('Syria', -0.6997794580091161),\n",
       " ('Taiwan', -0.6936525034820514),\n",
       " ('Honduras', -0.6905054650182083),\n",
       " ('Cuba', -0.682301062680423),\n",
       " ('Italy', -0.6784260733707939),\n",
       " ('Japan', -0.6782502602631221),\n",
       " ('Greece', -0.6779608266253304),\n",
       " ('Chile', -0.6735561390914349),\n",
       " ('Colombia', -0.6700200104178569),\n",
       " ('Nigeria', -0.6562026933667625),\n",
       " ('Pakistan', -0.6538645147123524),\n",
       " ('Russia', -0.6533063216604166),\n",
       " ('Spain', -0.6524725303677634),\n",
       " ('Brazil', -0.652465417278172),\n",
       " ('Malaysia', -0.6507223544653791),\n",
       " ('South Africa', -0.6494079949322327),\n",
       " ('Ireland', -0.6455674080546938),\n",
       " ('Mexico', -0.6283235779231214),\n",
       " ('Ethiopia', -0.6191976485362768),\n",
       " ('New Zealand', -0.5955855420187366),\n",
       " ('Slovenia', -0.5700697094330222),\n",
       " ('China', -0.5696217802397837),\n",
       " ('Israel', -0.5637110567902801),\n",
       " ('Canada', -0.5259925097986516),\n",
       " ('Australia', -0.5105532686097681),\n",
       " ('India', -0.5096429891699138),\n",
       " ('El Salvador', -0.47712128287428945),\n",
       " ('Switzerland', -0.46942322245670104),\n",
       " ('Philippines', -0.3250876784078021),\n",
       " ('United States of America', -0.27327192103854286),\n",
       " ('United Kingdom', -0.2616890789790998)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdm_wiktor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e0180c08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    124.000000\n",
       "mean      -0.812131\n",
       "std        0.166701\n",
       "min       -1.000000\n",
       "25%       -1.000000\n",
       "50%       -0.833592\n",
       "75%       -0.702824\n",
       "max       -0.261689\n",
       "dtype: float64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([sdm[1] for sdm in sdm_wiktor]).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e56418f3",
   "metadata": {},
   "source": [
    "## Measuring Spatial (Diversity) Misalignment between GeoCorpora and GeoNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2d3c018c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_geocorpora_grids = os.path.dirname(os.getcwd())+'\\\\data\\\\grids-100sqkm-admin0-natural-earth\\\\grids_summary_GeoCorpora_poi.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "dbf97b51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>Join_Count</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>PageName</th>\n",
       "      <th>PageNumber</th>\n",
       "      <th>featurecla</th>\n",
       "      <th>scalerank</th>\n",
       "      <th>LABELRANK</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>SOV_A3</th>\n",
       "      <th>...</th>\n",
       "      <th>FCLASS_PL</th>\n",
       "      <th>FCLASS_GR</th>\n",
       "      <th>FCLASS_IT</th>\n",
       "      <th>FCLASS_NL</th>\n",
       "      <th>FCLASS_SE</th>\n",
       "      <th>FCLASS_BD</th>\n",
       "      <th>FCLASS_UA</th>\n",
       "      <th>Shape_Length</th>\n",
       "      <th>Shape_Area</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1456</td>\n",
       "      <td>A1456</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1457</td>\n",
       "      <td>A1457</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1458</td>\n",
       "      <td>A1458</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1459</td>\n",
       "      <td>A1459</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1461</td>\n",
       "      <td>A1461</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 169 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  Join_Count  TARGET_FID PageName  PageNumber featurecla  scalerank  \\\n",
       "0     1           0        1456    A1456           1        NaN        NaN   \n",
       "1     2           0        1457    A1457           2        NaN        NaN   \n",
       "2     3           0        1458    A1458           3        NaN        NaN   \n",
       "3     4           0        1459    A1459           4        NaN        NaN   \n",
       "4     5           0        1461    A1461           5        NaN        NaN   \n",
       "\n",
       "   LABELRANK SOVEREIGNT SOV_A3  ...  FCLASS_PL  FCLASS_GR FCLASS_IT FCLASS_NL  \\\n",
       "0        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "1        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "2        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "3        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "4        NaN        NaN    NaN  ...        NaN        NaN       NaN       NaN   \n",
       "\n",
       "  FCLASS_SE  FCLASS_BD FCLASS_UA Shape_Length   Shape_Area Point_Count  \n",
       "0       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "1       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "2       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "3       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "4       NaN        NaN       NaN      40000.0  100000000.0           0  \n",
       "\n",
       "[5 rows x 169 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geocorpora_grids = pd.read_csv(dir_geocorpora_grids)\n",
    "df_geocorpora_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "076b704e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count\n",
       "0     1        1456        NaN            0\n",
       "1     2        1457        NaN            0\n",
       "2     3        1458        NaN            0\n",
       "3     4        1459        NaN            0\n",
       "4     5        1461        NaN            0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geocorpora_grids = df_geocorpora_grids[['OID_','TARGET_FID','SOVEREIGNT','Point_Count']]\n",
    "df_geocorpora_grids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2a75b976",
   "metadata": {},
   "outputs": [],
   "source": [
    "admin0_list = list(df_geocorpora_grids['SOVEREIGNT'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "155851dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OID_</th>\n",
       "      <th>TARGET_FID</th>\n",
       "      <th>SOVEREIGNT</th>\n",
       "      <th>Point_Count_x</th>\n",
       "      <th>Point_Count_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1461</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OID_  TARGET_FID SOVEREIGNT  Point_Count_x  Point_Count_y\n",
       "0     1        1456        NaN              1              0\n",
       "1     2        1457        NaN              0              0\n",
       "2     3        1458        NaN              0              0\n",
       "3     4        1459        NaN              0              0\n",
       "4     5        1461        NaN              1              0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_geocorpora_geonames_grids_merged = df_geonames_grids.merge(df_geocorpora_grids, on=['OID_','SOVEREIGNT','TARGET_FID'])\n",
    "df_geocorpora_geonames_grids_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4c719e21",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_geocorpora = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_geocorpora_geonames_grids_merged_admin0 = df_geocorpora_geonames_grids_merged[df_geocorpora_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_geocorpora_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_geocorpora_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    if (np.isnan(1 - dot(gp, gc)/(norm(gp)*norm(gc))) == False):\n",
    "        sm_geocorpora[admin0] = 1 - dot(gp, gc)/(norm(gp)*norm(gc))\n",
    "    else:\n",
    "        continue\n",
    "        \n",
    "sm_geocorpora = sorted(sm_geocorpora.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ad831bec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Greece', 0.35083925393247495),\n",
       " ('United Kingdom', 0.38757852137981086),\n",
       " ('France', 0.39938106829190745),\n",
       " ('Australia', 0.5071440807727251),\n",
       " ('Syria', 0.5126939341159864),\n",
       " ('Italy', 0.6646059448149255),\n",
       " ('United States of America', 0.7045352378950709),\n",
       " ('Iraq', 0.7221713886667772),\n",
       " ('Jordan', 0.7338413690503784),\n",
       " ('Egypt', 0.7384502236613346),\n",
       " ('Spain', 0.7596415237187403),\n",
       " ('Republic of Serbia', 0.7775027653823088),\n",
       " ('Israel', 0.786623989516731),\n",
       " ('Kuwait', 0.8047200527442997),\n",
       " ('Malaysia', 0.8165033265875271),\n",
       " ('Philippines', 0.8248085943546257),\n",
       " ('Cyprus', 0.8302514584315609),\n",
       " ('Libya', 0.8540500750899789),\n",
       " ('Ireland', 0.8556545562413276),\n",
       " ('Belgium', 0.8596239775561308),\n",
       " ('Costa Rica', 0.8650672422911417),\n",
       " ('Lebanon', 0.8704073119265568),\n",
       " ('Ukraine', 0.8806574510030248),\n",
       " ('Pakistan', 0.8814053045822475),\n",
       " ('Qatar', 0.8828732651162372),\n",
       " ('Netherlands', 0.8851494069129785),\n",
       " ('Bangladesh', 0.8917012486222581),\n",
       " ('Liberia', 0.8954797982853817),\n",
       " ('Canada', 0.8960703780230498),\n",
       " ('New Zealand', 0.9003253526217418),\n",
       " ('Nepal', 0.9031022193719778),\n",
       " ('Germany', 0.9050087209446063),\n",
       " ('Brazil', 0.90531697369787),\n",
       " ('Russia', 0.9070482281266193),\n",
       " ('Japan', 0.9094108234331691),\n",
       " ('Georgia', 0.919698402183061),\n",
       " ('Iran', 0.9227853488003093),\n",
       " ('Haiti', 0.9248209820262407),\n",
       " ('Slovenia', 0.9252518948888038),\n",
       " ('Nigeria', 0.9261443166326451),\n",
       " ('South Africa', 0.9271276297597946),\n",
       " ('Rwanda', 0.9272480092852085),\n",
       " ('Sierra Leone', 0.928266644124298),\n",
       " ('Belarus', 0.9292372824501536),\n",
       " ('South Korea', 0.9314994003464437),\n",
       " ('Panama', 0.9327636838187268),\n",
       " ('Guatemala', 0.9335731568267526),\n",
       " ('Albania', 0.9414464417437372),\n",
       " ('Mexico', 0.9484341000627203),\n",
       " ('Croatia', 0.9489361845962138),\n",
       " ('Dominican Republic', 0.9489515520877515),\n",
       " ('Mongolia', 0.9504220942895154),\n",
       " ('Malawi', 0.9519784544685169),\n",
       " ('China', 0.9558002958369713),\n",
       " ('India', 0.9563439198839698),\n",
       " ('Azerbaijan', 0.9577986854629716),\n",
       " ('Switzerland', 0.9611611444483821),\n",
       " ('Fiji', 0.961223131156219),\n",
       " ('Yemen', 0.9626586504947824),\n",
       " ('Chile', 0.965352144553134),\n",
       " ('Turkey', 0.9667624771624742),\n",
       " ('Namibia', 0.9692580058354172),\n",
       " ('South Sudan', 0.9699074074074074),\n",
       " ('Kenya', 0.9710852547657611),\n",
       " ('Myanmar', 0.9716059231832289),\n",
       " ('Taiwan', 0.971790643613533),\n",
       " ('Burkina Faso', 0.9731371145872989),\n",
       " ('Cuba', 0.9754442310498636),\n",
       " ('Poland', 0.9779846166195573),\n",
       " ('Democratic Republic of the Congo', 0.9788820049805247),\n",
       " ('El Salvador', 0.980154054385152),\n",
       " ('Norway', 0.9831017089149175),\n",
       " ('United Republic of Tanzania', 0.9841119255521625),\n",
       " ('Afghanistan', 0.9845365872232482),\n",
       " ('Zimbabwe', 0.9854768292277184),\n",
       " ('Somalia', 0.9854815893036812),\n",
       " ('Niger', 0.9855825454677227),\n",
       " ('Cambodia', 0.9861904328138348),\n",
       " ('Ivory Coast', 0.986425026677978),\n",
       " ('Central African Republic', 0.9869580625197466),\n",
       " ('Sri Lanka', 0.987166033337572),\n",
       " ('Honduras', 0.9876781865309132),\n",
       " ('Romania', 0.9878119586278707),\n",
       " ('Togo', 0.9882534351232288),\n",
       " ('Zambia', 0.9885720297549573),\n",
       " ('Saudi Arabia', 0.9888750358640453),\n",
       " ('Ethiopia', 0.9888895318371932),\n",
       " ('Sweden', 0.9893017088287647),\n",
       " ('Hungary', 0.9895284628388412),\n",
       " ('Sudan', 0.9898037893293334),\n",
       " ('Uganda', 0.990701187840847),\n",
       " ('Senegal', 0.9914586398710526),\n",
       " ('Thailand', 0.9918999838278041),\n",
       " ('Venezuela', 0.9922553239793749),\n",
       " ('Tunisia', 0.9930054482732288),\n",
       " ('Guinea', 0.9930320778968087),\n",
       " ('Algeria', 0.9931724553565058),\n",
       " ('Botswana', 0.993231955817197),\n",
       " ('Mozambique', 0.9942140622604132),\n",
       " ('Indonesia', 0.9947760226327556),\n",
       " ('Armenia', 0.9949377171612447),\n",
       " ('Mali', 0.995430183698658),\n",
       " ('Benin', 0.9957189442882111),\n",
       " ('Peru', 0.9959674569160153),\n",
       " ('Iceland', 0.9962717828434586),\n",
       " ('Colombia', 0.9965907803182348),\n",
       " ('Nicaragua', 0.9970770977540272),\n",
       " ('Argentina', 0.9980317477376631),\n",
       " ('Ghana', 0.9986301819692661),\n",
       " ('Angola', 0.9989438848093681)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_geocorpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "96047986",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    110.000000\n",
       "mean       0.910579\n",
       "std        0.128534\n",
       "min        0.350839\n",
       "25%        0.895627\n",
       "50%        0.957071\n",
       "75%        0.987779\n",
       "max        0.998944\n",
       "dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([sm[1] for sm in sm_geocorpora]).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1acfc2dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdm_geocorpora = {}\n",
    "for admin0 in admin0_list:\n",
    "    df_geocorpora_geonames_grids_merged_admin0 = df_geocorpora_geonames_grids_merged[df_geocorpora_geonames_grids_merged['SOVEREIGNT'] == admin0]\n",
    "    gc = list(df_geocorpora_geonames_grids_merged_admin0['Point_Count_x'].values)\n",
    "    gp = list(df_geocorpora_geonames_grids_merged_admin0['Point_Count_y'].values)\n",
    "    gc /= np.sum(gc)\n",
    "    gp /= np.sum(gp)\n",
    "    sdi_p = 0\n",
    "    for i in range(len(gp)):\n",
    "        if gp[i] != 0:\n",
    "            sdi_p -= gp[i]*np.log(gp[i])\n",
    "    sdi_c = 0\n",
    "    for i in range(len(gc)):\n",
    "        if gc[i] != 0:\n",
    "            sdi_c -= gc[i]*np.log(gc[i])\n",
    "    if (max(sdi_p, sdi_c) == 0):\n",
    "        continue\n",
    "    elif (np.isnan((sdi_p-sdi_c)/max(sdi_p, sdi_c)) == False):\n",
    "        sdm_geocorpora[admin0] = (sdi_p-sdi_c)/max(sdi_p, sdi_c)\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "sdm_geocorpora = sorted(sdm_geocorpora.items(), key = lambda kv:(kv[1], kv[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bbb8d52b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Albania', -1.0),\n",
       " ('Angola', -1.0),\n",
       " ('Argentina', -1.0),\n",
       " ('Armenia', -1.0),\n",
       " ('Azerbaijan', -1.0),\n",
       " ('Belarus', -1.0),\n",
       " ('Benin', -1.0),\n",
       " ('Botswana', -1.0),\n",
       " ('Burkina Faso', -1.0),\n",
       " ('Colombia', -1.0),\n",
       " ('Costa Rica', -1.0),\n",
       " ('Croatia', -1.0),\n",
       " ('Cuba', -1.0),\n",
       " ('Cyprus', -1.0),\n",
       " ('Dominican Republic', -1.0),\n",
       " ('El Salvador', -1.0),\n",
       " ('Ethiopia', -1.0),\n",
       " ('Fiji', -1.0),\n",
       " ('Ghana', -1.0),\n",
       " ('Greece', -1.0),\n",
       " ('Guinea', -1.0),\n",
       " ('Haiti', -1.0),\n",
       " ('Honduras', -1.0),\n",
       " ('Hungary', -1.0),\n",
       " ('Ivory Coast', -1.0),\n",
       " ('Jordan', -1.0),\n",
       " ('Kuwait', -1.0),\n",
       " ('Lebanon', -1.0),\n",
       " ('Malawi', -1.0),\n",
       " ('Mongolia', -1.0),\n",
       " ('Mozambique', -1.0),\n",
       " ('Myanmar', -1.0),\n",
       " ('Nicaragua', -1.0),\n",
       " ('Niger', -1.0),\n",
       " ('Norway', -1.0),\n",
       " ('Panama', -1.0),\n",
       " ('Peru', -1.0),\n",
       " ('Poland', -1.0),\n",
       " ('Qatar', -1.0),\n",
       " ('Republic of Serbia', -1.0),\n",
       " ('Romania', -1.0),\n",
       " ('Rwanda', -1.0),\n",
       " ('Saudi Arabia', -1.0),\n",
       " ('Senegal', -1.0),\n",
       " ('Slovenia', -1.0),\n",
       " ('Somalia', -1.0),\n",
       " ('Sri Lanka', -1.0),\n",
       " ('Sweden', -1.0),\n",
       " ('Switzerland', -1.0),\n",
       " ('Taiwan', -1.0),\n",
       " ('Thailand', -1.0),\n",
       " ('Togo', -1.0),\n",
       " ('Tunisia', -1.0),\n",
       " ('Uganda', -1.0),\n",
       " ('United Republic of Tanzania', -1.0),\n",
       " ('Venezuela', -1.0),\n",
       " ('Zambia', -1.0),\n",
       " ('Zimbabwe', -1.0),\n",
       " ('Turkey', -0.9199658260526248),\n",
       " ('Sudan', -0.9191736726801185),\n",
       " ('Afghanistan', -0.9157465105563767),\n",
       " ('Algeria', -0.9146194743564244),\n",
       " ('Namibia', -0.9135445576818195),\n",
       " ('Central African Republic', -0.9112096443097195),\n",
       " ('Kenya', -0.9096590959914654),\n",
       " ('Russia', -0.90228343783732),\n",
       " ('Cambodia', -0.8988095199453349),\n",
       " ('Bangladesh', -0.8966149036889687),\n",
       " ('France', -0.8965515246424399),\n",
       " ('Guatemala', -0.8934046845877858),\n",
       " ('South Korea', -0.8922575151444173),\n",
       " ('Liberia', -0.8904307665133622),\n",
       " ('Iceland', -0.8900150586514943),\n",
       " ('Sierra Leone', -0.8895542741111464),\n",
       " ('Georgia', -0.8873469774155247),\n",
       " ('Brazil', -0.8854194686582221),\n",
       " ('Democratic Republic of the Congo', -0.8816167287891129),\n",
       " ('Netherlands', -0.8714634012951233),\n",
       " ('Belgium', -0.8709836052501747),\n",
       " ('Mali', -0.8707440243553689),\n",
       " ('Chile', -0.8641047948713197),\n",
       " ('Spain', -0.8622954981655988),\n",
       " ('South Sudan', -0.8597496283822232),\n",
       " ('New Zealand', -0.8540675254193575),\n",
       " ('Yemen', -0.8355505463191264),\n",
       " ('Germany', -0.8259970935281327),\n",
       " ('Libya', -0.8150677196614148),\n",
       " ('Italy', -0.8140885775011346),\n",
       " ('Egypt', -0.7994182614508376),\n",
       " ('Mexico', -0.7949021613543928),\n",
       " ('China', -0.7914361452245854),\n",
       " ('Iraq', -0.7774452596095492),\n",
       " ('Nepal', -0.772416550907136),\n",
       " ('Australia', -0.7655066343243739),\n",
       " ('Indonesia', -0.7585876573397888),\n",
       " ('Ukraine', -0.7572678556567106),\n",
       " ('Nigeria', -0.756971102381781),\n",
       " ('Iran', -0.7515947350381633),\n",
       " ('Canada', -0.725467937625598),\n",
       " ('Philippines', -0.7049653429803846),\n",
       " ('Ireland', -0.6812809099720187),\n",
       " ('South Africa', -0.6755882189687666),\n",
       " ('Pakistan', -0.6698821567317608),\n",
       " ('Japan', -0.6649321745864525),\n",
       " ('India', -0.6320958144516937),\n",
       " ('Israel', -0.6207268069176274),\n",
       " ('Syria', -0.6107118510992879),\n",
       " ('Malaysia', -0.5990459963383357),\n",
       " ('United States of America', -0.47155249330730986),\n",
       " ('United Kingdom', -0.4318350440857098)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdm_geocorpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4058520a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    110.000000\n",
       "mean      -0.906054\n",
       "std        0.127746\n",
       "min       -1.000000\n",
       "25%       -1.000000\n",
       "50%       -1.000000\n",
       "75%       -0.860386\n",
       "max       -0.431835\n",
       "dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([sdm[1] for sdm in sdm_geocorpora]).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3392a3db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}