first_experiments_drop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T11:59:10.969449Z",
     "start_time": "2018-06-18T11:59:10.952075Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T11:59:11.976911Z",
     "start_time": "2018-06-18T11:59:10.970881Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from math import ceil\n",
    "from CIoTS import *\n",
    "from tqdm import trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T11:59:12.004112Z",
     "start_time": "2018-06-18T11:59:11.978469Z"
    }
   },
   "outputs": [],
   "source": [
    "runs = 20\n",
    "max_p = 22\n",
    "dimensions = 3\n",
    "data_length = 10000\n",
    "alpha = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T11:59:12.030464Z",
     "start_time": "2018-06-18T11:59:12.005666Z"
    }
   },
   "outputs": [],
   "source": [
    "ics = [\"aic\", \"bic\", \"hqic\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T12:33:16.072889Z",
     "start_time": "2018-06-18T12:33:16.042249Z"
    }
   },
   "outputs": [],
   "source": [
    "p_estimations = pd.DataFrame(columns=['p' ,'method', 'mean_p', 'std_p'])\n",
    "scores = pd.DataFrame(columns=['p' ,'method', 'mean_f1', 'std_f1', 'mean_recall', 'std_recall',\n",
    "                               'mean_precision', 'std_precision', 'mean_fpr', 'std_fpr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T16:11:42.088739Z",
     "start_time": "2018-06-18T12:33:16.940040Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for p in trange(2, max_p, 2):\n",
    "    incoming_edges = 3\n",
    "    f1 = {ic: [] for ic in ics}\n",
    "    f1.update({'incremental_' + ic: [] for ic in ics})\n",
    "    f1['real'] = []\n",
    "    \n",
    "    precision = {ic: [] for ic in ics}\n",
    "    precision.update({'incremental_' + ic: [] for ic in ics})\n",
    "    precision['real'] = []\n",
    "    \n",
    "    recall = {ic: [] for ic in ics}\n",
    "    recall.update({'incremental_' + ic: [] for ic in ics})\n",
    "    recall['real'] = []\n",
    "    \n",
    "    fpr = {ic: [] for ic in ics}\n",
    "    fpr.update({'incremental_' + ic: [] for ic in ics})\n",
    "    fpr['real'] = []\n",
    "    \n",
    "    p_est = {ic: [] for ic in ics}\n",
    "    p_est.update({'incremental_' + ic: [] for ic in ics})\n",
    "    \n",
    "    for i in trange(runs):\n",
    "        generator = CausalTSGenerator(dimensions=dimensions, max_p=p, data_length=data_length,\n",
    "                                      incoming_edges=incoming_edges)\n",
    "        ts = generator.generate()\n",
    "        predicted_graph = pc_chen_modified(partial_corr_test, ts, p, alpha)\n",
    "        \n",
    "        eval_result = evaluate_edges(generator.graph, predicted_graph)\n",
    "        f1['real'].append(eval_result['f1-score'])\n",
    "        recall['real'].append(eval_result['TPR'])\n",
    "        fpr['real'].append(eval_result['FPR'])\n",
    "        precision['real'].append(eval_result['precision'])\n",
    "        \n",
    "        # VAR estimation\n",
    "        var_ranking, var_scores = var_order_select(ts, 2*(max_p-2), ics, drop=True) \n",
    "        for ic in ics:\n",
    "            predicted_graph = pc_chen_modified(partial_corr_test, ts, var_ranking[ic][0], alpha)\n",
    "            eval_result = evaluate_edges(generator.graph, predicted_graph)\n",
    "            \n",
    "            p_est[ic].append(var_ranking[ic][0])\n",
    "            f1[ic].append(eval_result['f1-score'])\n",
    "            recall[ic].append(eval_result['TPR'])\n",
    "            fpr[ic].append(eval_result['FPR'])\n",
    "            precision[ic].append(eval_result['precision'])\n",
    "        \n",
    "        # incremental\n",
    "        for ic in ics:\n",
    "            predicted_graph = pc_incremental(partial_corr_test, ts, 0.05, 2*(max_p-2), ic=ic)\n",
    "            eval_result = evaluate_edges(generator.graph, predicted_graph)\n",
    "            \n",
    "            p_est['incremental_' + ic].append(len(predicted_graph.nodes())/dimensions - 1)\n",
    "            f1['incremental_' + ic].append(eval_result['f1-score'])\n",
    "            recall['incremental_' + ic].append(eval_result['TPR'])\n",
    "            fpr['incremental_' + ic].append(eval_result['FPR'])\n",
    "            precision['incremental_' + ic].append(eval_result['precision'])\n",
    "    \n",
    "    \n",
    "    scores = scores.append({'p': p, 'method': 'real',\n",
    "                            'mean_f1': np.mean(f1['real']), 'std_f1': np.std(f1['real']),\n",
    "                            'mean_recall': np.mean(recall['real']), 'std_recall': np.std(recall['real']),\n",
    "                            'mean_precision': np.mean(precision['real']), 'std_precision': np.std(precision['real']),\n",
    "                            'mean_fpr': np.mean(fpr['real']), 'std_fpr': np.std(fpr['real'])},\n",
    "                           ignore_index=True)\n",
    "    for ic in ics:\n",
    "        p_estimations = p_estimations.append({'p': p, 'method': ic, 'mean_p': np.mean(p_est[ic]), \n",
    "                                              'std_p': np.std(p_est[ic])},\n",
    "                                             ignore_index=True)\n",
    "        scores = scores.append({'p': p, 'method': ic,\n",
    "                                'mean_f1': np.mean(f1[ic]), 'std_f1': np.std(f1[ic]),\n",
    "                                'mean_recall': np.mean(recall[ic]), 'std_recall': np.std(recall[ic]),\n",
    "                                'mean_precision': np.mean(precision[ic]), 'std_precision': np.std(precision[ic]),\n",
    "                                'mean_fpr': np.mean(fpr[ic]), 'std_fpr': np.std(fpr[ic])},\n",
    "                               ignore_index=True)\n",
    "   \n",
    "    for ic in ics:\n",
    "        p_estimations = p_estimations.append({'p': p, 'method': 'incremental_' + ic, \n",
    "                                              'mean_p': np.mean(p_est['incremental_' + ic]),\n",
    "                                              'std_p': np.std(p_est['incremental_' + ic])},\n",
    "                                             ignore_index=True)\n",
    "        scores = scores.append({'p': p, 'method': 'incremental_' + ic,\n",
    "                                'mean_f1': np.mean(f1['incremental_' + ic]), 'std_f1': np.std(f1['incremental_' + ic]),\n",
    "                                'mean_recall': np.mean(recall['incremental_' + ic]),\n",
    "                                'std_recall': np.std(recall['incremental_' + ic]),\n",
    "                                'mean_precision': np.mean(precision['incremental_' + ic]),\n",
    "                                'std_precision': np.std(precision['incremental_' + ic]),\n",
    "                                'mean_fpr': np.mean(fpr['incremental_' + ic]),\n",
    "                                'std_fpr': np.std(fpr['incremental_' + ic])},\n",
    "                               ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-18T16:11:42.088739Z",
     "start_time": "2018-06-18T12:33:16.940040Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "scores.to_csv('results/not_extensive/scores_drop.csv')\n",
    "p_estimations.to_csv('results/not_extensive/p_estimations_drop.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "notify_time": "30"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}