docs/devcontainer/tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ExaGO Jupyter Tutorial\n",
    "\n",
    "We have a Python wrapper for ExaGO installed in this container through sapck. You can view the Dockerfile in `.devcontainer/Dockerfile`. See the Appendix for more details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Did we install correctly?\n",
    "import exago"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/app/docs/devcontainer\n",
      "/home/app\n"
     ]
    }
   ],
   "source": [
    "# Get pwd and change directory to where datafiles are\n",
    "import os\n",
    "print(f'{os.getcwd()}')\n",
    "os.chdir('../..')\n",
    "print(f'{os.getcwd()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "******************************************************************************\n",
      "This program contains Ipopt, a library for large-scale nonlinear optimization.\n",
      " Ipopt is released as open source code under the Eclipse Public License (EPL).\n",
      "         For more information visit http://projects.coin-or.org/Ipopt\n",
      "******************************************************************************\n",
      "\n",
      "This is Ipopt version 3.12.10, running with linear solver ma27.\n",
      "\n",
      "Number of nonzeros in equality constraint Jacobian...:      114\n",
      "Number of nonzeros in inequality constraint Jacobian.:       72\n",
      "Number of nonzeros in Lagrangian Hessian.............:       96\n",
      "\n",
      "Total number of variables............................:       24\n",
      "                     variables with only lower bounds:        0\n",
      "                variables with lower and upper bounds:       16\n",
      "                     variables with only upper bounds:        0\n",
      "Total number of equality constraints.................:       18\n",
      "Total number of inequality constraints...............:       18\n",
      "        inequality constraints with only lower bounds:        0\n",
      "   inequality constraints with lower and upper bounds:       18\n",
      "        inequality constraints with only upper bounds:        0\n",
      "\n",
      "iter    objective    inf_pr   inf_du lg(mu)  ||d||  lg(rg) alpha_du alpha_pr  ls\n",
      "   0  7.4421250e+03 1.80e+00 1.00e+02  -1.0 0.00e+00    -  0.00e+00 0.00e+00   0\n",
      "   1  6.3366435e+03 1.62e+00 8.99e+01  -1.0 1.98e+00    -  1.82e-01 9.95e-02f  1\n",
      "   2  5.0084316e+03 1.33e+00 7.40e+01  -1.0 1.45e+00    -  5.25e-02 1.78e-01f  1\n",
      "   3  4.5539744e+03 1.10e+00 6.09e+01  -1.0 6.68e-01    -  4.48e-01 1.77e-01f  1\n",
      "   4  4.5260321e+03 1.08e+00 9.94e+01  -1.0 1.07e+00    -  2.40e-03 1.22e-02f  1\n",
      "   5  3.8651832e+03 8.63e-01 4.94e+03  -1.0 8.05e-01    -  5.00e-03 6.98e-01f  1\n",
      "   6  3.7924522e+03 7.81e-01 4.43e+03  -1.0 1.04e+00    -  1.92e-01 9.50e-02f  1\n",
      "   7  3.5649304e+03 7.72e-02 2.36e+03  -1.0 1.33e+00    -  2.89e-01 1.00e+00f  1\n",
      "   8  2.9970732e+03 1.07e-01 1.80e+02  -1.0 7.79e-01    -  3.36e-01 1.00e+00f  1\n",
      "   9  2.8542043e+03 5.64e-02 2.10e+01  -1.0 5.06e-01    -  7.72e-01 1.00e+00f  1\n",
      "iter    objective    inf_pr   inf_du lg(mu)  ||d||  lg(rg) alpha_du alpha_pr  ls\n",
      "  10  2.8514258e+03 1.36e-03 8.51e-01  -1.7 1.33e-01    -  8.66e-01 1.00e+00h  1\n",
      "  11  2.8503356e+03 6.12e-06 8.96e-03  -2.5 5.62e-03    -  1.00e+00 1.00e+00h  1\n",
      "  12  2.8501932e+03 3.44e-08 8.43e-05  -3.8 2.65e-04    -  1.00e+00 1.00e+00h  1\n",
      "  13  2.8501865e+03 8.49e-11 2.34e-07  -5.7 1.35e-05    -  1.00e+00 1.00e+00h  1\n",
      "  14  2.8501864e+03 2.05e-12 1.01e-05  -7.0 1.47e-06    -  7.77e-01 1.00e+00h  1\n",
      "  15  2.8501863e+03 3.98e-12 1.64e-07  -7.0 2.14e-06    -  9.86e-01 1.00e+00h  1\n",
      "\n",
      "Number of Iterations....: 15\n",
      "\n",
      "                                   (scaled)                 (unscaled)\n",
      "Objective...............:   6.3905523451457867e+01    2.8501863459350211e+03\n",
      "Dual infeasibility......:   1.6359318072423079e-07    7.2962558603006936e-06\n",
      "Constraint violation....:   1.1539658117953877e-12    1.1539658117953877e-12\n",
      "Complementarity.........:   1.9165791600022652e-07    8.5479430536101036e-06\n",
      "Overall NLP error.......:   1.9165791600022652e-07    8.5479430536101036e-06\n",
      "\n",
      "\n",
      "Number of objective function evaluations             = 19\n",
      "Number of objective gradient evaluations             = 19\n",
      "Number of equality constraint evaluations            = 19\n",
      "Number of inequality constraint evaluations          = 19\n",
      "Number of equality constraint Jacobian evaluations   = 19\n",
      "Number of inequality constraint Jacobian evaluations = 19\n",
      "Number of Lagrangian Hessian evaluations             = 18\n",
      "Total CPU secs in IPOPT (w/o function evaluations)   =      0.014\n",
      "Total CPU secs in NLP function evaluations           =      0.002\n",

      "\n",
      "EXIT: Optimal Solution Found.\n",
      "=============================================================\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\tOptimal Power Flow\n",
      "=============================================================\n",
      "Model                               POWER_BALANCE_POLAR\n",
      "Solver                              IPOPT\n",
      "Objective                           MIN_GEN_COST\n",
      "Initialization                      MIDPOINT\n",
      "Gen. bus voltage mode               VARIABLE_WITHIN_BOUNDS\n",
      "Load loss allowed                   NO\n",
      "Power imbalance allowed             NO\n",
      "Ignore line flow constraints        NO\n",
      "\n",
      "Number of variables                 24\n",
      "Number of equality constraints      18\n",
      "Number of inequality constraints    18\n",
      "\n",
      "Convergence status                  CONVERGED\n",
      "Objective value                     2850.19\n",
      "\n",
      "------------------------------------------------------------------------------------------------------\n",
      "Bus        Pd      Pdloss Qd      Qdloss Vm      Va      mult_Pmis      mult_Qmis      Pslack         Qslack        \n",
      "------------------------------------------------------------------------------------------------------\n",
      "1         0.00    0.00    0.00    0.00   1.040   0.000      2064.25         0.00         0.00         0.00\n",
      "2         0.00    0.00    0.00    0.00   1.025   4.538      2013.83        -0.00         0.00         0.00\n",
      "3         0.00    0.00    0.00    0.00   1.025   2.901      2018.37        -0.00         0.00         0.00\n",
      "4         0.00    0.00    0.00    0.00   1.031  -2.188      2064.55         0.78         0.00         0.00\n",
      "5        75.00    0.00   50.00    0.00   1.008  -3.380      2076.02        11.05         0.00         0.00\n",
      "6        90.00    0.00   30.00    0.00   1.017  -4.265      2093.63         5.44         0.00         0.00\n",
      "7         0.00    0.00    0.00    0.00   1.029   0.752      2014.50         3.54         0.00         0.00\n",
      "8       100.00    0.00   35.00    0.00   1.018  -1.725      2035.99         7.25         0.00         0.00\n",
      "9         0.00    0.00    0.00    0.00   1.034   0.206      2018.82         2.49         0.00         0.00\n",
      "\n",
      "------------------------------------------------------------------------------------------------------\n",
      "From       To       Status     Sft      Stf     Slim     mult_Sf  mult_St \n",
      "----------------------------------------------------------------------------------------\n",
      "1          4          1       73.27    72.63   380.00    -0.00    -0.00\n",
      "2          7          1      111.43   111.84   250.00    -0.00    -0.00\n",
      "3          9          1       86.05    86.79   300.00    -0.00    -0.00\n",
      "4          5          1       32.20    43.00   250.00    -0.00    -0.00\n",
      "4          6          1       42.77    44.87   250.00    -0.00    -0.00\n",
      "5          7          1       49.98    48.69   250.00    -0.00    -0.00\n",
      "6          9          1       50.00    51.77   150.00    -0.00    -0.00\n",
      "7          8          1       63.90    64.98   250.00    -0.00    -0.00\n",
      "8          9          1       42.30    36.62   150.00    -0.00    -0.00\n",
      "\n",
      "----------------------------------------------------------------------------------------\n",
      "Gen      Status     Fuel     Pg       Qg       Pmin     Pmax     Qmin     Qmax  \n",
      "----------------------------------------------------------------------------------------\n",
      "1          1          NG    71.10    17.70    10.00   350.00  -300.00   300.00\n",
      "2          1          NG   111.40    -2.61    10.00   300.00  -300.00   300.00\n",
      "3          1        WIND    85.00   -13.42     0.00    85.00  -300.00   300.00\n",
      "[ExaGO] Finalizing opflow application.\n"
     ]
    }
   ],
   "source": [
    "exago.initialize(\"opflow\")\n",
    "opf = exago.OPFLOW()\n",
    "opf.read_mat_power_data('datafiles/case9/case9mod_gen3_wind2.m')\n",
    "opf.solve()\n",
    "filepath = './docs/devcontainer/sample_opflow.output'\n",
    "opf.save_solution(exago.CSV, filepath)\n",
    "opf.print_solution()\n",
    "del opf\n",
    "exago.finalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'nbus': 9, 'ngen': 3, 'nbranch': 9, 'baseMVA': 100.0, 'Bus': {'Num': [1, 2, 3, 4, 5, 6, 7, 8, 9], 'Pd': [0.0, 0.0, 0.0, 0.0, 75.0, 90.0, 0.0, 100.0, 0.0], 'Qd': [0.0, 0.0, 0.0, 0.0, 50.0, 30.0, 0.0, 35.0, 0.0], 'Vm': [1.04, 1.02, 1.02, 1.03, 1.01, 1.02, 1.03, 1.02, 1.03], 'Va': [0.0, 4.54, 2.9, -2.19, -3.38, -4.26, 0.75, -1.72, 0.21], 'mult_Pmis': [2064.25, 2013.83, 2018.37, 2064.55, 2076.02, 2093.63, 2014.5, 2035.99, 2018.82], 'mult_Qmis': [0.0, -0.0, -0.0, 0.78, 11.05, 5.44, 3.54, 7.25, 2.49]}, 'Branch': {'From': [1, 2, 3, 4, 4, 5, 6, 7, 8], 'To': [4, 7, 9, 5, 6, 7, 9, 8, 9], 'Status': [1, 1, 1, 1, 1, 1, 1, 1, 1], 'Sft': [73.27, 111.43, 86.05, 32.2, 42.77, 49.98, 50.0, 63.9, 42.3], 'Stf': [72.63, 111.84, 86.79, 43.0, 44.87, 48.69, 51.77, 64.98, 36.62], 'Slim': [380.0, 250.0, 300.0, 250.0, 250.0, 250.0, 150.0, 250.0, 150.0], 'mult_Sf': [-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0], 'mult_St': [-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0]}, 'Gen': {'Genbus': [1, 2, 3], 'Status': [1, 1, 1], 'Fuel': ['NG', 'NG', 'WIND'], 'Pg': [71.1, 111.4, 85.0], 'Qg': [17.7, -2.61, -13.42], 'Pmin': [10.0, 10.0, 0.0], 'Pmax': [350.0, 300.0, 85.0], 'Qmin': [-300.0, -300.0, -300.0], 'Qmax': [300.0, 300.0, 300.0]}}\n",
      "data converted from ./docs/devcontainer/sample_opflow.output to ./docs/devcontainer/sample_opflow_data.json\n"
     ]
    }
   ],
   "source": [
    "import csv \n",
    "import json\n",
    "\n",
    "def add_keys(dict, row, tag=\"\"):\n",
    "    for i in range(0, len(row)):\n",
    "        if row[i] == \"Bus\":\n",
    "            row[i] = \"Num\"\n",
    "        elif row[i] == \"Gen bus\":\n",
    "            row[i] = \"Genbus\" \n",
    "    for key in row:\n",
    "        if tag:\n",
    "            if tag not in dict.keys():\n",
    "                dict[tag] = {}\n",
    "            if key == \"Bus\":\n",
    "                dict[tag][\"Num\"] = []\n",
    "            elif key == \"Gen bus\":\n",
    "                dict[tag][\"Genbus\"] = []\n",
    "            else:\n",
    "                dict[tag][key] = [] \n",
    "        else:\n",
    "            dict[key] = 0\n",
    "    return dict\n",
    "\n",
    "def write_json(dict):\n",
    "    #convert python jsonArray to JSON String and write to file\n",
    "    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: \n",
    "        jsonString = json.dumps(dict, indent=4)\n",
    "        jsonf.write(jsonString)\n",
    "\n",
    "def count_lines_in_file(filename):\n",
    "    lines = 0\n",
    "    with open(filename, encoding='utf-8') as csvf: \n",
    "        lines = sum(1 for _ in csvf)\n",
    "    csvf.close()\n",
    "    return lines\n",
    "\n",
    "def opflow_csv_to_json(csvFilePath, jsonFilePath):\n",
    "    #count the number of lines in the csv file\n",
    "    lines = count_lines_in_file(csvFilePath)\n",
    "\n",
    "    #read csv file\n",
    "    with open(csvFilePath, encoding='utf-8') as csvf: \n",
    "        #load csv file data using csv library's reader\n",
    "        csvReader = csv.reader(csvf) \n",
    "        current_row = 0\n",
    "        temp = {}\n",
    "        keys = True\n",
    "\n",
    "        #add first two rows manually\n",
    "        row = next(csvReader)\n",
    "        current_row += 1\n",
    "        temp = add_keys(temp, row)\n",
    "        current_keys = row\n",
    "        #check if there's another row\n",
    "        if current_row < lines:\n",
    "            row = next(csvReader)\n",
    "            current_row += 1\n",
    "        #add values to corresponding keys\n",
    "        for x in range(0, len(current_keys)):\n",
    "            if x < 3:\n",
    "                temp[current_keys[x]] =  int(row[x])\n",
    "            else:\n",
    "                temp[current_keys[x]] =  float(row[x])\n",
    "\n",
    "        #hard coded category names and how many rows\n",
    "        tags = [(\"Bus\", 9), (\"Branch\", 9), (\"Gen\", 3)]\n",
    "        for pair in tags:\n",
    "            tag = pair[0]\n",
    "            num = pair[1]\n",
    "            #check if there's another row\n",
    "            if current_row < lines:\n",
    "                row = next(csvReader)\n",
    "                current_row += 1\n",
    "            temp = add_keys(temp, row, tag)\n",
    "            current_keys = row\n",
    "\n",
    "            for k in range(0, num): #rows in each category\n",
    "                #check if there's another row\n",
    "                if current_row < lines:\n",
    "                    row = next(csvReader)\n",
    "                    current_row += 1\n",
    "\n",
    "                if tag == \"Bus\":\n",
    "                   #add values to corresponding keys\n",
    "                    for x in range(0, len(current_keys)):\n",
    "                        for j in range(0, len(row)):\n",
    "                            if j == 0:\n",
    "                                row[j] = int(row[j])\n",
    "                            else:\n",
    "                                row[j] = float(row[j])\n",
    "                        temp[tag][current_keys[x]].append(row[x])   \n",
    "                elif tag == \"Branch\": \n",
    "                    #add values to corresponding keys\n",
    "                    for x in range(0, len(current_keys)):\n",
    "                        for j in range(0, len(row)):\n",
    "                            if j >= 0 and j < 3:\n",
    "                                row[j] = int(row[j])\n",
    "                            else:\n",
    "                                row[j] = float(row[j])\n",
    "                        temp[tag][current_keys[x]].append(row[x])  \n",
    "                elif tag == \"Gen\": #cannot cast the string fields\n",
    "                    #add values to corresponding keys\n",
    "                    for x in range(0, len(current_keys)):\n",
    "                        for j in range(0, len(row)):\n",
    "                            if j >= 0 and j < 2:\n",
    "                                row[j] = int(row[j])\n",
    "                            elif j == 2:\n",
    "                                row[j] = row[2].strip()\n",
    "                            else:\n",
    "                                row[j] = float(row[j])\n",
    "                        temp[tag][current_keys[x]].append(row[x])  \n",
    "                else:\n",
    "                    #add values to corresponding keys\n",
    "                    for x in range(0, len(current_keys)):\n",
    "                        temp[tag][current_keys[x]].append(float(row[x]))\n",
    "            \n",
    "        print(temp)\n",
    "        write_json(temp) \n",
    "        return\n",
    "\n",
    "#if cell 2 is run, then ./ is the top level of the ExaGO dir\n",
    "csvFilePath = r'./docs/devcontainer/sample_opflow.output'\n",
    "jsonFilePath = r'./docs/devcontainer/sample_opflow_data.json'\n",
    "\n",
    "opflow_csv_to_json(csvFilePath, jsonFilePath)\n",
    "\n",
    "print(f\"data converted from {csvFilePath} to {jsonFilePath}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix\n",
    "\n",
    "Information about the Jupyter/Python/Container config is listed here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00s - Debugger warning: It seems that frozen modules are being used, which may\n",
      "0.00s - make the debugger miss breakpoints. Please pass -Xfrozen_modules=off\n",
      "0.00s - to python to disable frozen modules.\n",
      "0.00s - Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation.\n",
      "Available kernels:\n",
      "  python3        /opt/views/view/share/jupyter/kernels/python3\n",
      "  py311-exago    /usr/local/share/jupyter/kernels/py311-exago\n"
     ]
    }
   ],
   "source": [
    "!jupyter kernelspec list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      " \"argv\": [\n",
      "  \"/opt/views/view/bin/python3\",\n",
      "  \"-m\",\n",
      "  \"ipykernel_launcher\",\n",
      "  \"-f\",\n",
      "  \"{connection_file}\"\n",
      " ],\n",
      " \"display_name\": \"ExaGO\",\n",
      " \"language\": \"python\",\n",
      " \"metadata\": {\n",
      "  \"debugger\": true\n",
      " }\n",
      "}"
     ]
    }
   ],
   "source": [
    "!cat /usr/local/share/jupyter/kernels/py311-exago/kernel.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00s - Debugger warning: It seems that frozen modules are being used, which may\n",
      "0.00s - make the debugger miss breakpoints. Please pass -Xfrozen_modules=off\n",
      "0.00s - to python to disable frozen modules.\n",
      "0.00s - Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation.\n",
      "Available kernels:\n",
      "  python3               /opt/views/view/share/jupyter/kernels/python3\n",
      "  py311-exago           /usr/local/share/jupyter/kernels/py311-exago\n",
      "  py311-mpi4py-exago    /usr/local/share/jupyter/kernels/py311-mpi4py-exago\n"
     ]
    }
   ],
   "source": [
    "!jupyter kernelspec list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      " \"argv\": [\n",
      "  \"/opt/views/view/bin/python3\",\n",
      "  \"-m\",\n",
      "  \"ipykernel_launcher\",\n",
      "  \"-f\",\n",
      "  \"{connection_file}\"\n",
      " ],\n",
      " \"display_name\": \"ExaGO\",\n",
      " \"language\": \"python\",\n",
      " \"metadata\": {\n",
      "  \"debugger\": true\n",
      " }\n",
      "}"
     ]
    }
   ],
   "source": [
    "!cat /usr/local/share/jupyter/kernels/py311-exago/kernel.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                  Version\n",
      "------------------------ ---------\n",
      "anyio                    4.0.0\n",
      "argon2-cffi              21.3.0\n",
      "argon2-cffi-bindings     21.2.0\n",
      "arrow                    1.2.3\n",
      "asttokens                2.4.0\n",
      "async-lru                1.0.3\n",
      "attrs                    23.1.0\n",
      "Babel                    2.12.1\n",
      "backcall                 0.2.0\n",
      "beautifulsoup4           4.12.2\n",
      "bleach                   6.0.0\n",
      "certifi                  2023.7.22\n",
      "cffi                     1.15.1\n",
      "charset-normalizer       3.3.0\n",
      "comm                     0.1.4\n",
      "debugpy                  1.6.7\n",
      "decorator                5.1.1\n",
      "defusedxml               0.7.1\n",
      "editables                0.3\n",
      "executing                1.2.0\n",
      "fastjsonschema           2.16.3\n",
      "fqdn                     1.5.1\n",
      "gevent                   23.7.0\n",
      "greenlet                 2.0.2\n",
      "hatch-jupyter-builder    0.8.3\n",
      "hatchling                1.18.0\n",
      "idna                     3.4\n",
      "iniconfig                2.0.0\n",
      "ipykernel                6.23.1\n",
      "ipython                  8.14.0\n",
      "isoduration              20.11.0\n",
      "jedi                     0.18.2\n",
      "Jinja2                   3.0.3\n",
      "json5                    0.9.14\n",
      "jsonpointer              2.0\n",
      "jsonschema               4.17.3\n",
      "jupyter_client           8.2.0\n",
      "jupyter_core             5.3.0\n",
      "jupyter-events           0.6.3\n",
      "jupyter-lsp              2.2.0\n",
      "jupyter_server           2.6.0\n",
      "jupyter_server_terminals 0.4.4\n",
      "jupyterlab               4.0.1\n",
      "jupyterlab-pygments      0.2.2\n",
      "jupyterlab_server        2.22.1\n",
      "MarkupSafe               2.1.3\n",
      "matplotlib-inline        0.1.6\n",
      "mistune                  2.0.5\n",
      "mpi4py                   3.1.4\n",
      "nbclient                 0.8.0\n",
      "nbconvert                7.4.0\n",
      "nbformat                 5.8.0\n",
      "nest-asyncio             1.5.6\n",
      "notebook_shim            0.2.3\n",
      "overrides                7.3.1\n",
      "packaging                23.1\n",
      "pandocfilters            1.5.0\n",
      "parso                    0.8.3\n",
      "pathspec                 0.11.1\n",
      "pexpect                  4.8.0\n",
      "pickleshare              0.7.5\n",
      "pip                      23.2.1\n",
      "platformdirs             3.10.0\n",
      "pluggy                   1.0.0\n",
      "prometheus-client        0.17.0\n",
      "prompt-toolkit           3.0.38\n",
      "psutil                   5.9.5\n",
      "ptyprocess               0.7.0\n",
      "pure-eval                0.2.2\n",
      "pycparser                2.21\n",
      "Pygments                 2.16.1\n",
      "pyrsistent               0.19.3\n",
      "pytest                   7.3.2\n",
      "python-dateutil          2.8.2\n",
      "python-json-logger       2.0.7\n",
      "PyYAML                   6.0\n",
      "pyzmq                    25.0.2\n",
      "requests                 2.31.0\n",
      "rfc3339-validator        0.1.4\n",
      "rfc3986-validator        0.1.1\n",
      "Send2Trash               1.8.0\n",
      "setuptools               68.0.0\n",
      "six                      1.16.0\n",
      "sniffio                  1.3.0\n",
      "soupsieve                2.4.1\n",
      "stack-data               0.6.2\n",
      "terminado                0.17.1\n",
      "tinycss2                 1.2.1\n",
      "tornado                  6.3.3\n",
      "traitlets                5.9.0\n",
      "trove-classifiers        2023.8.7\n",
      "uri-template             1.2.0\n",
      "urllib3                  2.0.6\n",
      "wcwidth                  0.2.7\n",
      "webcolors                1.11.1\n",
      "webencodings             0.5.1\n",
      "websocket-client         1.6.3\n",
      "wheel                    0.41.3\n",
      "zope.event               4.6\n",
      "zope.interface           5.4.0\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's odd - ExaGO can import and run, but doesn't show up in pip. That's because we installed using Spack, and ExaGO is available directly through the `PYTHONPATH` environment variable.\n",
    "\n",
    "This is configured through `entrypoint.sh`, which is the last thing that happens in the configured `~/.bashrc`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            fi; \\\n",
      "        fi`'\n",
      "    local lightblue='\\[\\033[1;34m\\]'\n",
      "    local removecolor='\\[\\033[0m\\]'\n",
      "    PS1=\"${userpart} ${lightblue}\\w ${gitbranch}${removecolor}\\$ \"\n",
      "    unset -f __bash_prompt\n",
      "}\n",
      "__bash_prompt\n",
      "export PROMPT_DIRTRIM=4\n",
      "source /entrypoint.sh\n"
     ]
    }
   ],
   "source": [
    "!cat ~/.bashrc | tail -n 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#!/bin/sh\n",
      ". /opt/spack-environment/activate.sh\n",
      "exec \"$@\"\n"
     ]
    }
   ],
   "source": [
    "!cat /entrypoint.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export SPACK_ENV=/opt/spack-environment;\n",
      "export SPACK_ENV_VIEW=default;\n",
      "alias despacktivate='spack env deactivate';\n",
      "export ACLOCAL_PATH=/opt/views/view/share/aclocal;\n",
      "export CMAKE_PREFIX_PATH=/opt/views/view;\n",
      "export C_INCLUDE_PATH=/opt/views/view/include;\n",
      "export JUPYTERLAB_DIR=/opt/views/view/share/jupyter/lab;\n",
      "export JUPYTER_PATH=/opt/views/view/share/jupyter;\n",
      "export LIBRARY_PATH=/opt/views/view/lib;\n",
      "export MANPATH=/opt/views/view/share/man:/opt/views/view/man:;\n",
      "export MPICC=/opt/views/view/bin/mpicc;\n",
      "export MPICXX=/opt/views/view/bin/mpic++;\n",
      "export MPIF77=/opt/views/view/bin/mpif77;\n",
      "export MPIF90=/opt/views/view/bin/mpif90;\n",
      "export PATH=/opt/views/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin;\n",
      "export PETSC_ARCH=\"\";\n",
      "export PETSC_DIR=/opt/views/view;\n",
      "export PKG_CONFIG_PATH=/opt/views/view/lib/pkgconfig:/opt/views/view/share/pkgconfig:/opt/views/view/lib64/pkgconfig;\n",
      "export PYTHONPATH=/opt/views/view/lib/python3.11/site-packages;\n"
     ]
    }
   ],
   "source": [
    "!cat /opt/spack-environment/activate.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/views/view/lib/python3.11/site-packages:/opt/views/view/lib/python3.11/site-packages\n"
     ]
    }
   ],
   "source": [
    "!echo $PYTHONPATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 864\n",
      "drwxr-xr-x 193 root root 12288 Nov  6 20:19 .\n",
      "drwxr-xr-x  39 root root  4096 Nov  6 20:19 ..\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 Babel-2.12.1.dist-info\n",
      "drwxr-xr-x  11 root root  4096 Nov  6 20:19 IPython\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 Jinja2-3.0.3.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 MarkupSafe-2.1.3.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 PyYAML-6.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 Pygments-2.16.1.dist-info\n",
      "lrwxrwxrwx   1 root root   139 Nov  6 20:19 README.txt -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/python-3.11.6-ybsi7tlnssr3m4h3eqrzq6tyxdcsr2n7/lib/python3.11/site-packages/README.txt\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 Send2Trash-1.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 __pycache__\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 _argon2_cffi_bindings\n",
      "lrwxrwxrwx   1 root root   175 Nov  6 20:19 _cffi_backend.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-cffi-1.15.1-ns7kiujztzymljhmkwceej66qntnjlcr/lib/python3.11/site-packages/_cffi_backend.cpython-311-x86_64-linux-gnu.so\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 _distutils_hack\n",
      "lrwxrwxrwx   1 root root   158 Nov  6 20:19 _pyrsistent_version.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pyrsistent-0.19.3-ptruwnt5v52i3udbb6cbvhq5oeqn7dox/lib/python3.11/site-packages/_pyrsistent_version.py\n",
      "drwxr-xr-x   9 root root  4096 Nov  6 20:19 _pytest\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 _yaml\n",
      "drwxr-xr-x   7 root root  4096 Nov  6 20:19 anyio\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 anyio-4.0.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 argon2\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 argon2_cffi-21.3.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 argon2_cffi_bindings-21.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 arrow\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 arrow-1.2.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 asttokens\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 asttokens-2.4.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 async_lru-1.0.3.dist-info\n",
      "lrwxrwxrwx   1 root root   146 Nov  6 20:19 async_lru.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-async-lru-1.0.3-n4nwu7jegsgtzsmiplftm3ik7dsghnz7/lib/python3.11/site-packages/async_lru.py\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 attr\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 attrs\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 attrs-23.1.0.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 babel\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 backcall\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 backcall-0.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 beautifulsoup4-4.12.2.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 bleach\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 bleach-6.0.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 bs4\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 certifi\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 certifi-2023.7.22.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 cffi\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 cffi-1.15.1.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 charset_normalizer\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 charset_normalizer-3.3.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 comm\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 comm-0.1.4.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 dateutil\n",
      "drwxr-xr-x   8 root root  4096 Nov  6 20:19 debugpy\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 debugpy-1.6.7.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 decorator-5.1.1.dist-info\n",
      "lrwxrwxrwx   1 root root   146 Nov  6 20:19 decorator.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-decorator-5.1.1-y6omagu7er3rjtyisfdajw5upayicgxe/lib/python3.11/site-packages/decorator.py\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 defusedxml\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 defusedxml-0.7.1.dist-info\n",
      "lrwxrwxrwx   1 root root   160 Nov  6 20:19 distutils-precedence.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-setuptools-68.0.0-ceuwjz4g6i5jxk4babjsbh6oxmqggbba/lib/python3.11/site-packages/distutils-precedence.pth\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 editables\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 editables-0.3.dist-info\n",
      "lrwxrwxrwx   1 root root   164 Nov  6 20:19 exago.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/exago-1.6.0-ssrhw6tbxmal4jmksthjndq2wu33csoe/lib/python3.11/site-packages/exago.cpython-311-x86_64-linux-gnu.so\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 executing\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 executing-1.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 fastjsonschema\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 fastjsonschema-2.16.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 fqdn\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 fqdn-1.5.1.dist-info\n",
      "drwxr-xr-x   9 root root  4096 Nov  6 20:19 gevent\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 gevent-23.7.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 greenlet\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 greenlet-2.0.2.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 hatch_jupyter_builder\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 hatch_jupyter_builder-0.8.3.dist-info\n",
      "drwxr-xr-x  12 root root  4096 Nov  6 20:19 hatchling\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 hatchling-1.18.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 idna\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 idna-3.4.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 iniconfig\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 iniconfig-2.0.0.dist-info\n",
      "drwxr-xr-x   9 root root  4096 Nov  6 20:19 ipykernel\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 ipykernel-6.23.1.dist-info\n",
      "lrwxrwxrwx   1 root root   156 Nov  6 20:19 ipykernel_launcher.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-ipykernel-6.23.1-raogvtxj7buw67byp3cbpliaqqc4bgkc/lib/python3.11/site-packages/ipykernel_launcher.py\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 ipython-8.14.0.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 isoduration\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 isoduration-20.11.0.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  6 20:19 jedi\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 jedi-0.18.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jinja2\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 json5\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 json5-0.9.14.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 jsonpointer-2.0.dist-info\n",
      "lrwxrwxrwx   1 root root   148 Nov  6 20:19 jsonpointer.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-jsonpointer-2.0-6f5qc5t5ix6e77lewi5pgup5cwgmbngu/lib/python3.11/site-packages/jsonpointer.py\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 jsonschema\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jsonschema-4.17.3.dist-info\n",
      "lrwxrwxrwx   1 root root   147 Nov  6 20:19 jupyter.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-jupyter-core-5.3.0-bupox3eooyhmccqjf75vgk34tys5l75z/lib/python3.11/site-packages/jupyter.py\n",
      "drwxr-xr-x   8 root root  4096 Nov  6 20:19 jupyter_client\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_client-8.2.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 jupyter_core\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_core-5.3.0.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 jupyter_events\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_events-0.6.3.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  6 20:19 jupyter_lsp\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 jupyter_lsp-2.2.0.dist-info\n",
      "drwxr-xr-x  18 root root  4096 Nov  6 20:19 jupyter_server\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_server-2.6.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_server_terminals\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyter_server_terminals-0.4.4.dist-info\n",
      "drwxr-xr-x  11 root root  4096 Nov  6 20:19 jupyterlab\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyterlab-4.0.1.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 jupyterlab_pygments\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 jupyterlab_pygments-0.2.2.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 jupyterlab_server\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 jupyterlab_server-2.22.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 markupsafe\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 matplotlib_inline\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 matplotlib_inline-0.1.6.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 mistune\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 mistune-2.0.5.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  6 20:19 mpi4py\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 mpi4py-3.1.4.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 nbclient\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 nbclient-0.8.0.dist-info\n",
      "drwxr-xr-x  11 root root  4096 Nov  6 20:19 nbconvert\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 nbconvert-7.4.0.dist-info\n",
      "drwxr-xr-x   8 root root  4096 Nov  6 20:19 nbformat\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 nbformat-5.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 nest_asyncio-1.5.6.dist-info\n",
      "lrwxrwxrwx   1 root root   152 Nov  6 20:19 nest_asyncio.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-nest-asyncio-1.5.6-hbtnt2uhwofptyytpmbxi2uqcbngvlox/lib/python3.11/site-packages/nest_asyncio.py\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 notebook_shim\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 notebook_shim-0.2.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 overrides\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 overrides-7.3.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 packaging\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 packaging-23.1.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pandocfilters-1.5.0.dist-info\n",
      "lrwxrwxrwx   1 root root   154 Nov  6 20:19 pandocfilters.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pandocfilters-1.5.0-wseldytxe3leiottrhb5ogloooxslfd5/lib/python3.11/site-packages/pandocfilters.py\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 parso\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 parso-0.8.3.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 pathspec\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pathspec-0.11.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pexpect\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pexpect-4.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pickleshare-0.7.5.dist-info\n",
      "lrwxrwxrwx   1 root root   150 Nov  6 20:19 pickleshare.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pickleshare-0.7.5-wty2ecbqmknacpwe6nwpiyxbq5oqhcza/lib/python3.11/site-packages/pickleshare.py\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 pkg_resources\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 platformdirs\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 platformdirs-3.10.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pluggy\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pluggy-1.0.0.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 prometheus_client\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 prometheus_client-0.17.0.dist-info\n",
      "drwxr-xr-x  18 root root  4096 Nov  6 20:19 prompt_toolkit\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 prompt_toolkit-3.0.38.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 psutil\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 psutil-5.9.5.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 ptyprocess\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 ptyprocess-0.7.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pure_eval\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pure_eval-0.2.2.dist-info\n",
      "lrwxrwxrwx   1 root root   176 Nov  6 20:19 pvectorc.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pyrsistent-0.19.3-ptruwnt5v52i3udbb6cbvhq5oeqn7dox/lib/python3.11/site-packages/pvectorc.cpython-311-x86_64-linux-gnu.so\n",
      "lrwxrwxrwx   1 root root   136 Nov  6 20:19 py.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pytest-7.3.2-wx6nsrjdt6ftqyuprphukh4m64lu3hk6/lib/python3.11/site-packages/py.py\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 pycparser\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pycparser-2.21.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  6 20:19 pygments\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pyrsistent\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pyrsistent-0.19.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pytest\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pytest-7.3.2.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 python_dateutil-2.8.2.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 python_json_logger-2.0.7.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 pythonjsonlogger\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 pyzmq-25.0.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 requests\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 requests-2.31.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 rfc3339_validator-0.1.4.dist-info\n",
      "lrwxrwxrwx   1 root root   162 Nov  6 20:19 rfc3339_validator.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-rfc3339-validator-0.1.4-ifadhrcyquy632ftjvceaylpe4aqn7kt/lib/python3.11/site-packages/rfc3339_validator.py\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 rfc3986_validator-0.1.1.dist-info\n",
      "lrwxrwxrwx   1 root root   162 Nov  6 20:19 rfc3986_validator.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-rfc3986-validator-0.1.1-zkwpotfen5ywonax22utpdph5j4eczmr/lib/python3.11/site-packages/rfc3986_validator.py\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 send2trash\n",
      "drwxr-xr-x   8 root root  4096 Nov  6 20:19 setuptools\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 setuptools-68.0.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 six-1.16.0.dist-info\n",
      "lrwxrwxrwx   1 root root   135 Nov  6 20:19 six.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-six-1.16.0-nbnfdrflk6rrh3fu3pengfnlmslekoqs/lib/python3.11/site-packages/six.py\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 sniffio\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 sniffio-1.3.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 soupsieve\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 soupsieve-2.4.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 stack_data\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 stack_data-0.6.2.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 terminado\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 terminado-0.17.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 tinycss2\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 tinycss2-1.2.1.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 tornado\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 tornado-6.3.3.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  6 20:19 traitlets\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 traitlets-5.9.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 trove_classifiers\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 trove_classifiers-2023.8.7.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 uri_template\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 uri_template-1.2.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  6 20:19 urllib3\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 urllib3-2.0.6.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 wcwidth\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 wcwidth-0.2.7.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 webcolors-1.11.1.dist-info\n",
      "lrwxrwxrwx   1 root root   147 Nov  6 20:19 webcolors.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-webcolors-1.11.1-x6fsbcye464konn2u33psb3vrat7bmzj/lib/python3.11/site-packages/webcolors.py\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 webencodings\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 webencodings-0.5.1.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 websocket\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 websocket_client-1.6.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  6 20:19 yaml\n",
      "drwxr-xr-x  13 root root  4096 Nov  6 20:19 zmq\n",
      "drwxr-xr-x   4 root root  4096 Nov  6 20:19 zope\n",
      "lrwxrwxrwx   1 root root   164 Nov  6 20:19 zope.event-4.6-py3.11-nspkg.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-zope-event-4.6-mlah5dql76d33hl34zeysun66ik2nmjk/lib/python3.11/site-packages/zope.event-4.6-py3.11-nspkg.pth\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 zope.event-4.6.dist-info\n",
      "lrwxrwxrwx   1 root root   176 Nov  6 20:19 zope.interface-5.4.0-py3.11-nspkg.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-zope-interface-5.4.0-6jsmi36ohggmpihuchqcd3r3s2eiqaan/lib/python3.11/site-packages/zope.interface-5.4.0-py3.11-nspkg.pth\n",
      "drwxr-xr-x   2 root root  4096 Nov  6 20:19 zope.interface-5.4.0.dist-info\n"
     ]
    }
   ],
   "source": [
    "!ls -al /opt/views/view/lib/python3.11/site-packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here you can see the full list of spack configured python packages, including exago."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some info about this container"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRETTY_NAME=\"Debian GNU/Linux 12 (bookworm)\"\n",
      "NAME=\"Debian GNU/Linux\"\n",
      "VERSION_ID=\"12\"\n",
      "VERSION=\"12 (bookworm)\"\n",
      "VERSION_CODENAME=bookworm\n",
      "ID=debian\n",
      "HOME_URL=\"https://www.debian.org/\"\n",
      "SUPPORT_URL=\"https://www.debian.org/support\"\n",
      "BUG_REPORT_URL=\"https://bugs.debian.org/\"\n"
     ]
    }
   ],
   "source": [
    "!cat /etc/os-release"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Previous version of `csv_to_json` (more modularized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'nbus': [9.0], 'ngen': [3.0], 'nbranch': [9.0], 'baseMVA': [100.0], 'Bus': [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0], 'Pd': [0.0, 0.0, 0.0, 0.0, 75.0, 90.0, 0.0, 100.0, 0.0], 'Qd': [0.0, 0.0, 0.0, 0.0, 50.0, 30.0, 0.0, 35.0, 0.0], 'Vm': [1.04, 1.02, 1.02, 1.03, 1.01, 1.02, 1.03, 1.02, 1.03], 'Va': [0.0, 4.54, 2.9, -2.19, -3.38, -4.26, 0.75, -1.72, 0.21], 'mult_Pmis': [2064.25, 2013.83, 2018.37, 2064.55, 2076.02, 2093.63, 2014.5, 2035.99, 2018.82], 'mult_Qmis': [0.0, -0.0, -0.0, 0.78, 11.05, 5.44, 3.54, 7.25, 2.49], 'From': [1.0, 2.0, 3.0, 4.0, 4.0, 5.0, 6.0, 7.0, 8.0], 'To': [4.0, 7.0, 9.0, 5.0, 6.0, 7.0, 9.0, 8.0, 9.0], 'Status': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'Sft': [73.27, 111.43, 86.05, 32.2, 42.77, 49.98, 50.0, 63.9, 42.3], 'Stf': [72.63, 111.84, 86.79, 43.0, 44.87, 48.69, 51.77, 64.98, 36.62], 'Slim': [380.0, 250.0, 300.0, 250.0, 250.0, 250.0, 150.0, 250.0, 150.0], 'mult_Sf': [-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0], 'mult_St': [-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0], 'Gen bus': [1.0, 2.0, 3.0], 'Status-2': [1.0, 1.0, 1.0], 'Fuel': ['NG', 'NG', 'WIND'], 'Pg': [71.1, 111.4, 85.0], 'Qg': [17.7, -2.61, -13.42], 'Pmin': [10.0, 10.0, 0.0], 'Pmax': [350.0, 300.0, 85.0], 'Qmin': [-300.0, -300.0, -300.0], 'Qmax': [300.0, 300.0, 300.0]}\n",
      "data converted from ./tmp.output to ./data.json\n"
     ]
    }
   ],
   "source": [
    "import csv \n",
    "import json\n",
    "import time\n",
    "import numbers\n",
    "\n",
    "def csv_to_json(csvFilePath, jsonFilePath):\n",
    "    jsonArray = []\n",
    "\n",
    "    #count the number of lines in the csv file\n",
    "    lines = 0\n",
    "    with open(csvFilePath, encoding='utf-8') as csvf: \n",
    "        lines = sum(1 for _ in csvf)\n",
    "    csvf.close()\n",
    "      \n",
    "    #read csv file\n",
    "    with open(csvFilePath, encoding='utf-8') as csvf: \n",
    "        #load csv file data using csv library's reader\n",
    "        csvReader = csv.reader(csvf) \n",
    "\n",
    "        current_row = 0\n",
    "        temp = {}\n",
    "        keys = True\n",
    "        row = next(csvReader)\n",
    "        current_row += 1\n",
    "\n",
    "        while True:\n",
    "            #set non-number rows as keys in the dictionary\n",
    "            if keys:\n",
    "                for y in range(0, len(row)):\n",
    "                    item = row[y]\n",
    "                    if item in temp.keys():\n",
    "                        item += \"-2\"\n",
    "                        row[y] = item\n",
    "                    temp[item] = []\n",
    "                keys = False\n",
    "                current_keys = row\n",
    "\n",
    "            #check if there's another row\n",
    "            if current_row < lines:\n",
    "                row = next(csvReader)\n",
    "                current_row += 1\n",
    "            else:\n",
    "                print(temp)\n",
    "                jsonArray = temp\n",
    "                #convert python jsonArray to JSON String and write to file\n",
    "                with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: \n",
    "                    jsonString = json.dumps(jsonArray, indent=4)\n",
    "                    jsonf.write(jsonString)\n",
    "                return\n",
    "\n",
    "            #add values to the current keys until a non-number row\n",
    "            while not keys:  \n",
    "                try:\n",
    "                    #check for the non-number Fuel value\n",
    "                    if row[2].strip() in [\"COAL\", \"WIND\", \"SOLAR\", \"NG\", \"NUCLEAR\", \"HYDRO\", \"UNDEFINED\"]:\n",
    "                        for j in range(0, len(row)):\n",
    "                            if j != 2:\n",
    "                                row[j] = float(row[j])\n",
    "                            else:\n",
    "                                row[j] = row[2].strip()\n",
    "                    else:\n",
    "                        #convert numbers to floats\n",
    "                        row = [float(i) for i in row] #if this fails, go back to keys\n",
    "\n",
    "                    #add values to corresponding keys\n",
    "                    for x in range(0, len(current_keys)):\n",
    "                        temp[current_keys[x]].append(row[x])\n",
    "\n",
    "                    #check if there's another row\n",
    "                    if current_row < lines:\n",
    "                        row = next(csvReader)\n",
    "                        current_row += 1\n",
    "                    else:\n",
    "                        print(temp)\n",
    "                        jsonArray = temp\n",
    "                        #convert python jsonArray to JSON String and write to file\n",
    "                        with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: \n",
    "                            jsonString = json.dumps(jsonArray, indent=4)\n",
    "                            jsonf.write(jsonString)\n",
    "                        return\n",
    "                except ValueError:\n",
    "                    #back to keys loop\n",
    "                    keys = True\n",
    "\n",
    "#if cell 2 is run, then ./ is the top level of the ExaGO dir\n",
    "csvFilePath = r'./tmp.output'\n",
    "jsonFilePath = r'./data.json'\n",
    "\n",
    "csv_to_json(csvFilePath, jsonFilePath)\n",
    "\n",
    "print(f\"data converted from {csvFilePath} to {jsonFilePath}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ExaGO",
   "language": "python",
   "name": "py311-exago"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}