Group1_Data_Analysis_Ralph_Modeling_V2(2).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Group1_Data_Analysis_Ralph_Modeling_V2.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1RHV6KlU_6dj",
        "outputId": "c8f1737b-97dd-4684-e1f1-1da6f978030f"
      },
      "source": [
        "%%bash\n",
        "# Need to install pyspark\n",
        "# if pyspark is already installed, will print a message indicating pyspark already isntalled\n",
        "pip install pyspark\n",
        "pip install seaborn"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "  Downloading https://files.pythonhosted.org/packages/f0/26/198fc8c0b98580f617cb03cb298c6056587b8f0447e20fa40c5b634ced77/pyspark-3.0.1.tar.gz (204.2MB)\n",
            "Collecting py4j==0.10.9\n",
            "  Downloading https://files.pythonhosted.org/packages/9e/b6/6a4fb90cd235dc8e265a6a2067f2a2c99f0d91787f06aca4bcf7c23f3f80/py4j-0.10.9-py2.py3-none-any.whl (198kB)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py): started\n",
            "  Building wheel for pyspark (setup.py): finished with status 'done'\n",
            "  Created wheel for pyspark: filename=pyspark-3.0.1-py2.py3-none-any.whl size=204612243 sha256=f7336ec5213855df9ccd12e17f9856805f2cfe845a563c9a127f76b1de123a54\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/bd/07/031766ca628adec8435bb40f0bd83bb676ce65ff4007f8e73f\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.0.1\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.6/dist-packages (0.11.0)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.18.5)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.4.1)\n",
            "Requirement already satisfied: pandas>=0.23 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.1.4)\n",
            "Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.6/dist-packages (from seaborn) (3.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.23->seaborn) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.23->seaborn) (2018.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (1.3.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas>=0.23->seaborn) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bmu-n-IX_7jx",
        "outputId": "2b2aecbe-9733-48f7-e435-04f5d4613d1c"
      },
      "source": [
        "%matplotlib inline\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import SQLContext\n",
        "from pyspark.ml import feature\n",
        "import re\n",
        "from pyspark.sql import functions as fn\n",
        "from pyspark.sql.functions import col, regexp_replace, split\n",
        "from pyspark.sql.utils import AnalysisException\n",
        "from pyspark.sql import Row\n",
        "from pyspark.sql.types import IntegerType, DateType, FloatType, BooleanType\n",
        "import os\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "spark = SparkSession \\\n",
        "  .builder \\\n",
        "  .master(\"local[*]\")\\\n",
        "  .config(\"spark.memory.fraction\", 0.8) \\\n",
        "  .config(\"spark.executor.memory\", \"12g\") \\\n",
        "  .config(\"spark.driver.memory\", \"12g\")\\\n",
        "  .config(\"spark.memory.offHeap.enabled\",'true')\\\n",
        "  .config(\"spark.memory.offHeap.size\",\"12g\")\\\n",
        "  .getOrCreate()\n",
        "sc = spark.sparkContext\n",
        "sqlContext = SQLContext(sc)\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "from pyspark.sql.functions import col, udf\n",
        "from pyspark.ml.stat import Correlation\n",
        "from pyspark.ml.feature import VectorAssembler, StringIndexer\n",
        "from pyspark.sql.functions import create_map, lit\n",
        "from itertools import chain\n",
        "from pyspark.mllib import numpy as np\n",
        "from pyspark.ml.feature import PCA\n",
        "from pyspark.ml.clustering import KMeans\n",
        "from pyspark.ml.evaluation import ClusteringEvaluator\n",
        "from pyspark.ml.tuning import ParamGridBuilder"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "El-yjdShAI8a"
      },
      "source": [
        "new_df = spark.read.format(\"csv\").option(\"header\", \"true\").option('inferSchema','true').load(\"drive/Shared drives/IST 718 Group1/therightstuff.csv\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOp5mBoFAamY",
        "outputId": "f07e0c0f-12c1-4113-94b9-5964a3e555de"
      },
      "source": [
        "new_df.show(5)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "|back_legroom| bed|      body_type|cabin|         city|city_fuel_economy|daysonmarket|dealer_zip|engine_cylinders|engine_displacement|engine_type|fleet|frame_damaged|franchise_dealer|franchise_make|front_legroom|fuel_tank_volume|fuel_type|has_accidents|height|highway_fuel_economy|horsepower|interior_color|isCab|is_cpo|is_new|is_oemcpo|latitude|length|listed_date|listing_color|longitude|       major_options| make_name|maximum_seating|mileage|       model_name|owner_count|  price|salvage|savings_amount|seller_rating| sp_id|             sp_name|theft_title|transmission|transmission_display|wheel_system|wheelbase|width|year|torque_ftlb|torque_rpm|torque_grade|n_body_type|n_engine_type|n_fuel_type|n_transmission|n_wheel_system|n_city|n_make_name|n_dealer_zip|n_franchise_make|n_sp_name|n_interior_color|n_model_name|n_listing_color|kmeans| pc1| pc2|  pc3|pc4|   pc5|pc6|pc7|pc8| pc9|pc10|pc11|pc12|pc13| pc14|pc15|pc16|pc17|pc18|   pc19| pc20|    pc21|pc22|    pc23|pc24|pc25|  pc26|     pc27|pc28|pc29| pc30|pc31|  pc32| pc33|  pc34|       pc35|pc36|pc37|pc38|pc39|pc40|  pc41|pc42|  pc43|pc44|   pc45|pc46| pc47|pc48|\n",
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "|        38.4|null|SUV / Crossover| null| Little Ferry|             28.0|         104|      7643|               6|               3500|          V|false|        false|           false|          null|         43.2|            17.2|   Hybrid|         true|  69.3|                28.0|       280|          Gray|false| false| false|    false| 40.8558| 188.4|       null|        BLACK| -74.0493|Navigation System...|    Toyota|              7| 109851|Highlander Hybrid|          1|14999.0|  false|          3160|    3.5777779|351457|    Magic Auto Sales|      false|         CVT|                   6|         AWD|    109.8| 75.2|2013|        258|      4000|     0.06875|          6|            4|          7|             2|             3|1616.0|        2.0|      2271.0|            47.0|   5118.0|             2.0|       114.0|            1.0|     0|38.4|28.0|104.0|6.0|3500.0|0.0|0.0|0.0|43.2|17.2| 1.0|69.3|28.0|280.0| 0.0| 0.0| 0.0| 0.0|40.8558|188.4|-74.0493| 7.0|109851.0| 1.0| 0.0|3160.0|3.5777779| 0.0| 6.0|109.8|75.2|2013.0|258.0|4000.0|    0.06875| 6.0| 4.0| 7.0| 2.0| 3.0|1616.0| 2.0|2271.0|47.0| 5118.0| 2.0|114.0| 1.0|\n",
            "|        35.1|null|          Sedan| null|  Los Angeles|             32.0|          23|     90045|               4|               2000|          I|false|        false|           false|          null|         42.0|            15.8|   Diesel|        false|  56.3|                45.0|       180|          null|false| false| false|    false| 33.9592| 182.5|       null|        WHITE| -118.381|Sport Package, Le...|       BMW|              5|  97500|         3 Series|          2|13600.0|  false|          1036|          5.0|402955|   Cartag Auto Sales|      false|           A|                   6|         RWD|    110.6| 80.0|2014|        280|      1750|        0.16|          4|            1|          6|             3|             2|  64.0|       15.0|       574.0|            47.0|  19114.0|             1.0|        63.0|            0.0|     0|35.1|32.0| 23.0|4.0|2000.0|0.0|0.0|0.0|42.0|15.8| 0.0|56.3|45.0|180.0| 0.0| 0.0| 0.0| 0.0|33.9592|182.5|-118.381| 5.0| 97500.0| 2.0| 0.0|1036.0|      5.0| 0.0| 6.0|110.6|80.0|2014.0|280.0|1750.0|       0.16| 4.0| 1.0| 6.0| 3.0| 2.0|  64.0|15.0| 574.0|47.0|19114.0| 1.0| 63.0| 0.0|\n",
            "|        38.6|null|SUV / Crossover| null|Saint Charles|             18.0|          11|     55972|               6|               3600|          V|false|        false|            true|          Jeep|          4.3|            24.6| Gasoline|        false|  69.3|                25.0|       295|         Black|false| false|  true|    false| 43.8252| 189.8|       null|        WHITE| -91.7629|Sunroof/Moonroof,...|      Jeep|              5|     12|   Grand Cherokee|          1|37789.0|  false|             0|    4.7777777|435509|Whitewater Chrysl...|      false|           A|                   6|         RWD|    114.7| 84.8|2020|        260|      4000|       0.065|          6|            4|          3|             3|             2| 342.0|        5.0|      2981.0|             4.0|   3247.0|             0.0|         7.0|            0.0|     1|38.6|18.0| 11.0|6.0|3600.0|0.0|0.0|1.0| 4.3|24.6| 0.0|69.3|25.0|295.0| 0.0| 0.0| 1.0| 0.0|43.8252|189.8|-91.7629| 5.0|    12.0| 1.0| 0.0|   0.0|4.7777777| 0.0| 6.0|114.7|84.8|2020.0|260.0|4000.0|      0.065| 6.0| 4.0| 3.0| 3.0| 2.0| 342.0| 5.0|2981.0| 4.0| 3247.0| 0.0|  7.0| 0.0|\n",
            "|        35.8|null|SUV / Crossover| null|  East Meadow|             20.0|          48|     11554|               4|               2000|          I|false|        false|            true|           Kia|          4.1|            16.9| Gasoline|        false|  67.1|                23.0|       200|          null|false| false| false|    false| 40.7604| 174.5|       null|        BLACK| -73.5319|Sunroof/Moonroof,...|Volkswagen|              5|  40351|           Tiguan|          1|18879.0|  false|           824|    3.5555556|293986|       Autoworld Kia|      false|           A|                   6|         AWD|    102.5| 71.2|2017|        207|      1700| 0.121764705|          6|            1|          3|             3|             3|2771.0|       11.0|      4472.0|             7.0|  10563.0|             1.0|        53.0|            1.0|     1|35.8|20.0| 48.0|4.0|2000.0|0.0|0.0|1.0| 4.1|16.9| 0.0|67.1|23.0|200.0| 0.0| 0.0| 0.0| 0.0|40.7604|174.5|-73.5319| 5.0| 40351.0| 1.0| 0.0| 824.0|3.5555556| 0.0| 6.0|102.5|71.2|2017.0|207.0|1700.0|0.121764705| 6.0| 1.0| 3.0| 3.0| 3.0|2771.0|11.0|4472.0| 7.0|10563.0| 1.0| 53.0| 1.0|\n",
            "|        35.7|null|SUV / Crossover| null|      Madison|             18.0|           1|     37115|               6|               3600|          V|false|        false|            true|          Jeep|         41.2|            17.5| Gasoline|        false|  73.6|                23.0|       285|         Black|false| false|  true|    false| 36.3062| 166.8|       null|       SILVER| -86.6868|Backup Camera, Qu...|      Jeep|              4|      3|         Wrangler|          1|32190.0|  false|             0|     4.642857|276219|Rhythm Chrysler D...|      false|           A|                   8|         RWD|     96.8| 73.8|2020|        295|      3000|  0.09833334|          6|            4|          3|             3|             2|  22.0|        5.0|        67.0|             4.0|   1597.0|             0.0|       134.0|            3.0|     1|35.7|18.0|  1.0|6.0|3600.0|0.0|0.0|1.0|41.2|17.5| 0.0|73.6|23.0|285.0| 0.0| 0.0| 1.0| 0.0|36.3062|166.8|-86.6868| 4.0|     3.0| 1.0| 0.0|   0.0| 4.642857| 0.0| 8.0| 96.8|73.8|2020.0|295.0|3000.0| 0.09833334| 6.0| 4.0| 3.0| 3.0| 2.0|  22.0| 5.0|  67.0| 4.0| 1597.0| 0.0|134.0| 3.0|\n",
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9i47JnBVAjeD",
        "outputId": "4947b27e-da1c-4b02-9b59-39c6e35a95ae"
      },
      "source": [
        "new_df.describe().show()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+-----------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+-------------------+-----------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+\n",
            "|summary|      back_legroom|  bed|  body_type|      cabin|       city| city_fuel_economy|      daysonmarket|        dealer_zip|  engine_cylinders|engine_displacement|engine_type|franchise_make|    front_legroom|  fuel_tank_volume|fuel_type|           height|highway_fuel_economy|        horsepower|      interior_color|          latitude|            length|listed_date|listing_color|         longitude|       major_options| make_name|   maximum_seating|          mileage|       model_name|       owner_count|             price|   savings_amount|     seller_rating|             sp_id|             sp_name|transmission|transmission_display|wheel_system|         wheelbase|            width|              year|       torque_ftlb|        torque_rpm|       torque_grade|       n_body_type|     n_engine_type|       n_fuel_type|    n_transmission|    n_wheel_system|           n_city|      n_make_name|      n_dealer_zip|  n_franchise_make|        n_sp_name| n_interior_color|      n_model_name|   n_listing_color|            kmeans|               pc1|               pc2|               pc3|               pc4|               pc5|                pc6|                 pc7|                pc8|              pc9|              pc10|               pc11|             pc12|              pc13|              pc14|               pc15|              pc16|               pc17|               pc18|              pc19|              pc20|              pc21|              pc22|             pc23|              pc24|                pc25|             pc26|              pc27|                pc28|              pc29|              pc30|             pc31|              pc32|              pc33|              pc34|               pc35|              pc36|              pc37|              pc38|              pc39|              pc40|             pc41|             pc42|              pc43|              pc44|             pc45|             pc46|              pc47|              pc48|\n",
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+-----------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+-------------------+-----------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+\n",
            "|  count|            595365| 3871|     595365|      12684|     595365|            595365|            595365|            595365|            595365|             595365|     595365|        482257|           595365|            595365|   595365|           595365|              595365|            595365|              519048|            595365|            595365|          0|       595365|            595365|              556158|    595365|            595365|           595365|           595365|            595365|            595365|           595365|            595365|            595344|              595365|      595365|              595365|      595365|            595365|           595365|            595365|            595365|            595365|             595365|            595365|            595365|            595365|            595365|            595365|           595365|           595365|            595365|            595365|           595365|           595365|            595365|            595365|            595365|            595365|            595365|            595365|            595365|            595365|             595365|              595365|             595365|           595365|            595365|             595365|           595365|            595365|            595365|             595365|            595365|             595365|             595365|            595365|            595365|            595365|            595365|           595365|            595365|              595365|           595365|            595365|              595365|            595365|            595365|           595365|            595365|            595365|            595365|             595365|            595365|            595365|            595365|            595365|            595365|           595365|           595365|            595365|            595365|           595365|           595365|            595365|            595365|\n",
            "|   mean| 35.01224727688241| null|       null|       null|       null|22.736442057847125| 75.60854937727277| 50656.61339794898| 5.071114358418785| 2930.4633292182107|       null|          null|35.38860064834292|18.571329254829084|     null|65.89472213111952|  29.522490984365746|245.81981305585649|            21481.02| 36.99037174523052|193.64100233803762|       null|         null|-90.61292403651154|                null|      null|  5.45510569146658|30091.31102097033|1609.530466282742|1.2616344595332276|29122.640618459267|542.9157558808462| 4.269955669207039|233134.60238954285|                null|        null|  6.3325254255792665|        null|115.30140786024361|78.32899866640568|2017.7391348164572| 262.5168392498719|3595.6089642488223|0.09930819156954414| 5.795375945848345|2.2444399653993767|3.2405549536838745| 2.813519437655892|1.7568886313437975|627.3688090499105|6.888864814021651|1384.2782847496915|14.168782175640153|5473.464499928616|450.3719818934603| 78.89573286975217|2.5057418558363356|0.8231840971504876| 35.01224727688241|22.736442057847125| 75.60854937727277| 5.071114358418785|2930.4633292182107|  0.112740923635081|0.005003653221133254| 0.8112519210904235|35.38860064834292|18.571329254829084|0.08091338926540861|65.89472213111952|29.522490984365746|245.81981305585649|0.09656261285094019|0.0608987763808756|0.48972814995842884|0.04520084318023397| 36.99037174523052|193.64100233803762|-90.61292403651154|  5.45510569146658|30091.31102097033|1.2616344595332276|0.003493655152721...|542.9157558808462| 4.269955669207039|0.001766983279164...|6.3325254255792665|115.30140786024361|78.32899866640568|2017.7391348164572| 262.5168392498719|3595.6089642488223|0.09930819156954414| 5.795375945848345|2.2444399653993767|3.2405549536838745| 2.813519437655892|1.7568886313437975|627.3688090499105|6.888864814021651|1384.2782847496915|14.168782175640153|5473.464499928616|450.3719818934603| 78.89573286975217|2.5057418558363356|\n",
            "| stddev|10.612139839452606| null|       null|       null|       null| 8.065007005084531|108.29550835482237|27311.015607177556|1.4405978403711457|  1311.977623839942|       null|          null|14.46608386157531| 5.311582742562993|     null|7.326111761912236|   7.104947554252336| 85.65004202341784|  49254.437036010124|5.0005631655065175|19.728630325982667|       null|         null|13.861869264110142|                null|      null|1.1413298551459448|44497.12673159059|765.1391646283429|0.6954421586320668|15178.176015498808| 962.658260158554|0.5089753898842864|132359.27822546006|                null|        null|  0.9896728011849474|        null|14.149761991873081|7.465642730495498| 4.148136701570988|107.72906794260957|1152.5251163474845|0.11113554112494868|1.8537867557315062|1.4621862183350727|0.8337393507699541|0.4285941799657922|0.8024999199167335|758.9883265502424|7.296148579023628|1450.8472769748169|17.079718754349237|5100.142109093558|1927.094269696164|103.99791312070387| 2.192957836081287|0.3815131502644954|10.612139839452606| 8.065007005084531|108.29550835482237|1.4405978403711457| 1311.977623839942|0.31627610688848473| 0.07055937243118833|0.39130870014412633|14.46608386157531| 5.311582742562993|0.27270228750809133|7.326111761912236| 7.104947554252336| 85.65004202341784| 0.2953615093062176|0.2391447500474632|0.49989489778420737|0.20774455334780348|5.0005631655065175|19.728630325982667|13.861869264110142|1.1413298551459448|44497.12673159059|0.6954421586320668|  0.0590038589754403| 962.658260158554|0.5089753898842864|0.041998381063035516|0.9896728011849474|14.149761991873081|7.465642730495498| 4.148136701570988|107.72906794260957|1152.5251163474845|0.11113554112494868|1.8537867557315062|1.4621862183350727|0.8337393507699541|0.4285941799657922|0.8024999199167335|758.9883265502424|7.296148579023628|1450.8472769748169|17.079718754349237|5100.142109093558|1927.094269696164|103.99791312070387| 2.192957836081287|\n",
            "|    min|               0.0| Long|Convertible|   Crew Cab|  Abbeville|               8.0|                 0|        08816-4351|                 2|                700|          H|         Acura|              4.0|               1.9|Biodiesel|             43.9|                10.0|                67|              #NAME?|           18.3988|             106.1|       null|        BLACK|          -157.913|101A Mid Equipmen...|AM General|                 2|                0|         1 Series|                 1|             165.0|                0|               1.0|             41593|\"The Kar Store \"\"...|           A|                   1|         AWD|              73.5|             42.5|              1923|                68|               200|        0.015111111|                 1|                 1|                 1|                 1|                 1|              0.0|              0.0|               0.0|               0.0|              0.0|              0.0|               0.0|               0.0|                 0|               0.0|               8.0|               0.0|               2.0|             700.0|                0.0|                 0.0|                0.0|              4.0|               1.9|                0.0|             43.9|              10.0|              67.0|                0.0|               0.0|                0.0|                0.0|           18.3988|             106.1|          -157.913|               2.0|              0.0|               1.0|                 0.0|              0.0|               1.0|                 0.0|               1.0|              73.5|             42.5|            1923.0|              68.0|             200.0|        0.015111111|               1.0|               1.0|               1.0|               1.0|               1.0|              0.0|              0.0|               0.0|               0.0|              0.0|              0.0|               0.0|               0.0|\n",
            "|    max|              47.5|Short|      Wagon|Regular Cab|north hills|             127.0|              2979|             99403|                12|               8400|          W|         smart|             67.0|              63.5|  Propane|            117.6|               127.0|               797|xDrive35i Premium...|           48.9237|             290.0|       null|       YELLOW|          -66.0785|         Z71 Package|     smart|                15|          1225238|               xD|                16|           89999.0|            31773|               5.0|            440951|      vBs Auto Sales|           M|                  10|         RWD|             201.0|            109.0|              2021|               935|              7500|                1.0|                 9|                 5|                 8|                 3|                 3|           4494.0|             74.0|            7878.0|              47.0|          24600.0|          18299.0|            1115.0|              14.0|                 1|              47.5|             127.0|            2979.0|              12.0|            8400.0|                1.0|                 1.0|                1.0|             67.0|              63.5|                1.0|            117.6|             127.0|             797.0|                1.0|               1.0|                1.0|                1.0|           48.9237|             290.0|          -66.0785|              15.0|        1225238.0|              16.0|                 1.0|          31773.0|               5.0|                 1.0|              10.0|             201.0|            109.0|            2021.0|             935.0|            7500.0|                1.0|               9.0|               5.0|               8.0|               3.0|               3.0|           4494.0|             74.0|            7878.0|              47.0|          24600.0|          18299.0|            1115.0|              14.0|\n",
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+-----------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+-------------------+-----------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+------------------+------------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WLNZq6e-AnAK",
        "outputId": "80a5490e-4258-44ae-b837-3eaaa80c875b"
      },
      "source": [
        "new_df.printSchema()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- back_legroom: double (nullable = true)\n",
            " |-- bed: string (nullable = true)\n",
            " |-- body_type: string (nullable = true)\n",
            " |-- cabin: string (nullable = true)\n",
            " |-- city: string (nullable = true)\n",
            " |-- city_fuel_economy: double (nullable = true)\n",
            " |-- daysonmarket: integer (nullable = true)\n",
            " |-- dealer_zip: string (nullable = true)\n",
            " |-- engine_cylinders: integer (nullable = true)\n",
            " |-- engine_displacement: integer (nullable = true)\n",
            " |-- engine_type: string (nullable = true)\n",
            " |-- fleet: boolean (nullable = true)\n",
            " |-- frame_damaged: boolean (nullable = true)\n",
            " |-- franchise_dealer: boolean (nullable = true)\n",
            " |-- franchise_make: string (nullable = true)\n",
            " |-- front_legroom: double (nullable = true)\n",
            " |-- fuel_tank_volume: double (nullable = true)\n",
            " |-- fuel_type: string (nullable = true)\n",
            " |-- has_accidents: boolean (nullable = true)\n",
            " |-- height: double (nullable = true)\n",
            " |-- highway_fuel_economy: double (nullable = true)\n",
            " |-- horsepower: integer (nullable = true)\n",
            " |-- interior_color: string (nullable = true)\n",
            " |-- isCab: boolean (nullable = true)\n",
            " |-- is_cpo: boolean (nullable = true)\n",
            " |-- is_new: boolean (nullable = true)\n",
            " |-- is_oemcpo: boolean (nullable = true)\n",
            " |-- latitude: double (nullable = true)\n",
            " |-- length: double (nullable = true)\n",
            " |-- listed_date: string (nullable = true)\n",
            " |-- listing_color: string (nullable = true)\n",
            " |-- longitude: double (nullable = true)\n",
            " |-- major_options: string (nullable = true)\n",
            " |-- make_name: string (nullable = true)\n",
            " |-- maximum_seating: integer (nullable = true)\n",
            " |-- mileage: integer (nullable = true)\n",
            " |-- model_name: string (nullable = true)\n",
            " |-- owner_count: integer (nullable = true)\n",
            " |-- price: double (nullable = true)\n",
            " |-- salvage: boolean (nullable = true)\n",
            " |-- savings_amount: integer (nullable = true)\n",
            " |-- seller_rating: double (nullable = true)\n",
            " |-- sp_id: integer (nullable = true)\n",
            " |-- sp_name: string (nullable = true)\n",
            " |-- theft_title: boolean (nullable = true)\n",
            " |-- transmission: string (nullable = true)\n",
            " |-- transmission_display: integer (nullable = true)\n",
            " |-- wheel_system: string (nullable = true)\n",
            " |-- wheelbase: double (nullable = true)\n",
            " |-- width: double (nullable = true)\n",
            " |-- year: integer (nullable = true)\n",
            " |-- torque_ftlb: integer (nullable = true)\n",
            " |-- torque_rpm: integer (nullable = true)\n",
            " |-- torque_grade: double (nullable = true)\n",
            " |-- n_body_type: integer (nullable = true)\n",
            " |-- n_engine_type: integer (nullable = true)\n",
            " |-- n_fuel_type: integer (nullable = true)\n",
            " |-- n_transmission: integer (nullable = true)\n",
            " |-- n_wheel_system: integer (nullable = true)\n",
            " |-- n_city: double (nullable = true)\n",
            " |-- n_make_name: double (nullable = true)\n",
            " |-- n_dealer_zip: double (nullable = true)\n",
            " |-- n_franchise_make: double (nullable = true)\n",
            " |-- n_sp_name: double (nullable = true)\n",
            " |-- n_interior_color: double (nullable = true)\n",
            " |-- n_model_name: double (nullable = true)\n",
            " |-- n_listing_color: double (nullable = true)\n",
            " |-- kmeans: integer (nullable = true)\n",
            " |-- pc1: double (nullable = true)\n",
            " |-- pc2: double (nullable = true)\n",
            " |-- pc3: double (nullable = true)\n",
            " |-- pc4: double (nullable = true)\n",
            " |-- pc5: double (nullable = true)\n",
            " |-- pc6: double (nullable = true)\n",
            " |-- pc7: double (nullable = true)\n",
            " |-- pc8: double (nullable = true)\n",
            " |-- pc9: double (nullable = true)\n",
            " |-- pc10: double (nullable = true)\n",
            " |-- pc11: double (nullable = true)\n",
            " |-- pc12: double (nullable = true)\n",
            " |-- pc13: double (nullable = true)\n",
            " |-- pc14: double (nullable = true)\n",
            " |-- pc15: double (nullable = true)\n",
            " |-- pc16: double (nullable = true)\n",
            " |-- pc17: double (nullable = true)\n",
            " |-- pc18: double (nullable = true)\n",
            " |-- pc19: double (nullable = true)\n",
            " |-- pc20: double (nullable = true)\n",
            " |-- pc21: double (nullable = true)\n",
            " |-- pc22: double (nullable = true)\n",
            " |-- pc23: double (nullable = true)\n",
            " |-- pc24: double (nullable = true)\n",
            " |-- pc25: double (nullable = true)\n",
            " |-- pc26: double (nullable = true)\n",
            " |-- pc27: double (nullable = true)\n",
            " |-- pc28: double (nullable = true)\n",
            " |-- pc29: double (nullable = true)\n",
            " |-- pc30: double (nullable = true)\n",
            " |-- pc31: double (nullable = true)\n",
            " |-- pc32: double (nullable = true)\n",
            " |-- pc33: double (nullable = true)\n",
            " |-- pc34: double (nullable = true)\n",
            " |-- pc35: double (nullable = true)\n",
            " |-- pc36: double (nullable = true)\n",
            " |-- pc37: double (nullable = true)\n",
            " |-- pc38: double (nullable = true)\n",
            " |-- pc39: double (nullable = true)\n",
            " |-- pc40: double (nullable = true)\n",
            " |-- pc41: double (nullable = true)\n",
            " |-- pc42: double (nullable = true)\n",
            " |-- pc43: double (nullable = true)\n",
            " |-- pc44: double (nullable = true)\n",
            " |-- pc45: double (nullable = true)\n",
            " |-- pc46: double (nullable = true)\n",
            " |-- pc47: double (nullable = true)\n",
            " |-- pc48: double (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zr1pZdtbpUbP"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "# Ralph's Modeling in Support of Research Quesitons 2A and 2B (Price Setting and PRice Expectations).\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TXT0XWPYAywj"
      },
      "source": [
        "# Create a new copy of the DF to prepare for regression\n",
        "reg_df = new_df"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUG3MyVzsZxj"
      },
      "source": [
        "# Drop select columns\n",
        "reg_df = reg_df.drop(*['latitude', 'listed_date', 'listing_color', 'longitude','city', 'dealer_zip','interior_color','sp_name','sp_id'])"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57ACDKxVuPxH"
      },
      "source": [
        "# Modify booleans to 1's and 0's\n",
        "reg_df = reg_df.withColumn('HasAccidents',fn.when(col('has_accidents') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('has_accidents')\n",
        " \n",
        "reg_df = reg_df.withColumn('frameDamaged',fn.when(col('frame_damaged') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('frame_damaged')\n",
        " \n",
        "reg_df = reg_df.withColumn('franchiseDealer',fn.when(col('franchise_dealer') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('franchise_dealer')\n",
        " \n",
        "reg_df = reg_df.withColumn('isCabNew',fn.when(col('isCab') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('isCab')\n",
        " \n",
        "reg_df = reg_df.withColumn('isCpo',fn.when(col('is_cpo') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('is_cpo')\n",
        " \n",
        "reg_df = reg_df.withColumn('isNew',fn.when(col('is_new') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('is_new')\n",
        " \n",
        "reg_df = reg_df.withColumn('isOemcpo',fn.when(col('is_oemcpo') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('is_oemcpo')\n",
        " \n",
        "reg_df = reg_df.withColumn('Salvage',fn.when(col('salvage') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('salvage')\n",
        " \n",
        "reg_df = reg_df.withColumn('theftTitle',fn.when(col('theft_title') == 'True', 1).otherwise(0))\n",
        "reg_df = reg_df.drop('theft_title')"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tmhy2rYJsOc9"
      },
      "source": [
        "# Create catagories for length of vehicle\n",
        "reg_df = reg_df.withColumn('length_category',fn.when(col('length') < 176.4, \"compact\").when(col(\"length\") > 187.2, \"fullsize\").otherwise('midsize'))"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPZ6l6zWsR98"
      },
      "source": [
        "# create catagories for width of vehicle\n",
        "reg_df = reg_df.withColumn('width_category',fn.when(col('width') < 70.9, \"narrow\").when(col(\"width\") > 74.8, \"wide\").otherwise(\"regular\"))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3jJe_q5AuZu"
      },
      "source": [
        "# Create a column that captures teh age of the vehicle which is calculated by teh year, and subtracted from 2020\n",
        "reg_df = reg_df.withColumn('age',2020 - (col('year')))"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kSKnpkNPrF00",
        "outputId": "af89e5ee-2705-4c20-bc44-3068401be022"
      },
      "source": [
        "reg_df.count()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "595365"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JxgDTwYko-t-"
      },
      "source": [
        "# Develop logic to scope out new cars \n",
        "new_car_logic = (reg_df[\"isNew\"] == 0)\n",
        "reg_df = reg_df[new_car_logic]"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gjQdloPerKj8",
        "outputId": "bfebe346-daae-42dd-f0ad-b18870bcfcc8"
      },
      "source": [
        "reg_df.count()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "303798"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8h1CEVxvukL"
      },
      "source": [
        "from pyspark.ml import feature, Pipeline #import modules/packages for the entire process\n",
        "# create ordinal pipline for select features\n",
        "ord_cat_feat_eng_pipe = Pipeline(stages=[ \n",
        "                                         feature.StringIndexerModel.from_labels(['compact','midsize','fullsize'],handleInvalid = \"keep\", inputCol=\"length_category\", outputCol=\"length_category_idx\"),\n",
        "                                         feature.StringIndexerModel.from_labels(['narrow','regular','wide'],handleInvalid = \"keep\",inputCol=\"width_category\", outputCol=\"width_category_idx\")])"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5NG15HbLrXYd"
      },
      "source": [
        "#Fit and Transform \n",
        "reg_df = ord_cat_feat_eng_pipe.fit(reg_df).transform(reg_df)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ljVDX6kkwBNQ",
        "outputId": "6c926608-57a2-447c-8880-d05ca0370090"
      },
      "source": [
        "# Inspect the work\n",
        "reg_df.show(2)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+\n",
            "|back_legroom| bed|      body_type|cabin|city_fuel_economy|daysonmarket|engine_cylinders|engine_displacement|engine_type|fleet|franchise_make|front_legroom|fuel_tank_volume|fuel_type|height|highway_fuel_economy|horsepower|length|       major_options|make_name|maximum_seating|mileage|       model_name|owner_count|  price|savings_amount|seller_rating|transmission|transmission_display|wheel_system|wheelbase|width|year|torque_ftlb|torque_rpm|torque_grade|n_body_type|n_engine_type|n_fuel_type|n_transmission|n_wheel_system|n_city|n_make_name|n_dealer_zip|n_franchise_make|n_sp_name|n_interior_color|n_model_name|n_listing_color|kmeans| pc1| pc2|  pc3|pc4|   pc5|pc6|pc7|pc8| pc9|pc10|pc11|pc12|pc13| pc14|pc15|pc16|pc17|pc18|   pc19| pc20|    pc21|pc22|    pc23|pc24|pc25|  pc26|     pc27|pc28|pc29| pc30|pc31|  pc32| pc33|  pc34|   pc35|pc36|pc37|pc38|pc39|pc40|  pc41|pc42|  pc43|pc44|   pc45|pc46| pc47|pc48|HasAccidents|frameDamaged|franchiseDealer|isCabNew|isCpo|isNew|isOemcpo|theftTitle|length_category|width_category|age|length_category_idx|width_category_idx|\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+\n",
            "|        38.4|null|SUV / Crossover| null|             28.0|         104|               6|               3500|          V|false|          null|         43.2|            17.2|   Hybrid|  69.3|                28.0|       280| 188.4|Navigation System...|   Toyota|              7| 109851|Highlander Hybrid|          1|14999.0|          3160|    3.5777779|         CVT|                   6|         AWD|    109.8| 75.2|2013|        258|      4000|     0.06875|          6|            4|          7|             2|             3|1616.0|        2.0|      2271.0|            47.0|   5118.0|             2.0|       114.0|            1.0|     0|38.4|28.0|104.0|6.0|3500.0|0.0|0.0|0.0|43.2|17.2| 1.0|69.3|28.0|280.0| 0.0| 0.0| 0.0| 0.0|40.8558|188.4|-74.0493| 7.0|109851.0| 1.0| 0.0|3160.0|3.5777779| 0.0| 6.0|109.8|75.2|2013.0|258.0|4000.0|0.06875| 6.0| 4.0| 7.0| 2.0| 3.0|1616.0| 2.0|2271.0|47.0| 5118.0| 2.0|114.0| 1.0|           1|           0|              0|       0|    0|    0|       0|         0|       fullsize|          wide|  7|                2.0|               2.0|\n",
            "|        35.1|null|          Sedan| null|             32.0|          23|               4|               2000|          I|false|          null|         42.0|            15.8|   Diesel|  56.3|                45.0|       180| 182.5|Sport Package, Le...|      BMW|              5|  97500|         3 Series|          2|13600.0|          1036|          5.0|           A|                   6|         RWD|    110.6| 80.0|2014|        280|      1750|        0.16|          4|            1|          6|             3|             2|  64.0|       15.0|       574.0|            47.0|  19114.0|             1.0|        63.0|            0.0|     0|35.1|32.0| 23.0|4.0|2000.0|0.0|0.0|0.0|42.0|15.8| 0.0|56.3|45.0|180.0| 0.0| 0.0| 0.0| 0.0|33.9592|182.5|-118.381| 5.0| 97500.0| 2.0| 0.0|1036.0|      5.0| 0.0| 6.0|110.6|80.0|2014.0|280.0|1750.0|   0.16| 4.0| 1.0| 6.0| 3.0| 2.0|  64.0|15.0| 574.0|47.0|19114.0| 1.0| 63.0| 0.0|           0|           0|              0|       0|    0|    0|       0|         0|        midsize|          wide|  6|                1.0|               2.0|\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WzpTAR0waNK"
      },
      "source": [
        "# Create a Pipeline to transform feature that will be modeled\n",
        "tot_feat_eng_pipe = Pipeline(stages=[\n",
        "    feature.VectorAssembler(inputCols=[ \"length_category_idx\",\"width_category_idx\",\"n_body_type\",\"n_engine_type\",\"n_fuel_type\",\"n_make_name\",\"n_franchise_make\",\"n_model_name\",\"n_transmission\",\"n_wheel_system\",\n",
        "                                       \"age\", \"city_fuel_economy\", \"highway_fuel_economy\",\"daysonmarket\",\"engine_cylinders\", \"horsepower\",\"mileage\",\n",
        "                                        \"owner_count\", \"maximum_seating\", \"engine_displacement\", \"torque_ftlb\"], outputCol = 'features'),\n",
        "                                     feature.StandardScaler(withMean = False,inputCol=\"features\", outputCol=\"scaledFeatures\")])"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ByrBcjJVzxbD"
      },
      "source": [
        "# Fit and transform\n",
        "reg_df_prepped = tot_feat_eng_pipe.fit(reg_df).transform(reg_df)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qBZpSEBw0LAN",
        "outputId": "e9015822-7d8f-4f1e-fad2-a1c738085726"
      },
      "source": [
        "# Inspect the work\n",
        "reg_df_prepped.show(2)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+--------------------+--------------------+\n",
            "|back_legroom| bed|      body_type|cabin|city_fuel_economy|daysonmarket|engine_cylinders|engine_displacement|engine_type|fleet|franchise_make|front_legroom|fuel_tank_volume|fuel_type|height|highway_fuel_economy|horsepower|length|       major_options|make_name|maximum_seating|mileage|       model_name|owner_count|  price|savings_amount|seller_rating|transmission|transmission_display|wheel_system|wheelbase|width|year|torque_ftlb|torque_rpm|torque_grade|n_body_type|n_engine_type|n_fuel_type|n_transmission|n_wheel_system|n_city|n_make_name|n_dealer_zip|n_franchise_make|n_sp_name|n_interior_color|n_model_name|n_listing_color|kmeans| pc1| pc2|  pc3|pc4|   pc5|pc6|pc7|pc8| pc9|pc10|pc11|pc12|pc13| pc14|pc15|pc16|pc17|pc18|   pc19| pc20|    pc21|pc22|    pc23|pc24|pc25|  pc26|     pc27|pc28|pc29| pc30|pc31|  pc32| pc33|  pc34|   pc35|pc36|pc37|pc38|pc39|pc40|  pc41|pc42|  pc43|pc44|   pc45|pc46| pc47|pc48|HasAccidents|frameDamaged|franchiseDealer|isCabNew|isCpo|isNew|isOemcpo|theftTitle|length_category|width_category|age|length_category_idx|width_category_idx|            features|      scaledFeatures|\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+--------------------+--------------------+\n",
            "|        38.4|null|SUV / Crossover| null|             28.0|         104|               6|               3500|          V|false|          null|         43.2|            17.2|   Hybrid|  69.3|                28.0|       280| 188.4|Navigation System...|   Toyota|              7| 109851|Highlander Hybrid|          1|14999.0|          3160|    3.5777779|         CVT|                   6|         AWD|    109.8| 75.2|2013|        258|      4000|     0.06875|          6|            4|          7|             2|             3|1616.0|        2.0|      2271.0|            47.0|   5118.0|             2.0|       114.0|            1.0|     0|38.4|28.0|104.0|6.0|3500.0|0.0|0.0|0.0|43.2|17.2| 1.0|69.3|28.0|280.0| 0.0| 0.0| 0.0| 0.0|40.8558|188.4|-74.0493| 7.0|109851.0| 1.0| 0.0|3160.0|3.5777779| 0.0| 6.0|109.8|75.2|2013.0|258.0|4000.0|0.06875| 6.0| 4.0| 7.0| 2.0| 3.0|1616.0| 2.0|2271.0|47.0| 5118.0| 2.0|114.0| 1.0|           1|           0|              0|       0|    0|    0|       0|         0|       fullsize|          wide|  7|                2.0|               2.0|[2.0,2.0,6.0,4.0,...|[2.83970212794484...|\n",
            "|        35.1|null|          Sedan| null|             32.0|          23|               4|               2000|          I|false|          null|         42.0|            15.8|   Diesel|  56.3|                45.0|       180| 182.5|Sport Package, Le...|      BMW|              5|  97500|         3 Series|          2|13600.0|          1036|          5.0|           A|                   6|         RWD|    110.6| 80.0|2014|        280|      1750|        0.16|          4|            1|          6|             3|             2|  64.0|       15.0|       574.0|            47.0|  19114.0|             1.0|        63.0|            0.0|     0|35.1|32.0| 23.0|4.0|2000.0|0.0|0.0|0.0|42.0|15.8| 0.0|56.3|45.0|180.0| 0.0| 0.0| 0.0| 0.0|33.9592|182.5|-118.381| 5.0| 97500.0| 2.0| 0.0|1036.0|      5.0| 0.0| 6.0|110.6|80.0|2014.0|280.0|1750.0|   0.16| 4.0| 1.0| 6.0| 3.0| 2.0|  64.0|15.0| 574.0|47.0|19114.0| 1.0| 63.0| 0.0|           0|           0|              0|       0|    0|    0|       0|         0|        midsize|          wide|  6|                1.0|               2.0|[1.0,2.0,4.0,1.0,...|[1.41985106397242...|\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+--------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+--------------------+--------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2SzALwA60Uvb"
      },
      "source": [
        "# Set enable grid\n",
        "enable_grid_search = True"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-8Mc_4080aDG",
        "outputId": "ca9933ec-6a96-4d83-9186-6948ae97ac5d"
      },
      "source": [
        "from pyspark.ml import regression, evaluation\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
        "\n",
        "if enable_grid_search:\n",
        "  \n",
        "  #Create a Regressor estimator\n",
        "  lr = regression.LinearRegression(featuresCol='scaledFeatures', labelCol='price')\n",
        "\n",
        "  #Establish the grid\n",
        "  paramGrid = ParamGridBuilder()\\\n",
        "    .addGrid(lr.regParam, [0.0, 0.01, 0.02, 0.03, 0.4, 0.7])\\\n",
        "    .addGrid(lr.elasticNetParam, [0.0, 0.2, 0.4, 0.5, 0.7])\\\n",
        "    .build()\n",
        "    \n",
        "  # Establish the evaluator\n",
        "  evaluator = RegressionEvaluator()\\\n",
        "                .setMetricName(\"mse\")\\\n",
        "                .setLabelCol(\"price\")\\\n",
        "                .setPredictionCol(\"prediction\")                \n",
        "\n",
        "  #Establish the Cross Validation\n",
        "  cv = CrossValidator()\\\n",
        "    .setEstimator(lr)\\\n",
        "    .setEstimatorParamMaps(paramGrid)\\\n",
        "    .setEvaluator(evaluator)\\\n",
        "    .setNumFolds(3)\n",
        "\n",
        "  cv_final_model_fitted = cv.fit(reg_df_prepped)\n",
        "\n",
        " \n",
        "  print(\"The resulting scores from 3-folds cross validation on the data: \", cv_final_model_fitted.avgMetrics)\n",
        "  print(\"The lowest score from 3-folds cross validation on the data: \", min(cv_final_model_fitted.avgMetrics))\n",
        "\n",
        "  pass"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The resulting scores from 3-folds cross validation on the data:  [45850607.24333177, 45850607.24333177, 45850607.24333177, 45850607.24333177, 45850607.24333177, 45850607.247499146, 45850536.9501101, 45851138.6483395, 45850646.32272164, 45850463.86284231, 45850607.25222736, 45851138.424835876, 45850523.65511502, 45850662.800147206, 45850177.38018238, 45850607.25751623, 45850290.66692879, 45850017.14383565, 45850196.676829964, 45850520.09308778, 45850607.84716314, 45850460.0427683, 45852422.20554151, 45850543.87468446, 45850592.02999973, 45850608.887844816, 45850691.617569536, 45850602.42648987, 45851181.35898497, 45853927.52949476]\n",
            "The lowest score from 3-folds cross validation on the data:  45850017.14383565\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3dM05w9o0rmG",
        "outputId": "4a3a4af9-e1e6-48f1-f532-094f219baf60"
      },
      "source": [
        "# Let's extract the param map\n",
        "cv.extractParamMap()\n"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{Param(parent='CrossValidator_89c864ea8f49', name='collectSubModels', doc='Param for whether to collect a list of sub-models trained during tuning. If set to false, then only the single best sub-model will be available after fitting. If set to true, then all sub-models will be available. Warning: For large models, collecting all sub-models can cause OOMs on the Spark driver.'): False,\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='estimator', doc='estimator to be cross-validated'): LinearRegression_89b454909de6,\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='estimatorParamMaps', doc='estimator param maps'): [{Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_89b454909de6', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_89b454909de6', name='regParam', doc='regularization parameter (>= 0).'): 0.7}],\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='evaluator', doc='evaluator used to select hyper-parameters that maximize the validator metric'): RegressionEvaluator_b4bab44f5792,\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='numFolds', doc='number of folds for cross validation'): 3,\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='parallelism', doc='the number of threads to use when running parallel algorithms (>= 1).'): 1,\n",
              " Param(parent='CrossValidator_89c864ea8f49', name='seed', doc='random seed.'): 9160494785359703607}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L2jXu7nx09w1"
      },
      "source": [
        "## It turns out the best model had the following hyperparameter values:\n",
        "\n",
        "elasticNetParam = 0.4\n",
        "\n",
        "regParam = 0.03\n",
        "\n",
        "The lowest score from 3-folds cross validation on the data:  63937702.61042934"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p-uH5R2g045k"
      },
      "source": [
        "# Now we'll partition the data\n",
        "training_reg_df_prepped , testing_reg_df_prepped = reg_df_prepped.randomSplit([0.7, 0.3])"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FdS2TZ0W1WVf",
        "outputId": "7658eb54-ed2d-41a0-a95e-ec67faad2eb5"
      },
      "source": [
        "# Inspect the split\n",
        "[training_reg_df_prepped.count(), testing_reg_df_prepped.count()]"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[212271, 91527]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d2FiLGSd1a0Z",
        "outputId": "85e9a2ac-4c20-49b9-8821-a1bf99595eff"
      },
      "source": [
        "from pyspark.ml import regression, evaluation\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
        "\n",
        "#Create a Regressor estimator\n",
        "lr_tuned = regression.LinearRegression(featuresCol='scaledFeatures', labelCol='price', elasticNetParam=0.4, regParam=0.03)\n",
        "\n",
        "# Create the model and preditions\n",
        "trainModel = lr_tuned.fit(training_reg_df_prepped)\n",
        "\n",
        "predicitons = trainModel.transform(testing_reg_df_prepped)\n",
        "\n",
        "# Establish the evaluator\n",
        "evaluator = RegressionEvaluator()\\\n",
        "  .setMetricName(\"mse\")\\\n",
        "  .setLabelCol(\"price\")\\\n",
        "  .setPredictionCol(\"prediction\") \n",
        "\n",
        "TestMSE = evaluator.evaluate(trainModel.transform(testing_reg_df_prepped))\n",
        "\n",
        "print(\"Test data MSE :\", TestMSE) "
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test data MSE : 45872695.899535485\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XDeUsWth2DVH",
        "outputId": "961d1f2c-1fba-484c-8b09-81b9848d6ce9"
      },
      "source": [
        "# Based on what we learned above, lets review some of the important statistics\n",
        "trainingSummary = trainModel.summary\n",
        "print(\"Fit model values\")\n",
        "print(\"********************\")\n",
        "print(\"MSE: %f\" % round(trainingSummary.meanSquaredError))\n",
        "print(\"r2: %f\" % trainingSummary.r2)\n",
        "print(\"Explained Variance: %f\" % round(trainingSummary.explainedVariance))\n",
        "\n",
        "print(\"********************\")\n",
        "print(\"Test data MSE :\", round(TestMSE)) "
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fit model values\n",
            "********************\n",
            "MSE: 45812726.000000\n",
            "r2: 0.703556\n",
            "Explained Variance: 108686859.000000\n",
            "********************\n",
            "Test data MSE : 45872696\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nxeqdP_C_nA4",
        "outputId": "94f92a74-7aab-42fd-f33d-8abd9a7a7720"
      },
      "source": [
        "# Inspect the coeficients\n",
        "Coefficients = trainModel.coefficients\n",
        "Coefficients"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DenseVector([118.6614, -48.8494, 1217.8965, -1572.1302, 49.1582, 842.823, -515.0671, 666.1964, -370.0792, 1151.5957, -1122.9249, 902.7017, -762.102, -35.4921, 1926.3923, 6580.0107, -5195.0119, -812.0935, -23.8504, -1953.6029, 1481.1041])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uKq8Mw0m_uzL"
      },
      "source": [
        "Now I'll create a new model and set elasticNetParam = 0 and solver to 0 so I can extract P-values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iI_jc10i_vgd",
        "outputId": "97dc7a30-45f0-4f7b-bbb4-b402bd4ee73e"
      },
      "source": [
        "#Create a Regressor estimator\n",
        "lr_tuned2 = regression.LinearRegression(featuresCol='scaledFeatures', labelCol='price', elasticNetParam=0, regParam=0.03, solver='normal')\n",
        "\n",
        "# Create the model and preditions\n",
        "trainModel2 = lr_tuned2.fit(training_reg_df_prepped)\n",
        "\n",
        "predicitons2 = trainModel2.transform(testing_reg_df_prepped)\n",
        "\n",
        "# Establish the evaluator\n",
        "evaluator2 = RegressionEvaluator()\\\n",
        "  .setMetricName(\"mse\")\\\n",
        "  .setLabelCol(\"price\")\\\n",
        "  .setPredictionCol(\"prediction\") \n",
        "\n",
        "TestMSE2 = evaluator2.evaluate(trainModel2.transform(testing_reg_df_prepped))\n",
        "\n",
        "print(\"Test data MSE :\", TestMSE2) "
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test data MSE : 45872590.7874914\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ghEDtOt__8e0",
        "outputId": "621d68cb-a935-4eb8-bc54-5a5816eadc8b"
      },
      "source": [
        "# Based on what we learned above, lets review some of the important statistics\n",
        "trainingSummary2 = trainModel2.summary\n",
        "print(\"Fit model values\")\n",
        "print(\"********************\")\n",
        "print(\"MSE: %f\" % round(trainingSummary2.meanSquaredError))\n",
        "print(\"r2: %f\" % trainingSummary2.r2)\n",
        "print(\"Explained Variance: %f\" % round(trainingSummary2.explainedVariance))\n",
        "\n",
        "print(\"********************\")\n",
        "print(\"Test data MSE :\", round(TestMSE2)) "
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fit model values\n",
            "********************\n",
            "MSE: 45812245.000000\n",
            "r2: 0.703559\n",
            "Explained Variance: 108727983.000000\n",
            "********************\n",
            "Test data MSE : 45872591\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "82wtyR1QAAzv"
      },
      "source": [
        "#Run this custom funciton to provide an organized output\n",
        "def modelsummary(model):\n",
        "    import numpy as np\n",
        "    print (\"Note: the last rows are the information for Intercept\")\n",
        "    print (\"##\",\"-------------------------------------------------\")\n",
        "    print (\"##\",\"  Estimate   |   Std.Error | t Values  |  P-value\")\n",
        "    coef = np.append(list(model.coefficients),model.intercept)\n",
        "    Summary=model.summary\n",
        "\n",
        "    for i in range(len(Summary.pValues)):\n",
        "        print (\"##\",'{:10.6f}'.format(coef[i]),\\\n",
        "        '{:10.6f}'.format(Summary.coefficientStandardErrors[i]),\\\n",
        "        '{:8.3f}'.format(Summary.tValues[i]),\\\n",
        "        '{:10.6f}'.format(Summary.pValues[i]))\n",
        "\n",
        "    print (\"##\",'---')\n",
        "    print (\"##\",\"Mean squared error: % .6f\" \\\n",
        "           % Summary.meanSquaredError, \", RMSE: % .6f\" \\\n",
        "           % Summary.rootMeanSquaredError )\n",
        "    print (\"##\",\"Multiple R-squared: %f\" % Summary.r2, \", \\\n",
        "            Total iterations: %i\"% Summary.totalIterations)"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFGnIzddAExX",
        "outputId": "5a67e743-f001-46d8-95d0-f5216794de63"
      },
      "source": [
        "# Run the function\n",
        "modelsummary(trainModel2)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Note: the last rows are the information for Intercept\n",
            "## -------------------------------------------------\n",
            "##   Estimate   |   Std.Error | t Values  |  P-value\n",
            "## 114.064021  20.084593    5.679   0.000000\n",
            "## -54.205265  21.043883   -2.576   0.010001\n",
            "## 1216.293878  20.749958   58.617   0.000000\n",
            "## -1537.987342  32.002283  -48.059   0.000000\n",
            "##  48.511781  19.486430    2.490   0.012792\n",
            "## 842.597684  18.485219   45.582   0.000000\n",
            "## -514.805046  15.961651  -32.253   0.000000\n",
            "## 664.783735  19.923123   33.367   0.000000\n",
            "## -367.442060  15.745158  -23.337   0.000000\n",
            "## 1146.797172  17.298725   66.294   0.000000\n",
            "## -1120.012182  21.471974  -52.162   0.000000\n",
            "## 924.757748  53.716351   17.216   0.000000\n",
            "## -787.293626  55.725991  -14.128   0.000000\n",
            "## -35.553499  14.895314   -2.387   0.016992\n",
            "## 1865.647889  45.776519   40.756   0.000000\n",
            "## 6588.010484  39.429667  167.083   0.000000\n",
            "## -5198.090631  20.821892 -249.645   0.000000\n",
            "## -810.217510  18.054482  -44.876   0.000000\n",
            "## -25.145760  16.844520   -1.493   0.135487\n",
            "## -1924.781024  38.813502  -49.591   0.000000\n",
            "## 1484.606941  27.802421   53.398   0.000000\n",
            "## 4067.564640 223.174409   18.226   0.000000\n",
            "## ---\n",
            "## Mean squared error:  45812244.847883 , RMSE:  6768.474337\n",
            "## Multiple R-squared: 0.703559 ,             Total iterations: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9Zzi9zuAjOd"
      },
      "source": [
        "# Lets save these to variables\n",
        "coeffs = trainModel2.coefficients.toArray()\n",
        "coeffStandError = trainModel2.summary.coefficientStandardErrors\n",
        "tValues = trainModel2.summary.tValues\n",
        "pValues = trainModel2.summary.pValues\n",
        "absCoeffs = abs(trainModel2.coefficients.toArray())"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ksXHyLFM9XFk"
      },
      "source": [
        "# Create a summary df\n",
        "summary_df = pd.DataFrame(list(zip([ \"length_category_idx\",\"width_category_idx\",\"n_body_type\",\"n_engine_type\",\"n_fuel_type\",\"n_make_name\",\"n_franchise_make\",\"n_model_name\",\"n_transmission\",\"n_wheel_system\",\n",
        "                                       \"age\", \"city_fuel_economy\", \"highway_fuel_economy\",\"daysonmarket\",\"engine_cylinders\", \"horsepower\",\"mileage\",\n",
        "                                        \"owner_count\", \"maximum_seating\", \"engine_displacement\", \"torque_ftlb\"],\n",
        "                                   coeffs,coeffStandError,tValues,pValues,absCoeffs)),\n",
        "                          columns = [\"Feature\", 'Coefficients', 'Coefficient_SE', 'T_Values', \"P_Values\", \"AbsValue_Coeffs\" ]).sort_values(\"AbsValue_Coeffs\", ascending = False)"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 708
        },
        "id": "TNq3uKJ39qtn",
        "outputId": "ae329092-6c3d-4a22-b88c-36bc17efd102"
      },
      "source": [
        "# Review the DF\n",
        "summary_df"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Feature</th>\n",
              "      <th>Coefficients</th>\n",
              "      <th>Coefficient_SE</th>\n",
              "      <th>T_Values</th>\n",
              "      <th>P_Values</th>\n",
              "      <th>AbsValue_Coeffs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>horsepower</td>\n",
              "      <td>6588.010484</td>\n",
              "      <td>39.429667</td>\n",
              "      <td>167.082578</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>6588.010484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>mileage</td>\n",
              "      <td>-5198.090631</td>\n",
              "      <td>20.821892</td>\n",
              "      <td>-249.645450</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>5198.090631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>engine_displacement</td>\n",
              "      <td>-1924.781024</td>\n",
              "      <td>38.813502</td>\n",
              "      <td>-49.590502</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1924.781024</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>engine_cylinders</td>\n",
              "      <td>1865.647889</td>\n",
              "      <td>45.776519</td>\n",
              "      <td>40.755565</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1865.647889</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>n_engine_type</td>\n",
              "      <td>-1537.987342</td>\n",
              "      <td>32.002283</td>\n",
              "      <td>-48.058676</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1537.987342</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>torque_ftlb</td>\n",
              "      <td>1484.606941</td>\n",
              "      <td>27.802421</td>\n",
              "      <td>53.398478</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1484.606941</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>n_body_type</td>\n",
              "      <td>1216.293878</td>\n",
              "      <td>20.749958</td>\n",
              "      <td>58.616691</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1216.293878</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>n_wheel_system</td>\n",
              "      <td>1146.797172</td>\n",
              "      <td>17.298725</td>\n",
              "      <td>66.293741</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1146.797172</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>age</td>\n",
              "      <td>-1120.012182</td>\n",
              "      <td>21.471974</td>\n",
              "      <td>-52.161585</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>1120.012182</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>city_fuel_economy</td>\n",
              "      <td>924.757748</td>\n",
              "      <td>53.716351</td>\n",
              "      <td>17.215573</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>924.757748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>n_make_name</td>\n",
              "      <td>842.597684</td>\n",
              "      <td>18.485219</td>\n",
              "      <td>45.582239</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>842.597684</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>owner_count</td>\n",
              "      <td>-810.217510</td>\n",
              "      <td>18.054482</td>\n",
              "      <td>-44.876252</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>810.217510</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>highway_fuel_economy</td>\n",
              "      <td>-787.293626</td>\n",
              "      <td>55.725991</td>\n",
              "      <td>-14.127943</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>787.293626</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>n_model_name</td>\n",
              "      <td>664.783735</td>\n",
              "      <td>19.923123</td>\n",
              "      <td>33.367446</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>664.783735</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>n_franchise_make</td>\n",
              "      <td>-514.805046</td>\n",
              "      <td>15.961651</td>\n",
              "      <td>-32.252619</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>514.805046</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>n_transmission</td>\n",
              "      <td>-367.442060</td>\n",
              "      <td>15.745158</td>\n",
              "      <td>-23.336829</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>367.442060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>length_category_idx</td>\n",
              "      <td>114.064021</td>\n",
              "      <td>20.084593</td>\n",
              "      <td>5.679180</td>\n",
              "      <td>1.355181e-08</td>\n",
              "      <td>114.064021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>width_category_idx</td>\n",
              "      <td>-54.205265</td>\n",
              "      <td>21.043883</td>\n",
              "      <td>-2.575820</td>\n",
              "      <td>1.000093e-02</td>\n",
              "      <td>54.205265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>n_fuel_type</td>\n",
              "      <td>48.511781</td>\n",
              "      <td>19.486430</td>\n",
              "      <td>2.489516</td>\n",
              "      <td>1.279247e-02</td>\n",
              "      <td>48.511781</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>daysonmarket</td>\n",
              "      <td>-35.553499</td>\n",
              "      <td>14.895314</td>\n",
              "      <td>-2.386892</td>\n",
              "      <td>1.699237e-02</td>\n",
              "      <td>35.553499</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>maximum_seating</td>\n",
              "      <td>-25.145760</td>\n",
              "      <td>16.844520</td>\n",
              "      <td>-1.492815</td>\n",
              "      <td>1.354870e-01</td>\n",
              "      <td>25.145760</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Feature  Coefficients  ...      P_Values  AbsValue_Coeffs\n",
              "15            horsepower   6588.010484  ...  0.000000e+00      6588.010484\n",
              "16               mileage  -5198.090631  ...  0.000000e+00      5198.090631\n",
              "19   engine_displacement  -1924.781024  ...  0.000000e+00      1924.781024\n",
              "14      engine_cylinders   1865.647889  ...  0.000000e+00      1865.647889\n",
              "3          n_engine_type  -1537.987342  ...  0.000000e+00      1537.987342\n",
              "20           torque_ftlb   1484.606941  ...  0.000000e+00      1484.606941\n",
              "2            n_body_type   1216.293878  ...  0.000000e+00      1216.293878\n",
              "9         n_wheel_system   1146.797172  ...  0.000000e+00      1146.797172\n",
              "10                   age  -1120.012182  ...  0.000000e+00      1120.012182\n",
              "11     city_fuel_economy    924.757748  ...  0.000000e+00       924.757748\n",
              "5            n_make_name    842.597684  ...  0.000000e+00       842.597684\n",
              "17           owner_count   -810.217510  ...  0.000000e+00       810.217510\n",
              "12  highway_fuel_economy   -787.293626  ...  0.000000e+00       787.293626\n",
              "7           n_model_name    664.783735  ...  0.000000e+00       664.783735\n",
              "6       n_franchise_make   -514.805046  ...  0.000000e+00       514.805046\n",
              "8         n_transmission   -367.442060  ...  0.000000e+00       367.442060\n",
              "0    length_category_idx    114.064021  ...  1.355181e-08       114.064021\n",
              "1     width_category_idx    -54.205265  ...  1.000093e-02        54.205265\n",
              "4            n_fuel_type     48.511781  ...  1.279247e-02        48.511781\n",
              "13          daysonmarket    -35.553499  ...  1.699237e-02        35.553499\n",
              "18       maximum_seating    -25.145760  ...  1.354870e-01        25.145760\n",
              "\n",
              "[21 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1deyJQBtJ6VT"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "# Now I'll fit a model using PCA\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjyvnS5bKFCc"
      },
      "source": [
        "# Create a Pipeline to transform features\n",
        "pca_feat_eng_pipe = Pipeline(stages=[\n",
        "    feature.VectorAssembler(inputCols=[ \"pc1\",\"pc2\",\"pc3\",\"pc4\",\"pc5\",\"pc6\",\"pc7\",\"pc8\",\"pc9\", \"pc10\",\"pc11\", \"pc12\",\n",
        "                                       \"pc13\", \"pc14\", \"pc15\",\"pc16\",\"pc17\",\"pc18\",\"pc19\",\"pc20\",\"pc21\",\"pc22\",\"pc23\", \"pc24\",\"pc25\",\n",
        "                                       \"pc26\",\"pc27\",\"pc28\",\"pc29\",\"pc25\",\"pc30\",\"pc31\",\"pc32\",\"pc33\",\"pc34\",\"pc35\"], outputCol = 'features'),\n",
        "                                     feature.StandardScaler(withMean = False,inputCol=\"features\", outputCol=\"scaledFeatures\")])"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3Leg-TVO6i9"
      },
      "source": [
        "# Fit and transform for PCA\n",
        "reg_df_prepped_pca = pca_feat_eng_pipe.fit(reg_df).transform(reg_df)"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ODcCY0KWPV-1",
        "outputId": "f1d614e1-a713-4206-b9a5-3bb6a4ea47d3"
      },
      "source": [
        "reg_df_prepped_pca.show(2, False)"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|back_legroom|bed |body_type      |cabin|city_fuel_economy|daysonmarket|engine_cylinders|engine_displacement|engine_type|fleet|franchise_make|front_legroom|fuel_tank_volume|fuel_type|height|highway_fuel_economy|horsepower|length|major_options                                                                                                                                                                                                                                                                                            |make_name|maximum_seating|mileage|model_name       |owner_count|price  |savings_amount|seller_rating|transmission|transmission_display|wheel_system|wheelbase|width|year|torque_ftlb|torque_rpm|torque_grade|n_body_type|n_engine_type|n_fuel_type|n_transmission|n_wheel_system|n_city|n_make_name|n_dealer_zip|n_franchise_make|n_sp_name|n_interior_color|n_model_name|n_listing_color|kmeans|pc1 |pc2 |pc3  |pc4|pc5   |pc6|pc7|pc8|pc9 |pc10|pc11|pc12|pc13|pc14 |pc15|pc16|pc17|pc18|pc19   |pc20 |pc21    |pc22|pc23    |pc24|pc25|pc26  |pc27     |pc28|pc29|pc30 |pc31|pc32  |pc33 |pc34  |pc35   |pc36|pc37|pc38|pc39|pc40|pc41  |pc42|pc43  |pc44|pc45   |pc46|pc47 |pc48|HasAccidents|frameDamaged|franchiseDealer|isCabNew|isCpo|isNew|isOemcpo|theftTitle|length_category|width_category|age|length_category_idx|width_category_idx|features                                                                                                                                                                                              |scaledFeatures                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|38.4        |null|SUV / Crossover|null |28.0             |104         |6               |3500               |V          |false|null          |43.2         |17.2            |Hybrid   |69.3  |28.0                |280       |188.4 |Navigation System, Alloy Wheels, Bluetooth, Backup Camera                                                                                                                                                                                                                                                |Toyota   |7              |109851 |Highlander Hybrid|1          |14999.0|3160          |3.5777779    |CVT         |6                   |AWD         |109.8    |75.2 |2013|258        |4000      |0.06875     |6          |4            |7          |2             |3             |1616.0|2.0        |2271.0      |47.0            |5118.0   |2.0             |114.0       |1.0            |0     |38.4|28.0|104.0|6.0|3500.0|0.0|0.0|0.0|43.2|17.2|1.0 |69.3|28.0|280.0|0.0 |0.0 |0.0 |0.0 |40.8558|188.4|-74.0493|7.0 |109851.0|1.0 |0.0 |3160.0|3.5777779|0.0 |6.0 |109.8|75.2|2013.0|258.0|4000.0|0.06875|6.0 |4.0 |7.0 |2.0 |3.0 |1616.0|2.0 |2271.0|47.0|5118.0 |2.0 |114.0|1.0 |1           |0           |0              |0       |0    |0    |0       |0         |fullsize       |wide          |7  |2.0                |2.0               |[38.4,28.0,104.0,6.0,3500.0,0.0,0.0,0.0,43.2,17.2,1.0,69.3,28.0,280.0,0.0,0.0,0.0,0.0,40.8558,188.4,-74.0493,7.0,109851.0,1.0,0.0,3160.0,3.5777779,0.0,6.0,0.0,109.8,75.2,2013.0,258.0,4000.0,0.06875]|[3.913297062530403,3.6489333393458225,1.0016332880360113,4.12051686390947,2.723813315947174,0.0,0.0,0.0,3.1909798259707527,3.3635410424140546,2.737898053930924,9.626180459566143,3.961367228517195,3.328992065963559,0.0,0.0,0.0,0.0,8.275605009292995,10.35107666812058,-5.319607925913618,5.777351695974497,2.3104175708463486,1.1049475775433368,0.0,2.813230513866511,7.017865229252874,0.0,6.742528816598669,0.0,8.470004716438705,10.604833130720014,418.2149388663342,2.4975412496606864,3.635997118673155,0.728765356539714]|\n",
            "|35.1        |null|Sedan          |null |32.0             |23          |4               |2000               |I          |false|null          |42.0         |15.8            |Diesel   |56.3  |45.0                |180       |182.5 |Sport Package, Leather Seats, Driver Assistance Package, Power Package, Navigation System, Storage Package, Technology Package, Premium Wheels, Heat Package, Premium Package, Cold Weather Package, Luxury Package, Ambient Light Package, Appearance Package, Bluetooth, Backup Camera, Comfort Package|BMW      |5              |97500  |3 Series         |2          |13600.0|1036          |5.0          |A           |6                   |RWD         |110.6    |80.0 |2014|280        |1750      |0.16        |4          |1            |6          |3             |2             |64.0  |15.0       |574.0       |47.0            |19114.0  |1.0             |63.0        |0.0            |0     |35.1|32.0|23.0 |4.0|2000.0|0.0|0.0|0.0|42.0|15.8|0.0 |56.3|45.0|180.0|0.0 |0.0 |0.0 |0.0 |33.9592|182.5|-118.381|5.0 |97500.0 |2.0 |0.0 |1036.0|5.0      |0.0 |6.0 |110.6|80.0|2014.0|280.0|1750.0|0.16   |4.0 |1.0 |6.0 |3.0 |2.0 |64.0  |15.0|574.0 |47.0|19114.0|1.0 |63.0 |0.0 |0           |0           |0              |0       |0    |0    |0       |0         |midsize        |wide          |6  |1.0                |2.0               |[35.1,32.0,23.0,4.0,2000.0,0.0,0.0,0.0,42.0,15.8,0.0,56.3,45.0,180.0,0.0,0.0,0.0,0.0,33.9592,182.5,-118.381,5.0,97500.0,2.0,0.0,1036.0,5.0,0.0,6.0,0.0,110.6,80.0,2014.0,280.0,1750.0,0.16]           |[3.5769980962191967,4.17020953068094,0.2215150540848871,2.747011242606313,1.5564647519698136,0.0,0.0,0.0,3.102341497471565,3.0897644459384925,0.0,7.820403461379133,6.366483045831206,2.1400663281194308,0.0,0.0,0.0,0.0,6.878654331369908,10.026918746985169,-8.504341106230308,4.126679782838926,2.0506478152908847,2.2098951550866737,0.0,0.9223122823942106,9.807575295902065,0.0,6.742528816598669,0.0,8.531716954809843,11.281737373106397,418.4226959149513,2.710509883352683,1.5907487394195052,1.6960357388560616]          |\n",
            "+------------+----+---------------+-----+-----------------+------------+----------------+-------------------+-----------+-----+--------------+-------------+----------------+---------+------+--------------------+----------+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+---------------+-------+-----------------+-----------+-------+--------------+-------------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+----+------+---------+----+----+-----+----+------+-----+------+-------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+------------+------------+---------------+--------+-----+-----+--------+----------+---------------+--------------+---+-------------------+------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEY3aX04J-1R"
      },
      "source": [
        "# Set enable grid\n",
        "enable_grid_search = True"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dc6DYlRxPgnI",
        "outputId": "ae875f30-5c25-463f-fa4a-cd170a325e73"
      },
      "source": [
        "from pyspark.ml import regression, evaluation\n",
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
        "\n",
        "if enable_grid_search:\n",
        "  \n",
        "  #Create a Regressor estimator\n",
        "  lr_pca = regression.LinearRegression(featuresCol='scaledFeatures', labelCol='price')\n",
        "\n",
        "  #Establish the grid\n",
        "  paramGrid = ParamGridBuilder()\\\n",
        "    .addGrid(lr_pca.regParam, [0.0, 0.01, 0.02, 0.03, 0.4, 0.7])\\\n",
        "    .addGrid(lr_pca.elasticNetParam, [0.0, 0.2, 0.4, 0.5, 0.7])\\\n",
        "    .build()\n",
        "    \n",
        "  # Establish the evaluator\n",
        "  evaluator = RegressionEvaluator()\\\n",
        "                .setMetricName(\"mse\")\\\n",
        "                .setLabelCol(\"price\")\\\n",
        "                .setPredictionCol(\"prediction\")                \n",
        "\n",
        "  #Establish the Cross Validation\n",
        "  cv = CrossValidator()\\\n",
        "    .setEstimator(lr_pca)\\\n",
        "    .setEstimatorParamMaps(paramGrid)\\\n",
        "    .setEvaluator(evaluator)\\\n",
        "    .setNumFolds(3)\n",
        "\n",
        "  cv_final_model_fitted_pca = cv.fit(reg_df_prepped_pca)\n",
        "\n",
        " \n",
        "  print(\"The resulting scores from 3-folds cross validation on the data: \", cv_final_model_fitted_pca.avgMetrics)\n",
        "  print(\"The lowest score from 3-folds cross validation on the data: \", min(cv_final_model_fitted_pca.avgMetrics))\n",
        "  print(\"The best model is\", cv_final_model_fitted_pca.bestModel)\n",
        "\n",
        "  pass"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The resulting scores from 3-folds cross validation on the data:  [43612662.6041002, 43612662.6041002, 43612662.6041002, 43612662.6041002, 43612662.6041002, 43612671.08993922, 43613044.41068794, 43613071.73616887, 43613071.9584053, 43612714.175368525, 43612671.07850227, 43613071.50514676, 43612769.00886549, 43612034.54705628, 43611996.39532416, 43612671.06786284, 43612791.76957442, 43612063.40047908, 43611879.55416459, 43612876.49999614, 43612671.248065084, 43616206.95297031, 43616014.94558488, 43614062.009351484, 43613001.324314974, 43612672.21330192, 43613790.44200804, 43612957.8805073, 43613476.5388089, 43614121.24901378]\n",
            "The lowest score from 3-folds cross validation on the data:  43611879.55416459\n",
            "The best model is LinearRegressionModel: uid=LinearRegression_0ac5c30c04b1, numFeatures=36\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VlNQfrSjQTsO",
        "outputId": "050e1812-65bc-4344-f106-0d72e3107cfb"
      },
      "source": [
        "# Let's extract the param map\n",
        "cv.extractParamMap()"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{Param(parent='CrossValidator_c706f244d2ce', name='collectSubModels', doc='Param for whether to collect a list of sub-models trained during tuning. If set to false, then only the single best sub-model will be available after fitting. If set to true, then all sub-models will be available. Warning: For large models, collecting all sub-models can cause OOMs on the Spark driver.'): False,\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='estimator', doc='estimator to be cross-validated'): LinearRegression_0ac5c30c04b1,\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='estimatorParamMaps', doc='estimator param maps'): [{Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.0},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.01},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.02},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.03},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.4},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.2,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.4,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.5,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.7},\n",
              "  {Param(parent='LinearRegression_0ac5c30c04b1', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.7,\n",
              "   Param(parent='LinearRegression_0ac5c30c04b1', name='regParam', doc='regularization parameter (>= 0).'): 0.7}],\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='evaluator', doc='evaluator used to select hyper-parameters that maximize the validator metric'): RegressionEvaluator_89bc2b3b785b,\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='numFolds', doc='number of folds for cross validation'): 3,\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='parallelism', doc='the number of threads to use when running parallel algorithms (>= 1).'): 1,\n",
              " Param(parent='CrossValidator_c706f244d2ce', name='seed', doc='random seed.'): 9160494785359703607}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tEwsu2xmQYSa"
      },
      "source": [
        "## It turns out the best model had the following hyperparameter values:\n",
        "elasticNetParam = 0.5\n",
        "\n",
        "regParam = 0.03\n",
        "\n",
        "The lowest score from 3-folds cross validation on the data: 64596023\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7aMa46muQa4g"
      },
      "source": [
        "# Now we'll partition the data\n",
        "training_reg_df_prepped_pca, testing_reg_df_prepped_pca = reg_df_prepped_pca.randomSplit([0.7, 0.3])"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EFNWCQOXQhhL",
        "outputId": "4e8d5ec0-5a88-4cdb-8770-22a0fbe384e9"
      },
      "source": [
        "# Inspect the split\n",
        "[training_reg_df_prepped_pca.count(), testing_reg_df_prepped_pca.count()]"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[212671, 91127]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fx7VozM9Q10w"
      },
      "source": [
        "Now I'll create a new model and set elasticNetParam = 0 and solver to 0 so I can extract P-values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0vZUtnxBQwxx",
        "outputId": "327cbea7-4d2e-4b18-e2da-48345b9e2aca"
      },
      "source": [
        "#Create a Regressor estimator with PCA\n",
        "lr_pca_tuned2 = regression.LinearRegression(featuresCol='scaledFeatures', labelCol='price', elasticNetParam=0.5, regParam=0.03)\n",
        "\n",
        "# Create the model and preditions\n",
        "trainModel_pca = lr_pca_tuned2.fit(training_reg_df_prepped_pca)\n",
        "\n",
        "predicitons_pca = trainModel_pca.transform(testing_reg_df_prepped_pca)\n",
        "\n",
        "# Establish the evaluator\n",
        "evaluator_pca = RegressionEvaluator()\\\n",
        "  .setMetricName(\"mse\")\\\n",
        "  .setLabelCol(\"price\")\\\n",
        "  .setPredictionCol(\"prediction\") \n",
        "\n",
        "TestMSE3 = evaluator_pca.evaluate(trainModel_pca.transform(testing_reg_df_prepped_pca))\n",
        "\n",
        "print(\"Test data MSE :\", TestMSE3) "
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test data MSE : 43829980.91780948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wd-iyTOzUCTE",
        "outputId": "0773bc59-279c-451f-ec7a-ce46d3d158a2"
      },
      "source": [
        "# Based on what we learned above, lets review some of the important statistics\n",
        "trainModel_pca_Summary = trainModel_pca.summary\n",
        "print(\"Fit model values\")\n",
        "print(\"********************\")\n",
        "print(\"MSE: %f\" % round(trainModel_pca_Summary.meanSquaredError))\n",
        "print(\"r2: %f\" % trainModel_pca_Summary.r2)\n",
        "print(\"Explained Variance: %f\" % round(trainModel_pca_Summary.explainedVariance))\n",
        "\n",
        "print(\"********************\")\n",
        "print(\"Test data MSE :\", round(TestMSE3)) "
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fit model values\n",
            "********************\n",
            "MSE: 43482553.000000\n",
            "r2: 0.718172\n",
            "Explained Variance: 110832635.000000\n",
            "********************\n",
            "Test data MSE : 43829981\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}