2022-10-24 - DBFS Example.ipynb

{"cells":[{"cell_type":"markdown","source":["## Overview\n\nThis notebook will show you how to create and query a table or DataFrame that you uploaded to DBFS. [DBFS](https://docs.databricks.com/user-guide/dbfs-databricks-file-system.html) is a Databricks File System that allows you to store data for querying inside of Databricks. This notebook assumes that you have a file already inside of DBFS that you would like to read from.\n\nThis notebook is written in **Python** so the default cell type is Python. However, you can use different languages by using the `%LANGUAGE` syntax. Python, Scala, SQL, and R are all supported."],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"96816ed7-b08a-4ca3-abb9-f99880c3535d"}}},{"cell_type":"code","source":["# File location and type\nfile_location = \"/FileStore/tables/tips-2.csv\"\nfile_type = \"csv\"\n\n# The applied options are for CSV files. For other file types, these will be ignored.\ndf = spark.read.csv(file_location, header=True, inferSchema=True)\ndf.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6482be4c-f067-47c9-b0ac-35c938b94601"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+\n|total_bill| tip|   sex|smoker|day|  time|size|\n+----------+----+------+------+---+------+----+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+\n|total_bill| tip|   sex|smoker|day|  time|size|\n+----------+----+------+------+---+------+----+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["df.printSchema()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"bd82bb99-1479-4d5c-be10-8c36df0f1d44"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- total_bill: double (nullable = true)\n |-- tip: double (nullable = true)\n |-- sex: string (nullable = true)\n |-- smoker: string (nullable = true)\n |-- day: string (nullable = true)\n |-- time: string (nullable = true)\n |-- size: integer (nullable = true)\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["root\n |-- total_bill: double (nullable = true)\n |-- tip: double (nullable = true)\n |-- sex: string (nullable = true)\n |-- smoker: string (nullable = true)\n |-- day: string (nullable = true)\n |-- time: string (nullable = true)\n |-- size: integer (nullable = true)\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["#get columns\ndf.columns"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b5f66379-6f7f-42ec-8e82-d0e0926a1721"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[4]: ['total_bill', 'tip', 'sex', 'smoker', 'day', 'time', 'size']","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[4]: ['total_bill', 'tip', 'sex', 'smoker', 'day', 'time', 'size']"]}}],"execution_count":0},{"cell_type":"code","source":["df.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"96651fb2-6cac-4c8c-b506-f82e10c2841e"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+\n|total_bill| tip|   sex|smoker|day|  time|size|\n+----------+----+------+------+---+------+----+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+\n|total_bill| tip|   sex|smoker|day|  time|size|\n+----------+----+------+------+---+------+----+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["#convert categorical features to numerical features\nfrom pyspark.ml.feature import StringIndexer"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"c09ff204-ae49-424f-98c4-f0040951bd6f"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["#convert categorical features to numerical features for single element\nindexer=StringIndexer(inputCol=\"sex\",outputCol=\"sex_indexed\")\ndf_r=indexer.fit(df).transform(df)\ndf_r.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"14c17c40-1877-4709-a031-4cb58f5726bc"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|\n+----------+----+------+------+---+------+----+-----------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|\n+----------+----+------+------+---+------+----+-----------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+-----------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|\n+----------+----+------+------+---+------+----+-----------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|\n+----------+----+------+------+---+------+----+-----------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["#convert categorical features to numerical features for multiple elements\nindexer=StringIndexer(inputCols=[\"smoker\",\"day\",\"time\"],outputCols=[\"smoker_indexed\",\"day_indexed\",\"time_indexed\"])\ndf_r=indexer.fit(df_r).transform(df_r)\ndf_r.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"e4281cb8-6864-428b-abfe-e353f256a6df"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["df_r.columns"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fda81819-8b67-4e8a-99bc-73289d06464f"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[10]: ['total_bill',\n 'tip',\n 'sex',\n 'smoker',\n 'day',\n 'time',\n 'size',\n 'sex_indexed',\n 'smoker_indexed',\n 'day_indexed',\n 'time_indexed']","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[10]: ['total_bill',\n 'tip',\n 'sex',\n 'smoker',\n 'day',\n 'time',\n 'size',\n 'sex_indexed',\n 'smoker_indexed',\n 'day_indexed',\n 'time_indexed']"]}}],"execution_count":0},{"cell_type":"code","source":["#separate dependent and independent features\nfrom pyspark.ml.feature import VectorAssembler\nfeatureassembler=VectorAssembler(inputCols=['tip','size','sex_indexed','smoker_indexed','day_indexed','time_indexed'], outputCol=\"Independent Features\")\noutput=featureassembler.transform(df_r)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ca39c69c-4cea-43b3-8221-9b143fe4ab70"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["output.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"23568628-0163-4b80-a9ac-dfe2ea1d8aa4"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[1.01,2.0,1.0,0.0...|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[1.66,3.0,0.0,0.0...|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.5,3.0,0.0,0.0,...|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.31,2.0,0.0,0.0...|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[3.61,4.0,1.0,0.0...|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[4.71,4.0,0.0,0.0...|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[2.0,2.0,0.0,0.0,...|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.12,4.0,0.0,0.0...|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.96,2.0,0.0,0.0...|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.23,2.0,0.0,0.0...|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.71,2.0,0.0,0.0...|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[5.0,4.0,1.0,0.0,...|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.57,2.0,0.0,0.0...|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.0,4.0,0.0,0.0,...|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[3.02,2.0,1.0,0.0...|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.92,2.0,0.0,0.0...|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[1.67,3.0,1.0,0.0...|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.71,3.0,0.0,0.0...|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[3.5,3.0,1.0,0.0,...|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[1.01,2.0,1.0,0.0...|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[1.66,3.0,0.0,0.0...|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.5,3.0,0.0,0.0,...|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.31,2.0,0.0,0.0...|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[3.61,4.0,1.0,0.0...|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[4.71,4.0,0.0,0.0...|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[2.0,2.0,0.0,0.0,...|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.12,4.0,0.0,0.0...|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.96,2.0,0.0,0.0...|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.23,2.0,0.0,0.0...|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.71,2.0,0.0,0.0...|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[5.0,4.0,1.0,0.0,...|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.57,2.0,0.0,0.0...|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.0,4.0,0.0,0.0,...|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[3.02,2.0,1.0,0.0...|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.92,2.0,0.0,0.0...|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[1.67,3.0,1.0,0.0...|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.71,3.0,0.0,0.0...|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[3.5,3.0,1.0,0.0,...|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["output.select('Independent Features').show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"a87b4e7f-fbd8-4416-9c75-5455c0a0ced8"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+\n|Independent Features|\n+--------------------+\n|[1.01,2.0,1.0,0.0...|\n|[1.66,3.0,0.0,0.0...|\n|[3.5,3.0,0.0,0.0,...|\n|[3.31,2.0,0.0,0.0...|\n|[3.61,4.0,1.0,0.0...|\n|[4.71,4.0,0.0,0.0...|\n|[2.0,2.0,0.0,0.0,...|\n|[3.12,4.0,0.0,0.0...|\n|[1.96,2.0,0.0,0.0...|\n|[3.23,2.0,0.0,0.0...|\n|[1.71,2.0,0.0,0.0...|\n|[5.0,4.0,1.0,0.0,...|\n|[1.57,2.0,0.0,0.0...|\n|[3.0,4.0,0.0,0.0,...|\n|[3.02,2.0,1.0,0.0...|\n|[3.92,2.0,0.0,0.0...|\n|[1.67,3.0,1.0,0.0...|\n|[3.71,3.0,0.0,0.0...|\n|[3.5,3.0,1.0,0.0,...|\n|(6,[0,1],[3.35,3.0])|\n+--------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+--------------------+\n|Independent Features|\n+--------------------+\n|[1.01,2.0,1.0,0.0...|\n|[1.66,3.0,0.0,0.0...|\n|[3.5,3.0,0.0,0.0,...|\n|[3.31,2.0,0.0,0.0...|\n|[3.61,4.0,1.0,0.0...|\n|[4.71,4.0,0.0,0.0...|\n|[2.0,2.0,0.0,0.0,...|\n|[3.12,4.0,0.0,0.0...|\n|[1.96,2.0,0.0,0.0...|\n|[3.23,2.0,0.0,0.0...|\n|[1.71,2.0,0.0,0.0...|\n|[5.0,4.0,1.0,0.0,...|\n|[1.57,2.0,0.0,0.0...|\n|[3.0,4.0,0.0,0.0,...|\n|[3.02,2.0,1.0,0.0...|\n|[3.92,2.0,0.0,0.0...|\n|[1.67,3.0,1.0,0.0...|\n|[3.71,3.0,0.0,0.0...|\n|[3.5,3.0,1.0,0.0,...|\n|(6,[0,1],[3.35,3.0])|\n+--------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["output.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6c015098-83e1-45c6-a699-7058b5438f72"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[1.01,2.0,1.0,0.0...|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[1.66,3.0,0.0,0.0...|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.5,3.0,0.0,0.0,...|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.31,2.0,0.0,0.0...|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[3.61,4.0,1.0,0.0...|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[4.71,4.0,0.0,0.0...|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[2.0,2.0,0.0,0.0,...|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.12,4.0,0.0,0.0...|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.96,2.0,0.0,0.0...|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.23,2.0,0.0,0.0...|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.71,2.0,0.0,0.0...|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[5.0,4.0,1.0,0.0,...|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.57,2.0,0.0,0.0...|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.0,4.0,0.0,0.0,...|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[3.02,2.0,1.0,0.0...|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.92,2.0,0.0,0.0...|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[1.67,3.0,1.0,0.0...|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.71,3.0,0.0,0.0...|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[3.5,3.0,1.0,0.0,...|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|total_bill| tip|   sex|smoker|day|  time|size|sex_indexed|smoker_indexed|day_indexed|time_indexed|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\n|     16.99|1.01|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[1.01,2.0,1.0,0.0...|\n|     10.34|1.66|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[1.66,3.0,0.0,0.0...|\n|     21.01| 3.5|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.5,3.0,0.0,0.0,...|\n|     23.68|3.31|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.31,2.0,0.0,0.0...|\n|     24.59|3.61|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[3.61,4.0,1.0,0.0...|\n|     25.29|4.71|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[4.71,4.0,0.0,0.0...|\n|      8.77| 2.0|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[2.0,2.0,0.0,0.0,...|\n|     26.88|3.12|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.12,4.0,0.0,0.0...|\n|     15.04|1.96|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.96,2.0,0.0,0.0...|\n|     14.78|3.23|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.23,2.0,0.0,0.0...|\n|     10.27|1.71|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.71,2.0,0.0,0.0...|\n|     35.26| 5.0|Female|    No|Sun|Dinner|   4|        1.0|           0.0|        1.0|         0.0|[5.0,4.0,1.0,0.0,...|\n|     15.42|1.57|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[1.57,2.0,0.0,0.0...|\n|     18.43| 3.0|  Male|    No|Sun|Dinner|   4|        0.0|           0.0|        1.0|         0.0|[3.0,4.0,0.0,0.0,...|\n|     14.83|3.02|Female|    No|Sun|Dinner|   2|        1.0|           0.0|        1.0|         0.0|[3.02,2.0,1.0,0.0...|\n|     21.58|3.92|  Male|    No|Sun|Dinner|   2|        0.0|           0.0|        1.0|         0.0|[3.92,2.0,0.0,0.0...|\n|     10.33|1.67|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[1.67,3.0,1.0,0.0...|\n|     16.29|3.71|  Male|    No|Sun|Dinner|   3|        0.0|           0.0|        1.0|         0.0|[3.71,3.0,0.0,0.0...|\n|     16.97| 3.5|Female|    No|Sun|Dinner|   3|        1.0|           0.0|        1.0|         0.0|[3.5,3.0,1.0,0.0,...|\n|     20.65|3.35|  Male|    No|Sat|Dinner|   3|        0.0|           0.0|        0.0|         0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+------------+--------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["finalized_data=output.select(\"Independent Features\", \"total_bill\")\nfinalized_data.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b8c69cb0-de40-4887-b789-3aec04afc183"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+----------+\n|Independent Features|total_bill|\n+--------------------+----------+\n|[1.01,2.0,1.0,0.0...|     16.99|\n|[1.66,3.0,0.0,0.0...|     10.34|\n|[3.5,3.0,0.0,0.0,...|     21.01|\n|[3.31,2.0,0.0,0.0...|     23.68|\n|[3.61,4.0,1.0,0.0...|     24.59|\n|[4.71,4.0,0.0,0.0...|     25.29|\n|[2.0,2.0,0.0,0.0,...|      8.77|\n|[3.12,4.0,0.0,0.0...|     26.88|\n|[1.96,2.0,0.0,0.0...|     15.04|\n|[3.23,2.0,0.0,0.0...|     14.78|\n|[1.71,2.0,0.0,0.0...|     10.27|\n|[5.0,4.0,1.0,0.0,...|     35.26|\n|[1.57,2.0,0.0,0.0...|     15.42|\n|[3.0,4.0,0.0,0.0,...|     18.43|\n|[3.02,2.0,1.0,0.0...|     14.83|\n|[3.92,2.0,0.0,0.0...|     21.58|\n|[1.67,3.0,1.0,0.0...|     10.33|\n|[3.71,3.0,0.0,0.0...|     16.29|\n|[3.5,3.0,1.0,0.0,...|     16.97|\n|(6,[0,1],[3.35,3.0])|     20.65|\n+--------------------+----------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+--------------------+----------+\n|Independent Features|total_bill|\n+--------------------+----------+\n|[1.01,2.0,1.0,0.0...|     16.99|\n|[1.66,3.0,0.0,0.0...|     10.34|\n|[3.5,3.0,0.0,0.0,...|     21.01|\n|[3.31,2.0,0.0,0.0...|     23.68|\n|[3.61,4.0,1.0,0.0...|     24.59|\n|[4.71,4.0,0.0,0.0...|     25.29|\n|[2.0,2.0,0.0,0.0,...|      8.77|\n|[3.12,4.0,0.0,0.0...|     26.88|\n|[1.96,2.0,0.0,0.0...|     15.04|\n|[3.23,2.0,0.0,0.0...|     14.78|\n|[1.71,2.0,0.0,0.0...|     10.27|\n|[5.0,4.0,1.0,0.0,...|     35.26|\n|[1.57,2.0,0.0,0.0...|     15.42|\n|[3.0,4.0,0.0,0.0,...|     18.43|\n|[3.02,2.0,1.0,0.0...|     14.83|\n|[3.92,2.0,0.0,0.0...|     21.58|\n|[1.67,3.0,1.0,0.0...|     10.33|\n|[3.71,3.0,0.0,0.0...|     16.29|\n|[3.5,3.0,1.0,0.0,...|     16.97|\n|(6,[0,1],[3.35,3.0])|     20.65|\n+--------------------+----------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["#deploy the linear regression model\nfrom pyspark.ml.regression import LinearRegression\n"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"1be5afcc-ace0-4fdd-8a3b-e5b169b57267"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["## dataset divided into train-test split\ntrain_data, test_data=finalized_data.randomSplit([0.75,0.25])\nregressor=LinearRegression(featuresCol=\"Independent Features\", labelCol=\"total_bill\")\nregressor=regressor.fit(train_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f6facc79-a61c-4c8a-8809-3791c773c3e2"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["regressor.coefficients"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d8c72964-56d5-4499-9439-d4536d61c934"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[24]: DenseVector([3.0454, 3.104, -1.3884, 2.1683, -0.0726, -1.6176])","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[24]: DenseVector([3.0454, 3.104, -1.3884, 2.1683, -0.0726, -1.6176])"]}}],"execution_count":0},{"cell_type":"code","source":["regressor.intercept"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"32ebccd3-d39d-477f-9ee0-60dda8aaf8e7"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[25]: 2.796194687674888","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[25]: 2.796194687674888"]}}],"execution_count":0},{"cell_type":"code","source":["## predictions\npred_results=regressor.evaluate(test_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"75ba8c32-fe99-483e-add6-9d3a4b89a2a7"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["##final comparison\npred_results.predictions.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"409254f0-0080-4838-9ca1-6e13c8f585ae"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"/databricks/spark/python/pyspark/sql/context.py:134: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n  warnings.warn(\n+--------------------+----------+------------------+\n|Independent Features|total_bill|        prediction|\n+--------------------+----------+------------------+\n| (6,[0,1],[2.0,2.0])|     12.69|15.095010909744882|\n|(6,[0,1],[2.24,3.0])|     16.04|18.929904945592202|\n| (6,[0,1],[2.5,4.0])|     18.35|22.825707246576037|\n| (6,[0,1],[5.0,3.0])|     31.27| 27.33524553443167|\n|(6,[0,1],[5.92,3.0])|     29.03|30.137025730711493|\n| (6,[0,1],[9.0,4.0])|     48.33| 42.62089341594436|\n|[1.32,2.0,0.0,0.0...|      9.68|12.951544595414168|\n|[1.5,2.0,0.0,1.0,...|     11.59|15.740583293162528|\n|[1.5,2.0,1.0,0.0,...|     26.41|12.183908592012346|\n|[1.66,3.0,0.0,0.0...|     10.34|17.090979956944075|\n|[1.71,2.0,0.0,0.0...|     10.27|14.139255765576266|\n|[1.76,2.0,0.0,1.0...|     11.24|16.532390739937263|\n|[1.8,2.0,1.0,0.0,...|     12.43|11.334797738033078|\n|[1.83,1.0,1.0,0.0...|     10.07| 8.322165281528752|\n|[2.0,2.0,0.0,0.0,...|     13.03|13.332276078717845|\n|[2.0,2.0,1.0,0.0,...|     10.33|11.943880389398256|\n|[2.0,2.0,1.0,0.0,...|     12.26|11.943880389398256|\n|[2.0,2.0,1.0,1.0,...|     10.63|15.874894232255887|\n|[2.0,2.0,1.0,1.0,...|     10.09|14.039574101539742|\n|[2.0,4.0,0.0,0.0,...|     24.55|21.230415318473984|\n+--------------------+----------+------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["/databricks/spark/python/pyspark/sql/context.py:134: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n  warnings.warn(\n+--------------------+----------+------------------+\n|Independent Features|total_bill|        prediction|\n+--------------------+----------+------------------+\n| (6,[0,1],[2.0,2.0])|     12.69|15.095010909744882|\n|(6,[0,1],[2.24,3.0])|     16.04|18.929904945592202|\n| (6,[0,1],[2.5,4.0])|     18.35|22.825707246576037|\n| (6,[0,1],[5.0,3.0])|     31.27| 27.33524553443167|\n|(6,[0,1],[5.92,3.0])|     29.03|30.137025730711493|\n| (6,[0,1],[9.0,4.0])|     48.33| 42.62089341594436|\n|[1.32,2.0,0.0,0.0...|      9.68|12.951544595414168|\n|[1.5,2.0,0.0,1.0,...|     11.59|15.740583293162528|\n|[1.5,2.0,1.0,0.0,...|     26.41|12.183908592012346|\n|[1.66,3.0,0.0,0.0...|     10.34|17.090979956944075|\n|[1.71,2.0,0.0,0.0...|     10.27|14.139255765576266|\n|[1.76,2.0,0.0,1.0...|     11.24|16.532390739937263|\n|[1.8,2.0,1.0,0.0,...|     12.43|11.334797738033078|\n|[1.83,1.0,1.0,0.0...|     10.07| 8.322165281528752|\n|[2.0,2.0,0.0,0.0,...|     13.03|13.332276078717845|\n|[2.0,2.0,1.0,0.0,...|     10.33|11.943880389398256|\n|[2.0,2.0,1.0,0.0,...|     12.26|11.943880389398256|\n|[2.0,2.0,1.0,1.0,...|     10.63|15.874894232255887|\n|[2.0,2.0,1.0,1.0,...|     10.09|14.039574101539742|\n|[2.0,4.0,0.0,0.0,...|     24.55|21.230415318473984|\n+--------------------+----------+------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["##Performance metrics\npred_results.r2,pred_results.meanAbsoluteError, pred_results.meanSquaredError"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"28f7b7dc-def8-49cb-83cc-ec8e5143f42d"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[29]: (0.6116551655228613, 5.066743400165097, 44.75218419390135)","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[29]: (0.6116551655228613, 5.066743400165097, 44.75218419390135)"]}}],"execution_count":0},{"cell_type":"code","source":["regressor.save(\"leanier_model.model\")"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"209107ff-7ddf-4701-9945-dac063b549f7"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":[""],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"be139770-0b28-4c80-9777-4c6507eb4945"}},"outputs":[],"execution_count":0}],"metadata":{"application/vnd.databricks.v1+notebook":{"notebookName":"2022-10-24 - DBFS Example","dashboards":[],"notebookMetadata":{"pythonIndentUnit":4},"language":"python","widgets":{},"notebookOrigID":2950377881468744}},"nbformat":4,"nbformat_minor":0}