Spark_python.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mJSq2kJh6qQ8",
        "outputId": "4946e76e-6aad-4f50-b6f0-ce60d093684b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.3.0.tar.gz (281.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.3 MB 45 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.5\n",
            "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 56.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.3.0-py2.py3-none-any.whl size=281764026 sha256=ec7e88de0e24c599b647335ad031e6e5a31e6fea39986867deb9d22267a110e3\n",
            "  Stored in directory: /root/.cache/pip/wheels/7a/8e/1b/f73a52650d2e5f337708d9f6a1750d451a7349a867f928b885\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.5 pyspark-3.3.0\n"
          ]
        }
      ],
      "source": [
        "pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "EM2zAs2h7xzf"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder \\\n",
        "      .master(\"local[1]\") \\\n",
        "      .appName(\"SparkByExamples.com\") \\\n",
        "      .getOrCreate() \n",
        "\n",
        "        "
      ],
      "metadata": {
        "id": "rLmh4w-Q78cU"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataList = [(\"Java\", 20000), (\"Python\", 100000), (\"Scala\", 3000)]\n",
        "rdd=spark.sparkContext.parallelize(dataList)"
      ],
      "metadata": {
        "id": "8uTLmrHN9UOK"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jSwK1vwU_nfr",
        "outputId": "cfcc3a80-a9c0-416f-d90b-752b1bcb595a"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.collect()"
      ],
      "metadata": {
        "id": "6fChBQZzDVM3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9dbf7dc0-5d81-4be9-b67b-8a3a0da5548b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Java', 20000), ('Python', 100000), ('Scala', 3000)]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.first()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mFNWKgR7kbTL",
        "outputId": "8cf7de32-caef-4473-8b6f-6c1d841bdd1b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Java', 20000)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.max()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jYomuGixlWBz",
        "outputId": "79f57f98-88bb-461d-a87c-45595235fc85"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Scala', 3000)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.min()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vP4PoLf_lqM9",
        "outputId": "2af04e0b-6a47-4eab-ae9d-c03ae8e9bea7"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Java', 20000)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd = spark.sparkContext.parallelize([1.0,5.0,41.0,10.0])\n",
        "rdd.max()"
      ],
      "metadata": {
        "id": "yRHP77jGmDIR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53c59423-6dde-4821-c5fc-496002bdb243"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "41.0"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Creating dataframe\n",
        "data = [('James' , '' , 'Smith' , '1991-04-01' , 'M' ,3000),\n",
        "        ('Micheal' , 'Rose' , '' , '2000-05-19' , 'M' , 4000),\n",
        "        ('Robert' , '' , 'Williams' , '1978-09-05' , 'M' , 4000),\n",
        "        ('Maria', 'Anne' , 'Jones' , '1967-12-17', 'F' , 4000),\n",
        "        ('Jen' , 'Mary' , 'Brown' , '1980-02-17','F',-1)]\n",
        "columns = ['Firstname' , 'Middlename' , 'lastname' , 'DOB' , 'Gender', 'salary']\n",
        "df = spark.createDataFrame(data=data , schema =columns)        "
      ],
      "metadata": {
        "id": "BqPu9tMsLmoa"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wn_MuZC4OkWz",
        "outputId": "fb08a44f-ad7e-4e45-8a1b-836331e52d40"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+----------+--------+----------+------+------+\n",
            "|Firstname|Middlename|lastname|       DOB|Gender|salary|\n",
            "+---------+----------+--------+----------+------+------+\n",
            "|    James|          |   Smith|1991-04-01|     M|  3000|\n",
            "|  Micheal|      Rose|        |2000-05-19|     M|  4000|\n",
            "|   Robert|          |Williams|1978-09-05|     M|  4000|\n",
            "|    Maria|      Anne|   Jones|1967-12-17|     F|  4000|\n",
            "|      Jen|      Mary|   Brown|1980-02-17|     F|    -1|\n",
            "+---------+----------+--------+----------+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#creating dataframe from rdd\n",
        "columns = ['Language' , 'user_count']\n",
        "data = [('Java' , 20000),('Python' , 100000) , ('Scala' , 1000)] "
      ],
      "metadata": {
        "id": "xTa387IXOpGi"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd = spark.sparkContext.parallelize(data)"
      ],
      "metadata": {
        "id": "lZ5CfDqmSAKR"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dffromrdd1 = rdd.toDF()"
      ],
      "metadata": {
        "id": "pPAl_rP_STNs"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dffromrdd1.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jOocRXm-SUMm",
        "outputId": "53b78060-5662-4826-fa44-2dd44ae9dd56"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- _1: string (nullable = true)\n",
            " |-- _2: long (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "columns = ['Language' , 'user_count']\n",
        "dffromrdd1 = rdd.toDF(columns)\n",
        "dffromrdd1.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VLoqTsLzSx2C",
        "outputId": "ef707962-de74-47bb-956f-61f41c95f0e2"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Language: string (nullable = true)\n",
            " |-- user_count: long (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#one line code / 2nd way\n",
        "dffromrdd2= spark.createDataFrame(rdd).toDF(*columns)\n",
        "dffromrdd2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uMGEd8XuTwOT",
        "outputId": "55e0c8c5-fa9f-4dd1-8cd7-3326b27e2d65"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[Language: string, user_count: bigint]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Creating Dataframe from list\n",
        "columns = ['Language' , 'user_count']\n",
        "data = [('Java' , 20000),('Python' , 100000) , ('Scala' , 1000)] "
      ],
      "metadata": {
        "id": "RwJMJWOhVCcT"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dffromlist = spark.createDataFrame(data).toDF(*columns)\n",
        "dffromlist.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HWl3g2Q7V5dp",
        "outputId": "5688536f-d0ba-4051-aaf0-124ea72c4b68"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+----------+\n",
            "|Language|user_count|\n",
            "+--------+----------+\n",
            "|    Java|     20000|\n",
            "|  Python|    100000|\n",
            "|   Scala|      1000|\n",
            "+--------+----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.types import StructType , StructField, StringType , IntegerType\n",
        "data2 = [('James' , '' , 'Smith' , '36636' , 'M' ,3000),\n",
        "        ('Micheal' , 'Rose' , '' , '40288' , 'M' , 4000),\n",
        "        ('Robert' , '' , 'Williams' , '42114' , 'M' , 4000),\n",
        "        ('Maria', 'Anne' , 'Jones' , '39192', 'F' , 4000),\n",
        "        ('Jen' , 'Mary' , 'Brown' , '','F',-1)]\n",
        "\n",
        "schema = StructType([\\\n",
        "                     StructField('Firstname',StringType(),True),\\\n",
        "                     StructField('Middlename' , StringType(),True),\\\n",
        "                     StructField('Lastname',StringType(),True),\\\n",
        "                     StructField('ID',StringType(),True),\\\n",
        "                     StructField('Gender',StringType(),True),\\\n",
        "                     StructField('Salary',IntegerType(),True)])     \n",
        "df = spark.createDataFrame(data = data2 , schema = schema)\n",
        "df.printSchema()\n",
        "df.show(truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hw2suJBuWCkG",
        "outputId": "b9e053d7-b3c8-4c22-ae5b-a7054bc9202e"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Firstname: string (nullable = true)\n",
            " |-- Middlename: string (nullable = true)\n",
            " |-- Lastname: string (nullable = true)\n",
            " |-- ID: string (nullable = true)\n",
            " |-- Gender: string (nullable = true)\n",
            " |-- Salary: integer (nullable = true)\n",
            "\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|Firstname|Middlename|Lastname|ID   |Gender|Salary|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|James    |          |Smith   |36636|M     |3000  |\n",
            "|Micheal  |Rose      |        |40288|M     |4000  |\n",
            "|Robert   |          |Williams|42114|M     |4000  |\n",
            "|Maria    |Anne      |Jones   |39192|F     |4000  |\n",
            "|Jen      |Mary      |Brown   |     |F     |-1    |\n",
            "+---------+----------+--------+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Converting dataframe into pandas dataframe\n",
        "pandasDF = df.toPandas()"
      ],
      "metadata": {
        "id": "DE53VNmbaiZs"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pandasDF"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "yLun0VEHdUrk",
        "outputId": "82b70f80-a988-4a8b-cbc8-18f93092a367"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  Firstname Middlename  Lastname     ID Gender  Salary\n",
              "0     James                Smith  36636      M    3000\n",
              "1   Micheal       Rose            40288      M    4000\n",
              "2    Robert             Williams  42114      M    4000\n",
              "3     Maria       Anne     Jones  39192      F    4000\n",
              "4       Jen       Mary     Brown             F      -1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-44cb66cb-7b4d-40ba-804d-b21931abc106\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Firstname</th>\n",
              "      <th>Middlename</th>\n",
              "      <th>Lastname</th>\n",
              "      <th>ID</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Salary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>James</td>\n",
              "      <td></td>\n",
              "      <td>Smith</td>\n",
              "      <td>36636</td>\n",
              "      <td>M</td>\n",
              "      <td>3000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Micheal</td>\n",
              "      <td>Rose</td>\n",
              "      <td></td>\n",
              "      <td>40288</td>\n",
              "      <td>M</td>\n",
              "      <td>4000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Robert</td>\n",
              "      <td></td>\n",
              "      <td>Williams</td>\n",
              "      <td>42114</td>\n",
              "      <td>M</td>\n",
              "      <td>4000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Maria</td>\n",
              "      <td>Anne</td>\n",
              "      <td>Jones</td>\n",
              "      <td>39192</td>\n",
              "      <td>F</td>\n",
              "      <td>4000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Jen</td>\n",
              "      <td>Mary</td>\n",
              "      <td>Brown</td>\n",
              "      <td></td>\n",
              "      <td>F</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-44cb66cb-7b4d-40ba-804d-b21931abc106')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-44cb66cb-7b4d-40ba-804d-b21931abc106 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-44cb66cb-7b4d-40ba-804d-b21931abc106');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pandasDF.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zK2iRXhRdZLy",
        "outputId": "83c6848b-2811-4a5f-fb62-d197e7ea5c0f"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Default - displays 20 rows and 20 characters from column value\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ddeBw08Kdq3j",
        "outputId": "4e1ce239-42c7-49ee-d396-5a763d151d47"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+----------+--------+-----+------+------+\n",
            "|Firstname|Middlename|Lastname|   ID|Gender|Salary|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|    James|          |   Smith|36636|     M|  3000|\n",
            "|  Micheal|      Rose|        |40288|     M|  4000|\n",
            "|   Robert|          |Williams|42114|     M|  4000|\n",
            "|    Maria|      Anne|   Jones|39192|     F|  4000|\n",
            "|      Jen|      Mary|   Brown|     |     F|    -1|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Display full column contents\n",
        "df.show(truncate = False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jJXiD35WeI4-",
        "outputId": "848d079b-00d9-470d-ca53-21118e3ca283"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+----------+--------+-----+------+------+\n",
            "|Firstname|Middlename|Lastname|ID   |Gender|Salary|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|James    |          |Smith   |36636|M     |3000  |\n",
            "|Micheal  |Rose      |        |40288|M     |4000  |\n",
            "|Robert   |          |Williams|42114|M     |4000  |\n",
            "|Maria    |Anne      |Jones   |39192|F     |4000  |\n",
            "|Jen      |Mary      |Brown   |     |F     |-1    |\n",
            "+---------+----------+--------+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Display 2 rows and full columns\n",
        "df.show(2,truncate = False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Wz2yLT5e9OT",
        "outputId": "ea36a0ef-bd85-4f3e-e5e0-58767af04445"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+----------+--------+-----+------+------+\n",
            "|Firstname|Middlename|Lastname|ID   |Gender|Salary|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|James    |          |Smith   |36636|M     |3000  |\n",
            "|Micheal  |Rose      |        |40288|M     |4000  |\n",
            "+---------+----------+--------+-----+------+------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Display 2 rows and column values 5 characters\n",
        "df.show(2,truncate=5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BcpIawBmfGv7",
        "outputId": "a5515fb2-2383-4935-8635-4b722b3a603f"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+----------+--------+-----+------+------+\n",
            "|Firstname|Middlename|Lastname|   ID|Gender|Salary|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "|    James|          |   Smith|36636|     M|  3000|\n",
            "|    Mi...|      Rose|        |40288|     M|  4000|\n",
            "+---------+----------+--------+-----+------+------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Display Dataframe rows and columns vertically\n",
        "df.show(n=3 , truncate=25, vertical = True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GT2OIPPifRAe",
        "outputId": "f31fd456-9a5e-47b6-9cf6-cb83ad4b80e2"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-RECORD 0--------------\n",
            " Firstname  | James    \n",
            " Middlename |          \n",
            " Lastname   | Smith    \n",
            " ID         | 36636    \n",
            " Gender     | M        \n",
            " Salary     | 3000     \n",
            "-RECORD 1--------------\n",
            " Firstname  | Micheal  \n",
            " Middlename | Rose     \n",
            " Lastname   |          \n",
            " ID         | 40288    \n",
            " Gender     | M        \n",
            " Salary     | 4000     \n",
            "-RECORD 2--------------\n",
            " Firstname  | Robert   \n",
            " Middlename |          \n",
            " Lastname   | Williams \n",
            " ID         | 42114    \n",
            " Gender     | M        \n",
            " Salary     | 4000     \n",
            "only showing top 3 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#TO have a better understanding on truncate\n",
        "columns= ['SeqNo.' , 'Quote']\n",
        "data = [('1' , 'Be the change you wish to see in the world'),\n",
        "        ('2' , 'Everyone thinks of changing the world , but no one thinks of changing themselves'),\n",
        "        ('3', 'Be cool')]\n",
        "df = spark.createDataFrame(data,columns)\n",
        "df.show()        "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qce7Tpkltv7s",
        "outputId": "703b2640-4876-45a4-bf57-cf38812630ad"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------+\n",
            "|SeqNo.|               Quote|\n",
            "+------+--------------------+\n",
            "|     1|Be the change you...|\n",
            "|     2|Everyone thinks o...|\n",
            "|     3|             Be cool|\n",
            "+------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vtql7OmGvFJN",
        "outputId": "775c669a-766a-4172-acae-11627ebe407d"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------------------------------------------------------------------+\n",
            "|SeqNo.|Quote                                                                           |\n",
            "+------+--------------------------------------------------------------------------------+\n",
            "|1     |Be the change you wish to see in the world                                      |\n",
            "|2     |Everyone thinks of changing the world , but no one thinks of changing themselves|\n",
            "|3     |Be cool                                                                         |\n",
            "+------+--------------------------------------------------------------------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(2,truncate=25)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kcFdoLf0xMaV",
        "outputId": "087d9bbb-8907-49fc-8411-dcdb5a57e9c4"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-------------------------+\n",
            "|SeqNo.|                    Quote|\n",
            "+------+-------------------------+\n",
            "|     1|Be the change you wish...|\n",
            "|     2|Everyone thinks of cha...|\n",
            "+------+-------------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(2,truncate=15)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3-UyWwDExb-Y",
        "outputId": "c6924602-eb25-4e6e-a3ff-77f0cf6648c7"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+---------------+\n",
            "|SeqNo.|          Quote|\n",
            "+------+---------------+\n",
            "|     1|Be the chang...|\n",
            "|     2|Everyone thi...|\n",
            "+------+---------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vePUcyYCxvsD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        ">> We use truncate for display and reviewing the data"
      ],
      "metadata": {
        "id": "v0PbPBo40kk0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Nesting\n",
        "structureData = [\n",
        "    ((\"James\",\"\",\"Smith\"),\"36636\",\"M\",3100),\n",
        "    ((\"Michael\",\"Rose\",\"\"),\"40288\",\"M\",4300),\n",
        "    ((\"Robert\",\"\",\"Williams\"),\"42114\",\"M\",1400),\n",
        "    ((\"Maria\",\"Anne\",\"Jones\"),\"39192\",\"F\",5500),\n",
        "    ((\"Jen\",\"Mary\",\"Brown\"),\"\",\"F\",-1)\n",
        "  ]\n",
        "structureSchema = StructType([\n",
        "        StructField('name', StructType([\n",
        "             StructField('firstname', StringType(), True),\n",
        "             StructField('middlename', StringType(), True),\n",
        "             StructField('lastname', StringType(), True)\n",
        "             ])),\n",
        "         StructField('id', StringType(), True),\n",
        "         StructField('gender', StringType(), True),\n",
        "         StructField('salary', IntegerType(), True)\n",
        "         ])\n",
        "\n",
        "df2 = spark.createDataFrame(data=structureData,schema=structureSchema)\n",
        "df2.printSchema()\n",
        "df2.show(truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cKRII7R-0s7C",
        "outputId": "f068906e-3b54-43f4-9e5e-2a06182de4e5"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- name: struct (nullable = true)\n",
            " |    |-- firstname: string (nullable = true)\n",
            " |    |-- middlename: string (nullable = true)\n",
            " |    |-- lastname: string (nullable = true)\n",
            " |-- id: string (nullable = true)\n",
            " |-- gender: string (nullable = true)\n",
            " |-- salary: integer (nullable = true)\n",
            "\n",
            "+--------------------+-----+------+------+\n",
            "|name                |id   |gender|salary|\n",
            "+--------------------+-----+------+------+\n",
            "|{James, , Smith}    |36636|M     |3100  |\n",
            "|{Michael, Rose, }   |40288|M     |4300  |\n",
            "|{Robert, , Williams}|42114|M     |1400  |\n",
            "|{Maria, Anne, Jones}|39192|F     |5500  |\n",
            "|{Jen, Mary, Brown}  |     |F     |-1    |\n",
            "+--------------------+-----+------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Adding and changing the structure of the data\n",
        "\n"
      ],
      "metadata": {
        "id": "VRSIyT5k5qSU"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col,struct,when\n",
        "updatedDF = df2.withColumn(\"OtherInfo\", \n",
        "    struct(col(\"id\").alias(\"identifier\"),\n",
        "    col(\"gender\").alias(\"gender\"),\n",
        "    col(\"salary\").alias(\"salary\"),\n",
        "    when(col(\"salary\").cast(IntegerType()) < 2000,\"Low\")\n",
        "      .when(col(\"salary\").cast(IntegerType()) < 4000,\"Medium\")\n",
        "      .otherwise(\"High\").alias(\"Salary_Grade\")\n",
        "  )).drop(\"id\",\"gender\",\"salary\")\n",
        "\n",
        "updatedDF.printSchema()\n",
        "updatedDF.show(truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L6X5J7HHBiS1",
        "outputId": "cda43d5c-a801-4058-e334-c718e2322f67"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- name: struct (nullable = true)\n",
            " |    |-- firstname: string (nullable = true)\n",
            " |    |-- middlename: string (nullable = true)\n",
            " |    |-- lastname: string (nullable = true)\n",
            " |-- OtherInfo: struct (nullable = false)\n",
            " |    |-- identifier: string (nullable = true)\n",
            " |    |-- gender: string (nullable = true)\n",
            " |    |-- salary: integer (nullable = true)\n",
            " |    |-- Salary_Grade: string (nullable = false)\n",
            "\n",
            "+--------------------+------------------------+\n",
            "|name                |OtherInfo               |\n",
            "+--------------------+------------------------+\n",
            "|{James, , Smith}    |{36636, M, 3100, Medium}|\n",
            "|{Michael, Rose, }   |{40288, M, 4300, High}  |\n",
            "|{Robert, , Williams}|{42114, M, 1400, Low}   |\n",
            "|{Maria, Anne, Jones}|{39192, F, 5500, High}  |\n",
            "|{Jen, Mary, Brown}  |{, F, -1, Low}          |\n",
            "+--------------------+------------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "WSOmkij2EgTG"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}