Setting_up_pySpark_in_Google_Colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Setting up pySpark in Google Colab.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMZIrlbaRDe+mtLdj/mP0NS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/blessymoses/spark/blob/master/Setting_up_pySpark_in_Google_Colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "metadata": {
        "id": "rYdTumzU0TfQ"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setting up PySpark in Colab\n",
        "Ref: https://www.analyticsvidhya.com/blog/2020/11/a-must-read-guide-on-how-to-work-with-pyspark-on-google-colab-for-data-scientists/\n",
        "\n",
        "- To get Apache Spark download site:\n",
        "  - Choose the Spark release and package type in https://spark.apache.org/downloads.html\n",
        "  - Clik on the package link and get the download site"
      ],
      "metadata": {
        "id": "ZJLCkPl8ueOB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# install java\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "\n",
        "# install Apache Spark\n",
        "!wget -q https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz\n",
        "\n",
        "!tar xf spark-3.2.1-bin-hadoop2.7.tgz\n",
        "\n",
        "# install findspark to locate Spark on the system and import it as a regular library\n",
        "!pip install -q findspark"
      ],
      "metadata": {
        "id": "tcuuzkYTuke3"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.1-bin-hadoop2.7\""
      ],
      "metadata": {
        "id": "DeUvWzUHwAT4"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q findspark"
      ],
      "metadata": {
        "id": "r-Q4orkMvapu"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "\n",
        "# locate Spark in the system\n",
        "findspark.init()\n",
        "\n",
        "# to know where Spark is installed\n",
        "findspark.find()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "zIyXCa_MwKVe",
        "outputId": "3c39c688-ddef-4200-a226-9523c581578b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/spark-3.2.1-bin-hadoop2.7/python/pyspark'"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create Spark Session"
      ],
      "metadata": {
        "id": "Lfanc5Fp2cSG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "qejZyOs_x1yq"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark_session = SparkSession.builder\\\n",
        "        .master(\"local\")\\\n",
        "        .appName(\"Colab\")\\\n",
        "        .config('spark.ui.port', '4050')\\\n",
        "        .getOrCreate()"
      ],
      "metadata": {
        "id": "NFuSt47vx5wt"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark_session"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "U5-v4YS6x-cz",
        "outputId": "be09f40a-a454-48c4-9bc7-27a71cae7714"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://c26cbb4d7ac1:4050\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.2.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Colab</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f46bc8beb90>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark_session.stop()"
      ],
      "metadata": {
        "id": "ggPOC9_m4Tof"
      },
      "execution_count": 9,
      "outputs": []
    }
  ]
}