Airbnb_Web_ScrapingColab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Airbnb Web Scraping",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Nada-gh/Web-Scraping-with-Selenium/blob/master/Airbnb_Web_ScrapingColab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "L8pqWyzD8cg9",
        "colab_type": "code",
        "outputId": "c612110d-ae8d-4eb5-dbbe-1433c92cf249",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install selenium\n",
        "!apt install chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin\n",
        "import sys\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n",
        "from selenium import webdriver\n",
        "chrome_options = webdriver.ChromeOptions()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: selenium in /usr/local/lib/python3.6/dist-packages (3.141.0)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.6/dist-packages (from selenium) (1.22)\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "chromium-chromedriver is already the newest version (71.0.3578.98-0ubuntu0.18.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 10 not upgraded.\n",
            "cp: cannot create regular file '/usr/bin/chromedriver': Text file busy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:11: DeprecationWarning: use options instead of chrome_options\n",
            "  # This is added back by InteractiveShellApp.init_path()\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "MZ0UUU4kT9XD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.common.exceptions import TimeoutException\n",
        "from selenium.common.exceptions import NoSuchElementException \n",
        "from selenium.common.exceptions import StaleElementReferenceException\n",
        "from selenium.common.exceptions import WebDriverException\n",
        "from selenium.webdriver.support.ui import WebDriverWait as wait\n",
        "from selenium.webdriver.common.keys import Keys #need to send keystrokes\n",
        "from datetime import datetime \n",
        "from datetime import timedelta\n",
        "from dateutil import parser\n",
        "import urllib\n",
        "from urllib.parse import urlparse\n",
        "from urllib.parse import parse_qs\n",
        "import csv\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0h2BC6p27fIe",
        "colab_type": "code",
        "outputId": "f047ed49-bb7d-4b63-88e0-3bd60433de81",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "cell_type": "code",
      "source": [
        "# create new instance of chrome \n",
        "#browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')\n",
        "browser = webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "\n",
        "start_date = '2019-03-06'\n",
        "start_date = parser.parse(start_date)\n",
        "\n",
        "SCROLL_PAUSE_TIME = 0.5\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: DeprecationWarning: use options instead of chrome_options\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "Zf1XSN2N85Ay",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "sgQb9rem7hzt",
        "colab_type": "code",
        "outputId": "39269d09-c878-4c91-abdd-d5f32d416f5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 369
        }
      },
      "cell_type": "code",
      "source": [
        "with open('AirBnbOneDayColab.csv','w',encoding=\"utf-8\",newline='') as csvfile:\n",
        "    fieldnames = ['Date','Has_Weekend','Title','Price','Super_host','Latitude','Longitude','EntireHomeVsRoom','Bedrooms_no','Beds_no','Bathrooms_num','Sleeping_engagment','Amenities','Accuracy_stars','Communication_stars','Cleanliness_stars','Location_stars','CheckIn_stars','Value_stars','Reviewers_no','Review']\n",
        "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
        "    writer.writeheader()\n",
        "    # how many days to loop through\n",
        "    for h in range(1,2):\n",
        "        Checkin_Date = start_date+timedelta(days=1)# check_in =19+1=20\n",
        "        Checkout_Date = start_date+timedelta(days=6)# check_out = 19+2=21\n",
        "        browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date.strftime('%Y-%m-%d')+\"&checkout=\"+Checkout_Date.strftime('%Y-%m-%d')+\"&map_toggle=false\")\n",
        "        browser.maximize_window()\n",
        "        last_height = browser.execute_script(\"return document.body.scrollHeight\")# Get scroll height\n",
        "        then = time.time()\n",
        "        while True:\n",
        "            browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")# Scroll down to bottom\n",
        "            time.sleep(SCROLL_PAUSE_TIME) # Wait to load page\n",
        "            new_height = browser.execute_script(\"return document.body.scrollHeight\")# Calculate new scroll height and compare with last scroll height\n",
        "            if new_height == last_height or time.time()-then > 60:\n",
        "                break\n",
        "        last_height = new_height\n",
        "        cont = browser.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
        "        prods = cont.find_elements_by_xpath(\"//meta[@itemprop='url']\")\n",
        "        links = [prod.get_attribute(\"content\") for prod in prods]  \n",
        "        for l in links: # 18 or 30 based on the num of the homes in the pag\n",
        "            driver = webdriver.Chrome('chromedriver',chrome_options=chrome_options)\n",
        "            driver.get(\"https://\"+l)\n",
        "            #driver.maximize_window()\n",
        "            tot = driver.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
        "\n",
        "            try:\n",
        "                title = tot.find_element_by_class_name('_18hrqvin').text\n",
        "            except NoSuchElementException:\n",
        "                title = None    \n",
        "            #try:\n",
        "            #elem = wait(driver,10).until(EC.presence_of_element_located((By.XPATH,\"//div[@class='_59m2yxn']/img\")))\n",
        "            map_url = driver.find_element_by_xpath(\"//*[@id='neighborhood']//div[@class='_59m2yxn']/img\").get_attribute(\"src\")\n",
        "            query = parse_qs(urlparse(map_url).query)['center'][0]\n",
        "            lat = query.split(\",\")[0]\n",
        "            long = query.split(\",\")[1]\n",
        "            #except NoSuchElementException:\n",
        "            #lat = None\n",
        "            #long = None\n",
        "                \n",
        "            try:\n",
        "                for i in range(0,5):\n",
        "                    day_no = Checkin_Date+timedelta(days=i)\n",
        "                    if(day_no.weekday()< 5):\n",
        "                        has_weekend = False\n",
        "                    else:\n",
        "                        has_weekend = True\n",
        "                        break\n",
        "                Has_weekend = has_weekend\n",
        "            except NoSuchElementException:\n",
        "                Has_weekend = None    \n",
        "\n",
        "            try:\n",
        "                price = tot.find_element_by_class_name('_doc79r').text\n",
        "            except NoSuchElementException:\n",
        "                price = None\n",
        "                \n",
        "            try:\n",
        "                EntireHomeVsRoom = tot.find_element_by_class_name(\"_1xxanas2\").text\n",
        "            except NoSuchElementException:\n",
        "                EntireHomeVsRoom = None\n",
        "                \n",
        "            #beds_bedrooms= tot.find_elements_by_xpath(\"//div[@id='room']//*[contains(text(),'bed')]\")\n",
        "                \n",
        "            try:  \n",
        "                num_bedrooms = tot.find_element_by_xpath(\"//div[@id='room']//*[contains(text(),'bedroom')]\").text\n",
        "            except NoSuchElementException:\n",
        "                num_bedrooms = None\n",
        "                                                    \n",
        "            try:                                          \n",
        "                num_beds = tot.find_element_by_xpath(\"//*[contains(text(),'bed')][not(contains(text(), 'bedroom'))]\").text\n",
        "            except NoSuchElementException:\n",
        "                num_beds = None\n",
        "            \n",
        "            try:\n",
        "                num_baths= tot.find_element_by_xpath(\"//*[@class='_czm8crp' and contains(text(),'bath')]\").text\n",
        "            except NoSuchElementException:\n",
        "                num_baths = None\n",
        "                                                   \n",
        "            try:\n",
        "                sleeping_engagments = tot.find_element_by_xpath(\"//div[@class='_520jgb']\").text\n",
        "            except NoSuchElementException:\n",
        "                sleeping_engagments = None \n",
        "                        \n",
        "            try:\n",
        "                open_amenities_window = tot.find_element_by_xpath(\"//button[@class='_pgfpyhv' and contains(text(),'amenities')]\").send_keys(\"\\n\")\n",
        "                amenities= tot.find_elements_by_xpath(\"//div[@aria-labelledby='dls-modal__AmenitiesModal']//div[@class='_czm8crp']\")\n",
        "                amenities_total=[amenities[i].text for i in range(0,len(amenities))] \n",
        "            except NoSuchElementException:\n",
        "                amenities_total= None\n",
        "            #close_amenities_window = tot.find_element_by_xpath(\"//div[@class='_152qbzi']/button[@class='_1rp5252' and @aria-label='Close']\").send_keys(\"\\n\")    \n",
        "            \n",
        "            time.sleep(3)   \n",
        "            driver.refresh()\n",
        "                                                        \n",
        "            try:\n",
        "                reviews_group = driver.find_elements_by_xpath(\"//*[@class='_1iu38l3']/span\")\n",
        "                Accuracy_stars = reviews_group[0].get_attribute(\"aria-label\")\n",
        "                Communication_stars = reviews_group[1].get_attribute(\"aria-label\")\n",
        "                Cleanliness_stars = reviews_group[2].get_attribute(\"aria-label\")\n",
        "                Location_stars = reviews_group[3].get_attribute(\"aria-label\")\n",
        "                CheckIn_stars = reviews_group[4].get_attribute(\"aria-label\")\n",
        "                Value_stars = reviews_group[5].get_attribute(\"aria-label\")\n",
        "            except:\n",
        "                Accuracy_stars = None\n",
        "                Communication_stars = None\n",
        "                Cleanliness_stars = None\n",
        "                Location_stars = None\n",
        "                CheckIn_stars = None\n",
        "                Value_stars = None\n",
        "                pass \n",
        "                                                                   \n",
        "            try:\n",
        "                review_txt = driver.find_element_by_xpath(\"//div[@id='reviews']\").text\n",
        "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
        "                review_txt = None\n",
        "            \n",
        "            try:\n",
        "                Super_host = driver.find_element_by_xpath(\"//*[@class='_1p3joamp' and contains(text(),'Superhos')]\").text\n",
        "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
        "                Super_host = None\n",
        "                                                      \n",
        "            try:                                         \n",
        "                reviewers_num = tot.find_element_by_xpath(\"//button[@class='_ff6jfq']//span[@class='_so3dpm2']\").text\n",
        "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
        "                reviewers_num = None\n",
        "                                                      \n",
        "\n",
        "            writer.writerow({'Date' : Checkin_Date,\n",
        "                             'Title': title,\n",
        "                             'Price': price,\n",
        "                             'Has_Weekend' : Has_weekend,\n",
        "                             'Latitude' : lat,\n",
        "                             'Longitude' : long,\n",
        "                             'EntireHomeVsRoom' : EntireHomeVsRoom,\n",
        "                             'Bedrooms_no': num_bedrooms,\n",
        "                             'Beds_no': num_beds,\n",
        "                             'Bathrooms_num' : num_baths,\n",
        "                             'Sleeping_engagment':sleeping_engagments,\n",
        "                             'Amenities': amenities_total,\n",
        "                             'Accuracy_stars': Accuracy_stars,\n",
        "                             'Communication_stars': Communication_stars,\n",
        "                             'Cleanliness_stars' : Cleanliness_stars,\n",
        "                             'Location_stars' : Location_stars,\n",
        "                             'CheckIn_stars' : CheckIn_stars,\n",
        "                             'Value_stars' : Value_stars,\n",
        "                             'Reviewers_no': reviewers_num,\n",
        "                             'Super_host': Super_host,\n",
        "                             'Review':review_txt})\n",
        "\n",
        "            time.sleep(10)\n",
        "            driver.close()\n",
        "            \n",
        "        start_date = Checkin_Date        \n",
        "                \n",
        "               # browser.find_element_by_xpath(\"//li[@class='_pgfpyhv']/a\").send_keys(\"\\n\") #send enter for links, buttons\n",
        "            #inputElement.send_keys(\"\\n\") #send enter for links, buttons\n",
        "         # Calculate new scroll height and compare with last scroll height\n",
        "        "
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: DeprecationWarning: use options instead of chrome_options\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}