AirBnb Scraping with Selenuim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "import csv\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#option = webdriver.ChromeOptions()\n",
    "#option.add_argument(\"--incognito\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Note: for final results, you need only to run the steps 1,2, &3 + importing the required packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new instance of chrome \n",
    "browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# go to website of interest\n",
    "browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&s_tag=K-CPRUF7\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#wait up to 10 seconds for page to load\n",
    "timeout = 10\n",
    "#try:\n",
    " #   WebDriverWait(browser, timeout).until(EC.visibility_of_element_located((By.XPATH, \"//div[@class='_ng4pvpo']\")))\n",
    "#except TimeoutException:\n",
    " #   print(\"Timed out waiting for page to load\")\n",
    "  #  browser.quit()\n",
    "encode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get page content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all contents\n",
    "all_element = browser.find_elements_by_xpath(\"//div[@class='_1wcpzyga']\")\n",
    "elements = [x.text for x in all_element]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contents:\n",
      "['Manhattan Lux Loft.Like.Love.Lots.Look !\\nPRIVATE ROOM · 2 BEDS\\nManhattan Lux Loft.Like.Love.Lots.Look !\\nPrice\\n$167 CAD per night\\n · Free cancellation\\n503 reviews\\n503 · Superhost', 'Private Studio Apartment in Harlem\\nENTIRE HOUSE · 1 BED\\nPrivate Studio Apartment in Harlem\\nPrice\\n$92 CAD per night\\n · Free cancellation\\n247 reviews\\n247', 'Soho loft with massive couch for extra sleepers!\\nPRIVATE ROOM · 1 BED\\nSoho loft with massive couch for extra sleepers!\\nPrice\\n$109 CAD per night\\n · Free cancellation\\n121 reviews\\n121', 'Studio Apartment Minutes Away from Times Square\\nENTIRE APARTMENT · 1 BED\\nStudio Apartment Minutes Away from Times Square\\nPrice\\n$53 CAD per night\\n · Free cancellation\\n101 reviews\\n101 · Superhost', 'Penthouse Studio East 50s Terrace\\nENTIRE APARTMENT · 1 BED\\nPenthouse Studio East 50s Terrace\\nPrice\\n$179 CAD per night\\n · Free cancellation\\n290 reviews\\n290', 'Sun-drenched 1BD in Upper East Side\\nPRIVATE ROOM · 1 BED\\nSun-drenched 1BD in Upper East Side\\nPrice\\n$100 CAD per night\\n · Free cancellation\\n248 reviews\\n248', 'Times Square Luxury Place\\nENTIRE APARTMENT · 0 BEDS\\nTimes Square Luxury Place\\nPrice\\n$129 CAD per night\\n · Free cancellation\\n259 reviews\\n259', 'Williamsburg Penthouse Guestroom\\nPRIVATE ROOM · 1 BED\\nWilliamsburg Penthouse Guestroom\\nPrice\\n$62 CAD per night\\n · Free cancellation\\n319 reviews\\n319 · Superhost', 'Times Square Cute Private Studio\\nENTIRE APARTMENT · 1 BED\\nTimes Square Cute Private Studio\\nPrice\\n$133 CAD per night\\n · Free cancellation\\n258 reviews\\n258', 'MASTER SUITE IN THE CENTRAL PARK\\nPRIVATE ROOM · 1 BED\\nMASTER SUITE IN THE CENTRAL PARK\\nPrice\\n$137 CAD per night\\n · Free cancellation\\n110 reviews\\n110', 'Luxury Central Park Apartment close to everything\\nENTIRE APARTMENT · 4 BEDS\\nLuxury Central Park Apartment close to everything\\nPrice\\n$171 CAD per night\\n · Free cancellation\\n114 reviews\\n114', 'CHEAP BIG room in Williamsburg\\nPRIVATE ROOM · 1 BED\\nCHEAP BIG room in Williamsburg\\nPrice\\n$86 CAD per night\\n · Free cancellation\\n215 reviews\\n215', 'Beautiful Private Home -minutes from NYC!\\nENTIRE HOUSE · 4 BEDS\\nBeautiful Private Home -minutes from NYC!\\nPrice\\n$73 CAD per night\\n · Free cancellation\\n195 reviews\\n195 · Superhost', 'Private Bedroom in Manhattan\\nPRIVATE ROOM · 1 BED\\nPrivate Bedroom in Manhattan\\nPrice\\n$65 CAD per night\\n · Free cancellation\\n570 reviews\\n570', '\"The Country\" & the NYC Waterfront!\\nENTIRE GUEST SUITE · 2 BEDS\\n\"The Country\" & the NYC Waterfront!\\nPrice\\n$233 CAD per night\\n · Free cancellation\\n385 reviews\\n385 · Superhost', 'Brooklyn Room Available [furnished]\\nPRIVATE ROOM · 1 BED\\nBrooklyn Room Available [furnished]\\nPrice\\n$53 CAD per night\\n · Free cancellation\\n39 reviews\\n39', 'Tropical Getaway @ The Funky Loft with Breakfast\\nPRIVATE ROOM · 1 BED\\nTropical Getaway @ The Funky Loft with Breakfast\\nPrice\\n$73 CAD per night\\n · Free cancellation\\n140 reviews\\n140 · Superhost', 'Soho 2 Bedroom Private Terrace\\nENTIRE CONDOMINIUM · 3 BEDS\\nSoho 2 Bedroom Private Terrace\\nPrice\\n$428 CAD per night\\n · Free cancellation\\n34 reviews\\n34 · Superhost']\n"
     ]
    }
   ],
   "source": [
    "print('Contents:')\n",
    "print(elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Manhattan Lux Loft.Like.Love.Lots.Look !\n",
      "PRIVATE ROOM · 2 BEDS\n",
      "Manhattan Lux Loft.Like.Love.Lots.Look !\n",
      "Price\n",
      "$167 CAD per night\n",
      " · Free cancellation\n",
      "503 reviews\n",
      "503 · Superhost\n"
     ]
    }
   ],
   "source": [
    "print(elements[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Titles:\n",
      "['Manhattan Lux Loft.Like.Love.Lots.Look !', 'Private Studio Apartment in Harlem', 'Beautiful Private Home -minutes from NYC!', 'Soho loft with massive couch for extra sleepers!', 'Sun-drenched 1BD in Upper East Side', 'Studio Apartment Minutes Away from Times Square', 'Penthouse Studio East 50s Terrace', 'Williamsburg Penthouse Guestroom', 'Spacious Manhattan Apartment near Central Park', 'MASTER SUITE IN THE CENTRAL PARK', 'Luxury Central Park Apartment close to everything', 'Times Square Cute Private Studio', 'Times Square Luxury Place', 'CHEAP BIG room in Williamsburg', 'Private Bedroom in Manhattan', 'Brooklyn Room Available [furnished]', '\"The Country\" & the NYC Waterfront!', 'Brownstone studio near central park']\n"
     ]
    }
   ],
   "source": [
    "# get all of the homes'titles\n",
    "title_element = browser.find_elements_by_xpath(\"//div[@class='_ng4pvpo']\")\n",
    "titles = [x.text for x in title_element]\n",
    "print('Titles:')\n",
    "print(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(title_element)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Manhattan Lux Loft.Like.Love.Lots.Look ! \n",
      "\n",
      "Private Studio Apartment in Harlem \n",
      "\n",
      "Soho loft with massive couch for extra sleepers! \n",
      "\n",
      "Studio Apartment Minutes Away from Times Square \n",
      "\n",
      "Penthouse Studio East 50s Terrace \n",
      "\n",
      "Sun-drenched 1BD in Upper East Side \n",
      "\n",
      "Times Square Luxury Place \n",
      "\n",
      "Williamsburg Penthouse Guestroom \n",
      "\n",
      "Times Square Cute Private Studio \n",
      "\n",
      "MASTER SUITE IN THE CENTRAL PARK \n",
      "\n",
      "Luxury Central Park Apartment close to everything \n",
      "\n",
      "CHEAP BIG room in Williamsburg \n",
      "\n",
      "Beautiful Private Home -minutes from NYC! \n",
      "\n",
      "Private Bedroom in Manhattan \n",
      "\n",
      "\"The Country\" & the NYC Waterfront! \n",
      "\n",
      "Brooklyn Room Available [furnished] \n",
      "\n",
      "Tropical Getaway @ The Funky Loft with Breakfast \n",
      "\n",
      "Soho 2 Bedroom Private Terrace \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for j in title_element:\n",
    "    print(j.text,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get home price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prices:\n",
      "['$166 CAD', '$92 CAD', '$65 CAD', '$109 CAD', '$100 CAD', '$53 CAD', '$179 CAD', '$62 CAD', '$122 CAD', '$137 CAD', '$171 CAD', '$133 CAD', '$129 CAD', '$86 CAD', '$65 CAD', '$53 CAD', '$233 CAD', '$105 CAD']\n"
     ]
    }
   ],
   "source": [
    "# get all of the homes'prices\n",
    "price_element = browser.find_elements_by_xpath(\"//span[@class='_12kw8n71']/span[position()=2]\")\n",
    "home_price = [x.text for x in price_element]  \n",
    "print('Prices:')\n",
    "print(home_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(home_price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get room type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Room type:\n",
      "['PRIVATE ROOM · 2 BEDS', 'ENTIRE HOUSE · 1 BED', 'ENTIRE HOUSE · 4 BEDS', 'PRIVATE ROOM · 1 BED', 'PRIVATE ROOM · 1 BED', 'ENTIRE APARTMENT · 1 BED', 'ENTIRE APARTMENT · 1 BED', 'PRIVATE ROOM · 1 BED', 'ENTIRE APARTMENT · 1 BED', 'PRIVATE ROOM · 1 BED', 'ENTIRE APARTMENT · 4 BEDS', 'ENTIRE APARTMENT · 1 BED', 'ENTIRE APARTMENT · 0 BEDS', 'PRIVATE ROOM · 1 BED', 'PRIVATE ROOM · 1 BED', 'PRIVATE ROOM · 1 BED', 'ENTIRE GUEST SUITE · 2 BEDS', 'ENTIRE APARTMENT · 1 BED']\n"
     ]
    }
   ],
   "source": [
    "# get all of the rooms'type\n",
    "room_element = browser.find_elements_by_xpath(\"//span[@style='color: rgb(118, 118, 118);']\")\n",
    "room_type = [x.text for x in room_element]  \n",
    "print('Room type:')\n",
    "print(room_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "room=[]\n",
    "num_beds= []\n",
    "for i in range(0,len(room_type)):\n",
    "        room.append(room_type[i].split('·')[0])\n",
    "        num_beds.append(room_type[i].split('·')[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(room)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['PRIVATE ROOM ', 'ENTIRE HOUSE ', 'ENTIRE HOUSE ', 'PRIVATE ROOM ', 'PRIVATE ROOM ', 'ENTIRE APARTMENT ', 'ENTIRE APARTMENT ', 'PRIVATE ROOM ', 'ENTIRE APARTMENT ', 'PRIVATE ROOM ', 'ENTIRE APARTMENT ', 'ENTIRE APARTMENT ', 'ENTIRE APARTMENT ', 'PRIVATE ROOM ', 'PRIVATE ROOM ', 'PRIVATE ROOM ', 'ENTIRE GUEST SUITE ', 'ENTIRE APARTMENT ']\n"
     ]
    }
   ],
   "source": [
    "print(room)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get tne number of beds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(num_beds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' 2 BEDS',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 0 BEDS',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 4 BEDS',\n",
       " ' 1 BED',\n",
       " ' 4 BEDS',\n",
       " ' 1 BED',\n",
       " ' 2 BEDS',\n",
       " ' 1 BED',\n",
       " ' 1 BED',\n",
       " ' 3 BEDS']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_beds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the cancellation policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cancellation Policy:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation',\n",
       " 'Free cancellation']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get number of beds \n",
    "cancel_element = browser.find_elements_by_xpath(\"//div[@class='_1cm6f7d']/span[position()=2]\")\n",
    "cancel = [x.text for x in cancel_element]  \n",
    "print('Cancellation Policy:')\n",
    "cancel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the number of reviewers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of Reviews:\n",
      "['', '', '503', '247', '195', '121', '248', '101', '290', '319', '268', '110', '114', '258', '259', '215', '570', '39', '385', '106']\n"
     ]
    }
   ],
   "source": [
    "# get the reviews part    \n",
    "review_element = browser.find_elements_by_xpath(\"//div[@class='_36rlri']\")\n",
    "num_reviews = [x.text.split(\" \")[0] for x in review_element]  \n",
    "print('Num of Reviews:')\n",
    "print(num_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(num_reviews)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the review rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the reviews part    \n",
    "reviewStars_element = browser.find_elements_by_xpath(\"//span[@class='_q27mtmr']\")\n",
    "review_stars = [x for x in reviewStars_element]  \n",
    "#print('Review rate:')\n",
    "#review_stars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "stars=[]\n",
    "for i in review_stars:\n",
    "    stars.append([i.find_element_by_css_selector('span').get_attribute(\"aria-label\")])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Rating 5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4 out of 5'],\n",
       " ['Rating 5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 5 out of 5'],\n",
       " ['Rating 5 out of 5'],\n",
       " ['Rating 5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5'],\n",
       " ['Rating 4.5 out of 5']]"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_element = browser.find_elements_by_xpath(\"//div[@class='_1wcpzyga']\")\n",
    "#for tst in all_element:\n",
    " #   print(tst,'\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#for elem in all_element:\n",
    "  #  print(elem.find_element_by_xpath(\"//div[@class='_ng4pvpo']\").text)\n",
    "  #  print(elem.find_element_by_xpath(\"//span[@class='_12kw8n71']/span[position()=2]\").text)\n",
    "  #  print(elem.find_element_by_xpath(\"//span[@style='color: rgb(118, 118, 118);']\").text.split('·')[0])\n",
    "  #  print(elem.find_element_by_xpath(\"//span[@style='color: rgb(118, 118, 118);']\").text.split('·')[1])\n",
    "  #  print(elem.find_element_by_xpath(\"//div[@class='_1cm6f7d']/span[position()=2]\").text)\n",
    "  #  print(elem.find_element_by_xpath(\"//span[@class='_q27mtmr']\").find_element_by_css_selector('span').get_attribute(\"aria-label\"))\n",
    "  #  print(elem.find_element_by_xpath(\"//div[@class='_36rlri']\").text.split(\" \")[0])\n",
    "  #  print('\\n')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont = browser.find_element_by_class_name('_fhph4u')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<selenium.webdriver.remote.webelement.WebElement (session=\"6710e294d6d92e41c3bc1844867cabb4\", element=\"0.30637284451947155-147\")>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "prods= cont.find_elements_by_class_name('_1wcpzyga')\n",
    "review_stars = browser.find_elements_by_xpath(\"//span[@class='_q27mtmr']\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('AirBnb.csv', 'w',encoding=\"utf-8\", newline='') as csvfile:\n",
    "    fieldnames = ['Title','Price','Room_type','num_of_beds','Cancel_policy','Reviewrs', 'Review_stars']\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    for page_num in range(1,18):\n",
    "        cont = browser.find_element_by_class_name('_fhph4u')\n",
    "        prods= cont.find_elements_by_class_name('_1wcpzyga')\n",
    "        review_stars = browser.find_elements_by_xpath(\"//span[@class='_q27mtmr']\")\n",
    "        for prod, i in zip(prods,review_stars):\n",
    "            title = prod.find_element_by_class_name('_ng4pvpo').text\n",
    "            price = prod.find_element_by_class_name('_12kw8n71').text\n",
    "            room_type = prod.find_element_by_class_name('_1bb2ucx1').text.split('·')[0]\n",
    "            num_of_beds = prod.find_element_by_class_name('_1bb2ucx1').text.split('·')[1]\n",
    "            cancel_policy = prod.find_element_by_class_name('_1cm6f7d').text\n",
    "            reviews_rate = i.find_element_by_css_selector('span').get_attribute(\"aria-label\")\n",
    "            num_of_reviews = prod.find_element_by_class_name('_7g6kz31').text\n",
    "            writer.writerow({'Title': title,\n",
    "                             'Price': price,\n",
    "                             'Room_type': room_type,\n",
    "                             'num_of_beds': num_of_beds,\n",
    "                             'Cancel_policy': cancel_policy,\n",
    "                             'Reviewrs': num_of_reviews,\n",
    "                             'Review_stars':reviews_rate})\n",
    "            \n",
    "        nxt_page =  browser.find_element_by_class_name('_b8vexar')\n",
    "        nxt_page.click()\n",
    "        time.sleep(10)\n",
    "        \n",
    "   \n",
    "           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}