airbnb scraping w Datefilter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.common.exceptions import NoSuchElementException \n",
    "from selenium.common.exceptions import StaleElementReferenceException\n",
    "from selenium.common.exceptions import WebDriverException\n",
    "from selenium.webdriver.support.ui import WebDriverWait as wait\n",
    "from selenium.webdriver.common.keys import Keys #need to send keystrokes\n",
    "from datetime import datetime \n",
    "from datetime import timedelta\n",
    "from dateutil import parser\n",
    "import urllib\n",
    "from urllib import parse\n",
    "import csv\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new instance of chrome \n",
    "browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# go to website of interest\n",
    "\n",
    "Checkin_Date='2019-03-01'\n",
    "Checkout_Date='2019-03-06'\n",
    "#browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date+\"&checkout=\"+Checkout_Date+\"&map_toggle=false\")\n",
    "\n",
    "\n",
    "browser.get(\"https://www.airbnb.ca/rooms/16194184?check_in=2019-03-01&check_out=2019-03-06&guests=1&adults=1&s=7uhz4tad\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Latitude & Longitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40.72872 -73.99798\n"
     ]
    }
   ],
   "source": [
    "map_url = browser.find_element_by_xpath(\"//div[@class='_59m2yxn']/img\").get_attribute(\"src\")\n",
    "query = parse.parse_qs(parse.urlparse(map_url).query)['center'][0]\n",
    "lat = query.split(\",\")[0]\n",
    "long = query.split(\",\")[1]\n",
    "print(lat, long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Duplex with Private Bathroom by Times Square'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title= browser.find_element_by_class_name('_18hrqvin').text\n",
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bedroom 1\\n1 queen bed, 1 sofa bed'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//div[@class='_520jgb']\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2 beds'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//*[contains(text(),'bed')][not(contains(text(), 'bedroom'))]\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## number of beds\n",
    "beds_bedrooms=browser.find_elements_by_xpath(\"//*[@class='_36rlri']/*[@class='_czm8crp']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-4\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-5\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-6\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-7\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-8\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"8e900a4a4a5289ae4185717c93387d4e\", element=\"0.6973769992582637-9\")>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beds_bedrooms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2 guests'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beds_bedrooms[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1 bedroom'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beds_bedrooms[1].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Wifi',\n",
       " 'Washer',\n",
       " 'Dryer',\n",
       " 'Iron',\n",
       " 'Laptop friendly workspace',\n",
       " 'TV',\n",
       " 'Essentials',\n",
       " 'Heating',\n",
       " 'Air conditioning',\n",
       " 'Hot water',\n",
       " 'Paid parking off premises',\n",
       " 'Free street parking',\n",
       " 'Kitchen',\n",
       " 'Coffee maker',\n",
       " 'Dishes and silverware',\n",
       " 'Dishwasher',\n",
       " 'Refrigerator',\n",
       " 'Oven',\n",
       " 'Stove',\n",
       " 'Host greets you',\n",
       " 'Private entrance',\n",
       " 'Long term stays allowed',\n",
       " 'Hangers',\n",
       " 'Hair dryer',\n",
       " 'Shampoo',\n",
       " 'Bed linens',\n",
       " 'Extra pillows and blankets',\n",
       " 'Fire extinguisher',\n",
       " 'Carbon monoxide detector',\n",
       " 'Smoke detector']"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "open_amenities_window = browser.find_element_by_xpath(\"//button[@class='_pgfpyhv' and contains(text(),'amenities')]\").send_keys(\"\\n\")\n",
    "amenities= browser.find_elements_by_xpath(\"//div[@aria-labelledby='dls-modal__AmenitiesModal']//div[@class='_czm8crp']\")\n",
    "amenities_total=[amenities[i].text for i in range(0,len(amenities))] \n",
    "#close_amenities_window = browser.find_element_by_xpath(\"//div[@class='_152qbzi']/button[@class='_1rp5252']\").send_keys(\"\\n\")\n",
    "amenities_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "close_amenities_window = browser.find_element_by_xpath(\"//div[@class='_152qbzi']/button[@class='_1rp5252']\").send_keys(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_group=browser.find_elements_by_xpath(\"//*[@class='_1iu38l3']/span\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Rating 4.5 out of 5',\n",
       " 'Rating 4.5 out of 5',\n",
       " 'Rating 4.5 out of 5',\n",
       " 'Rating 5 out of 5',\n",
       " 'Rating 4.5 out of 5',\n",
       " 'Rating 4 out of 5']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[reviews_group[i].get_attribute(\"aria-label\") for i in range(0,6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"89 Reviews\\nAccuracy\\nCommunication\\nCleanliness\\nLocation\\nCheck-in\\nValue\\nDanielle\\nDecember 2018\\nThis place was in a perfect neighbourhood. Close to subway stations and lots of bars and restaurants. For being so close to the action, it was nice and quiet at night. I would recommend Tellas place to anyone coming to NYC and will definitely be back.\\nCarol\\nJuly 2018\\nThis is a cozy but very comfortable apartment in a super fun location. I’d stay again. However there were problems with communicating check and other details. But the host was available and responsive.\\nJosh\\nMarch 2018\\nI didn't end up staying at the apartment because of a storm that cancelled my flight. However Tella was very accommodating and helpful to do all that she could to help with the cancellation. I hope to be able to stay there in the future!\\nBradley\\nAugust 2017\\nExcellent location! Great set up to stay. Very clean. Shower worked on and off though.\\nMathieu\\nFebruary 2019\\nThe flat is very well located and Tella has been a very helpful host, but I was very surprised by the heat inside the flat (some warm water pipes running throughout probably?) which was impossible to compensate even with the A/C on full blast. The flat was also very noisy because…Read more\\nLesley\\nJanuary 2019\\nThe location is unbeatable. Perfect for a weekend stay.\\nJacob\\nJanuary 2019\\nPerfect spot downtown to stay, host responded with any questions- everything was great.\\n1\\n2\\n3\\n…\\n13\""
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//div[@id='reviews']\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40.72872 -73.99798\n"
     ]
    }
   ],
   "source": [
    "map_url = browser.find_element_by_xpath(\"//*[@class='_59m2yxn']/img\").get_attribute(\"src\")\n",
    "query = parse.parse_qs(parse.urlparse(map_url).query)['center'][0]\n",
    "lat = query.split(\",\")[0]\n",
    "long = query.split(\",\")[1]\n",
    "print(lat, long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Indoor fireplace', 'Kitchen', 'Wifi', 'Washer']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amenities_total=[]\n",
    "amenities= browser.find_elements_by_xpath(\"//td[@class='_4xosax']/div\")\n",
    "for i in range(0,len(amenities)):\n",
    "    amenities_total.append(amenities[i].text)\n",
    "\n",
    "amenities_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Indoor fireplace',\n",
       " 'Kitchen',\n",
       " 'Wifi',\n",
       " 'Washer',\n",
       " 'Indoor fireplace',\n",
       " 'Wifi',\n",
       " 'Washer',\n",
       " 'Dryer',\n",
       " 'Iron',\n",
       " 'Laptop friendly workspace',\n",
       " 'Essentials',\n",
       " 'Heating',\n",
       " 'Air conditioning',\n",
       " 'Hot water',\n",
       " 'Pocket wifi',\n",
       " 'Bathtub',\n",
       " 'Room-darkening shades',\n",
       " 'Paid parking off premises',\n",
       " 'Kitchen',\n",
       " 'Smart lock',\n",
       " 'Luggage dropoff allowed',\n",
       " 'Long term stays allowed',\n",
       " 'Hangers',\n",
       " 'Hair dryer',\n",
       " 'Shampoo',\n",
       " 'Bed linens',\n",
       " 'Extra pillows and blankets',\n",
       " 'Lock on bedroom door',\n",
       " 'Fire extinguisher',\n",
       " 'Carbon monoxide detector',\n",
       " 'Smoke detector',\n",
       " 'First aid kit']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//button[@class='_pgfpyhv' and contains(text(),'amenities')]\").send_keys(\"\\n\")\n",
    "amenities= browser.find_elements_by_xpath(\"//div[@aria-labelledby='dls-modal__AmenitiesModal']//div[@class='_czm8crp']\")\n",
    "for i in range(0,len(amenities)):\n",
    "    amenities_total.append(amenities[i].text)\n",
    "\n",
    "amenities_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.find_element_by_xpath(\"//div[@class='_152qbzi']/button[@class='_1rp5252']\").send_keys(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//div[@id='reviews']\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Michael is a Superhost'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser.find_element_by_xpath(\"//div[@class='_1p3joamp' and contains(text(),'Superhost')]\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['uptown',\n",
       " 'roomy',\n",
       " 'young',\n",
       " 'settled',\n",
       " 'professionals',\n",
       " 'hipster',\n",
       " \"There are countless numbers of Restaurants,Shops, Grocery store, Beauty salons, Barber shop atc, Just right around the corner, Tha's the beauty of living in mantahhatn off the very famous Broadway ave.\"]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neighborhood_feats=browser.find_elements_by_xpath(\"//div[@id ='neighborhood']//span[@class='_czm8crp']\")\n",
    "[neighborhood_feats[i].text for i in range(0,len(neighborhood_feats))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'137'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#browser.find_element_by_xpath(\"//button[@class='_ff6jfq']\").get_attribute(\"aria-label\")\n",
    "browser.find_element_by_xpath(\"//button[@class='_ff6jfq']//span[@class='_so3dpm2']\").text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40.7658 -73.98795\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## longitude and latitude\n",
    "url=browser.find_element_by_xpath(\"//div[@class='_59m2yxn']/img\").get_attribute(\"src\")\n",
    "query_def=parse.parse_qs(parse.urlparse(url).query)['center'][0]\n",
    "query_def\n",
    "lat = query_def.split(\",\")[0]\n",
    "long = query_def.split(\",\")[1]\n",
    "print(lat, long)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import googlemaps\n",
    "#from datetime import datetime\n",
    "\n",
    "\n",
    "\n",
    "# Geocoding an address\n",
    "#geocode_result = gmaps.geocode('https://maps.googleapis.com/maps/api/staticmap?size=648x348&zoom=13&center=40.76037,-74.00664&scale=2&path=color%3A0x007A87%7Cfillcolor%3A0x00d1c166%7Cweight%3A1%7Cenc%3Ak%7DywFtwqbMLmEf%40gEbA%7BDxAiDlBqC~BuBlCwAvCu%40zCQzCPvCt%40lCvA~BtBlBpCxAhDbAzDf%40fELlEMlEg%40fEcAzDyAhDmBpC_CtBmCvAwCt%40%7BCP%7BCQwCu%40mCwA_CuBmBqCyAiDcA%7BDg%40gEMmE%3F%3F&path=color%3A0xFFAA91%7Cfillcolor%3A0xFFAA9133%7Cweight%3A2%7Cenc%3AokwwFdeubMQODq%40oA%7B%40e%40TSv%40iAq%40iDlKqCiBhDmKuBaAc%40Ak%40~Ak%40g%40t%40iBoEwCV%7D%40wByAcEtL%5Dc%40vDwLoD%7BB%7BDtLg%40YtDuLkBwAuDrL%7D%40y%40rFeQ%5D_AaCgBmGvRaAw%40xFqRoAqAWd%40U%3Fg%40_A%7DGnScBgAnGgScDaC%7BFdR%5Bj%40w%40s%40pCwINs%40fBsF%7BDiCiGbSM%3Fu%40q%40nEeNeCeByEfNcAw%40nEwMuFsDa%40v%40i%40n%40h%40eCi%40OKo%40SBUBoDvKs%40m%40pDaLyAi%40uClIYSzC_JKQeAs%40mDbM%7B%40g%40%7Ca%40ssAROXXzAg%40Xq%40zIrFluAb~%40__%40njA%3F%3F&client=gme-airbnbinc&channel=monorail-prod&signature=o6GB_jO0CbJiv5HcJtL2EucQKhY%3D')\n",
    "#geocode_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = browser.find_element_by_xpath(\"//li[@class='_b8vexar']\")\n",
    "browser.execute_script(\"arguments[0].click();\", element)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#browser.find_element_by_xpath(\"//div[@id='reviews']\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# go to website of interest\n",
    "Checkin_Date='2019-02-23'\n",
    "Checkout_Date='2019-02-24'\n",
    "browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date+\"&checkout=\"+Checkout_Date+\"&map_toggle=false\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.find_element_by_xpath(\"//li[@class='_b8vexar']\").click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = '2019-02-18'\n",
    "start_date = parser.parse(start_date)\n",
    "for i in range(5): \n",
    "    Checkin_Date = start_date+timedelta(days=1)\n",
    "    Checkout_Date = start_date+timedelta(days=2)\n",
    "    print(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+dt1.strftime('%Y-%m-%d')+\"&checkout=\"+dt2.strftime('%Y-%m-%d'))\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prices:\n",
      "['$99 CAD', '$132 CAD', '$149 CAD', '$99 CAD', '$86 CAD', '$99 CAD', '$60 CAD', '$43 CAD', '$130 CAD', '$65 CAD', '$118 CAD', '$123 CAD', '$68 CAD', '$78 CAD', '$71 CAD', '$58 CAD', '$73 CAD', '$50 CAD']\n"
     ]
    }
   ],
   "source": [
    "# get all of the homes'prices\n",
    "price_element = browser.find_elements_by_xpath(\"//span[@class='_1jlnvra2']/span[position()=2]\")\n",
    "home_price = [x.text for x in price_element]  \n",
    "print('Prices:')\n",
    "print(home_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot = browser.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
    "prods = tot.find_elements_by_xpath(\"//meta[@itemprop='url']\")\n",
    "links = [prod.get_attribute(\"content\") for prod in prods]\n",
    "for l in links:\n",
    "    browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')\n",
    "    browser.get(\"https://\"+l)all\n",
    "    browser.close\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Room type:\n",
      "['PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN BED AND BREAKFAST', 'PRIVATE ROOM IN APARTMENT', 'SHARED ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN GUESTHOUSE', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN TOWNHOUSE', 'ENTIRE APARTMENT', 'SHARED ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT', 'SHARED ROOM IN APARTMENT', 'PRIVATE ROOM IN APARTMENT']\n"
     ]
    }
   ],
   "source": [
    "# get all of the rooms'type\n",
    "room_element = browser.find_elements_by_xpath(\"//span[@style='color: rgb(118, 118, 118);']\")\n",
    "room_type = [x.text for x in room_element]  \n",
    "print('Room type:')\n",
    "print(room_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Hair dryer · Shampoo',\n",
       " 'Wifi · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " '',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo',\n",
       " 'Wifi · Hair dryer · Shampoo',\n",
       " 'Wifi · Hair dryer · Shampoo',\n",
       " 'Wifi · Kitchen · Hair dryer · Shampoo']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "additions = browser.find_elements_by_xpath(\"//div[@style='margin-top: 2px;']\")\n",
    "add = [j.text for j in additions]\n",
    "add\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['8',\n",
       " '121',\n",
       " '4',\n",
       " '177',\n",
       " '130',\n",
       " '43',\n",
       " '41',\n",
       " '503',\n",
       " '108',\n",
       " '213',\n",
       " '3',\n",
       " '40',\n",
       " '61',\n",
       " '65',\n",
       " '113',\n",
       " '10',\n",
       " '165']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tst=browser.find_elements_by_xpath(\"//span[@class='_1gvnvab']\")\n",
    "s= [j.text for j in tst]\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = '2019-02-18'\n",
    "start_date = parser.parse(start_date)\n",
    "for i in range(5): \n",
    "    Checkin_Date = start_date+timedelta(days=1)\n",
    "    Checkout_Date = start_date+timedelta(days=2)\n",
    "    print(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+dt1.strftime('%Y-%m-%d')+\"&checkout=\"+dt2.strftime('%Y-%m-%d'))\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')\n",
    "start_date = '2019-02-25'\n",
    "start_date = parser.parse(start_date)\n",
    "Checkin_Date = start_date+timedelta(days=1)# check_in =19+1=20\n",
    "Checkout_Date = start_date+timedelta(days=2)# check_out = 19+2=21\n",
    "browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date.strftime('%Y-%m-%d')+\"&checkout=\"+Checkout_Date.strftime('%Y-%m-%d'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "import datetime\n",
    "start_date = parser.parse('2019-02-10')\n",
    "Checkin_Date = start_date+timedelta(days=0)# check_in =19+1=20\n",
    "Checkout_Date = start_date+timedelta(days=5)# check_out = 19+2=21\n",
    "\n",
    "\n",
    "#start_date = parser.parse(start_date)\n",
    "#weekno = start_date.weekday()\n",
    "#weekno = datetime.datetime.today().weekday()\n",
    "\n",
    "for i in range(0,5):\n",
    "    day_no = Checkin_Date+timedelta(days=i)\n",
    "    if(day_no.weekday()< 5):\n",
    "        has_weekend = False\n",
    "    else:\n",
    "        has_weekend = True\n",
    "        break\n",
    "        \n",
    "print(has_weekend)\n",
    "        \n",
    "    \n",
    "    \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n",
      "https://www.airbnb.ca/rooms/14807332?check_in=2019-02-26&check_out=2019-02-27\n"
     ]
    }
   ],
   "source": [
    "cont = browser.find_element_by_class_name(\"_fhph4u\")\n",
    "\n",
    "prods = browser.find_elements_by_class_name(\"_8ssblpx\")\n",
    "\n",
    "for prod in prods:\n",
    "    elem = prod.find_element_by_xpath(\"//div[@class='_gig1e7']/div/meta[@itemprop='url']\")\n",
    "    l = elem.get_attribute(\"content\")\n",
    "    print(\"https://\"+l)\n",
    "    #print(prod.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new instance of chrome \n",
    "browser = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')\n",
    "\n",
    "start_date = '2019-02-28'\n",
    "start_date = parser.parse(start_date)\n",
    "\n",
    "SCROLL_PAUSE_TIME = 0.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "ename": "TimeoutException",
     "evalue": "Message: \n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutException\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-59-83b874fca7b3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m             \u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muntil\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mEC\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpresence_of_element_located\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXPATH\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"//div[@class='_59m2yxn']/img\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m             \u001b[0mmap_url\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"//div[@class='_59m2yxn']/img\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"src\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m             \u001b[0mquery\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse_qs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0murlparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmap_url\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'center'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\wait.py\u001b[0m in \u001b[0;36muntil\u001b[1;34m(self, method, message)\u001b[0m\n\u001b[0;32m     78\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mend_time\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 80\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0muntil_not\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTimeoutException\u001b[0m: Message: \n"
     ]
    }
   ],
   "source": [
    "with open('AirBnbOneDay.csv','w',encoding=\"utf-8\",newline='') as csvfile:\n",
    "    fieldnames = ['Date','Has_Weekend','Title','Price','Super_host','Latitude','Longitude','EntireHomeVsRoom','Bedrooms_no','Beds_no','Bathrooms_num','Sleeping_engagment','Amenities','Accuracy_stars','Communication_stars','Cleanliness_stars','Location_stars','CheckIn_stars','Value_stars','Reviewers_no','Review']\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    # how many days to loop through\n",
    "    for h in range(1,4):\n",
    "        Checkin_Date = start_date+timedelta(days=1)# check_in =19+1=20\n",
    "        Checkout_Date = start_date+timedelta(days=6)# check_out = 19+2=21\n",
    "        browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date.strftime('%Y-%m-%d')+\"&checkout=\"+Checkout_Date.strftime('%Y-%m-%d')+\"&map_toggle=false\")\n",
    "        browser.maximize_window()\n",
    "        last_height = browser.execute_script(\"return document.body.scrollHeight\")# Get scroll height\n",
    "        then = time.time()\n",
    "        while True:\n",
    "            browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")# Scroll down to bottom\n",
    "            time.sleep(SCROLL_PAUSE_TIME) # Wait to load page\n",
    "            new_height = browser.execute_script(\"return document.body.scrollHeight\")# Calculate new scroll height and compare with last scroll height\n",
    "            if new_height == last_height or time.time()-then > 60:\n",
    "                break\n",
    "        last_height = new_height\n",
    "        cont = browser.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
    "        prods = cont.find_elements_by_xpath(\"//meta[@itemprop='url']\")\n",
    "        links = [prod.get_attribute(\"content\") for prod in prods]  \n",
    "        for l in links: # 18 or 30 based on the num of the homes in the pag\n",
    "            driver = webdriver.Chrome(executable_path='C:/webdriver/chromedriver')\n",
    "            driver.get(\"https://\"+l)\n",
    "            driver.maximize_window()\n",
    "            tot = driver.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
    "\n",
    "            try:\n",
    "                title = tot.find_element_by_class_name('_18hrqvin').text\n",
    "            except NoSuchElementException:\n",
    "                title = None\n",
    "                \n",
    "            #try:\n",
    "           \n",
    "\n",
    "            wait(driver,5).until(EC.presence_of_element_located((By.XPATH,\"//div[@class='_59m2yxn']/img\")))\n",
    "            map_url = driver.find_element_by_xpath(\"//div[@class='_59m2yxn']/img\").get_attribute(\"src\")\n",
    "            query = parse.parse_qs(parse.urlparse(map_url).query)['center'][0]\n",
    "            lat = query.split(\",\")[0]\n",
    "            long = query.split(\",\")[1]\n",
    "            #except NoSuchElementException:\n",
    "            #lat = None\n",
    "            #long = None\n",
    "                \n",
    "            try:\n",
    "                for i in range(0,5):\n",
    "                    day_no = Checkin_Date+timedelta(days=i)\n",
    "                    if(day_no.weekday()< 5):\n",
    "                        has_weekend = False\n",
    "                    else:\n",
    "                        has_weekend = True\n",
    "                        break\n",
    "                Has_weekend = has_weekend\n",
    "            except NoSuchElementException:\n",
    "                Has_weekend = None    \n",
    "\n",
    "            try:\n",
    "                price = tot.find_element_by_class_name('_doc79r').text\n",
    "            except NoSuchElementException:\n",
    "                price = None\n",
    "                \n",
    "            try:\n",
    "                EntireHomeVsRoom = tot.find_element_by_class_name(\"_1xxanas2\").text\n",
    "            except NoSuchElementException:\n",
    "                EntireHomeVsRoom = None\n",
    "                \n",
    "            #beds_bedrooms= tot.find_elements_by_xpath(\"//div[@id='room']//*[contains(text(),'bed')]\")\n",
    "                \n",
    "            try:  \n",
    "                num_bedrooms = tot.find_element_by_xpath(\"//div[@id='room']//*[contains(text(),'bedroom')]\").text\n",
    "            except NoSuchElementException:\n",
    "                num_bedrooms = None\n",
    "                                                    \n",
    "            try:                                          \n",
    "                num_beds = tot.find_element_by_xpath(\"//*[contains(text(),'bed')][not(contains(text(), 'bedroom'))]\").text\n",
    "            except NoSuchElementException:\n",
    "                num_beds = None\n",
    "            \n",
    "            try:\n",
    "                num_baths= tot.find_element_by_xpath(\"//*[@class='_czm8crp' and contains(text(),'bath')]\").text\n",
    "            except NoSuchElementException:\n",
    "                num_baths = None\n",
    "                                                   \n",
    "            try:\n",
    "                sleeping_engagments = tot.find_element_by_xpath(\"//div[@class='_520jgb']\").text\n",
    "            except NoSuchElementException:\n",
    "                sleeping_engagments = None \n",
    "                        \n",
    "            try:\n",
    "                open_amenities_window = tot.find_element_by_xpath(\"//button[@class='_pgfpyhv' and contains(text(),'amenities')]\").send_keys(\"\\n\")\n",
    "                amenities= tot.find_elements_by_xpath(\"//div[@aria-labelledby='dls-modal__AmenitiesModal']//div[@class='_czm8crp']\")\n",
    "                amenities_total=[amenities[i].text for i in range(0,len(amenities))] \n",
    "            except NoSuchElementException:\n",
    "                amenities_total= None\n",
    "            #close_amenities_window = tot.find_element_by_xpath(\"//div[@class='_152qbzi']/button[@class='_1rp5252' and @aria-label='Close']\").send_keys(\"\\n\")    \n",
    "            \n",
    "            time.sleep(3)   \n",
    "            driver.refresh()                                              \n",
    "                                                        \n",
    "            try:\n",
    "                reviews_group = driver.find_elements_by_xpath(\"//*[@class='_1iu38l3']/span\")\n",
    "                Accuracy_stars = reviews_group[0].get_attribute(\"aria-label\")\n",
    "                Communication_stars = reviews_group[1].get_attribute(\"aria-label\")\n",
    "                Cleanliness_stars = reviews_group[2].get_attribute(\"aria-label\")\n",
    "                Location_stars = reviews_group[3].get_attribute(\"aria-label\")\n",
    "                CheckIn_stars = reviews_group[4].get_attribute(\"aria-label\")\n",
    "                Value_stars = reviews_group[5].get_attribute(\"aria-label\")\n",
    "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
    "                Accuracy_stars = None\n",
    "                Communication_stars = None\n",
    "                Cleanliness_stars = None\n",
    "                Location_stars = None\n",
    "                CheckIn_stars = None\n",
    "                Value_stars = None\n",
    "                                                                   \n",
    "            try:\n",
    "                review_txt = driver.find_element_by_xpath(\"//div[@id='reviews']\").text\n",
    "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
    "                review_txt = None\n",
    "            \n",
    "            try:\n",
    "                Super_host = driver.find_element_by_xpath(\"//*[@class='_1p3joamp' and contains(text(),'Superhos')]\").text\n",
    "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
    "                Super_host = None\n",
    "                                                      \n",
    "            try:                                         \n",
    "                reviewers_num = tot.find_element_by_xpath(\"//button[@class='_ff6jfq']//span[@class='_so3dpm2']\").text\n",
    "            except (NoSuchElementException,StaleElementReferenceException) as e:\n",
    "                reviewers_num = None\n",
    "                                                      \n",
    "\n",
    "            writer.writerow({'Date' : Checkin_Date,\n",
    "                             'Title': title,\n",
    "                             'Price': price,\n",
    "                             'Has_Weekend' : Has_weekend,\n",
    "                             'Latitude' : lat,\n",
    "                             'Longitude' : long,\n",
    "                             'EntireHomeVsRoom' : EntireHomeVsRoom,\n",
    "                             'Bedrooms_no': num_bedrooms,\n",
    "                             'Beds_no': num_beds,\n",
    "                             'Bathrooms_num' : num_baths,\n",
    "                             'Sleeping_engagment':sleeping_engagments,\n",
    "                             'Amenities': amenities_total,\n",
    "                             'Accuracy_stars': Accuracy_stars,\n",
    "                             'Communication_stars': Communication_stars,\n",
    "                             'Cleanliness_stars' : Cleanliness_stars,\n",
    "                             'Location_stars' : Location_stars,\n",
    "                             'CheckIn_stars' : CheckIn_stars,\n",
    "                             'Value_stars' : Value_stars,\n",
    "                             'Reviewers_no': reviewers_num,\n",
    "                             'Super_host': Super_host,\n",
    "                             'Review':review_txt})\n",
    "\n",
    "            time.sleep(15)\n",
    "            driver.close()\n",
    "            \n",
    "        start_date = Checkin_Date        \n",
    "                \n",
    "               # browser.find_element_by_xpath(\"//li[@class='_pgfpyhv']/a\").send_keys(\"\\n\") #send enter for links, buttons\n",
    "            #inputElement.send_keys(\"\\n\") #send enter for links, buttons\n",
    "         # Calculate new scroll height and compare with last scroll height\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "236.37601161003113"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time()-then"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06583333333333333"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "237/3600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//li[@class='_b8vexar']/a\"}\n  (Session info: chrome=72.0.3626.109)\n  (Driver info: chromedriver=71.0.3578.137 (86ee722808adfe9e3c92e6e8ea746ade08423c7e),platform=Windows NT 10.0.16299 x86_64)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-f40b7ec119ad>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     27\u001b[0m                                  \u001b[1;34m'Reviewrs'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mnum_of_reviews\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m                                  'Review_stars':reviews_rate})\n\u001b[1;32m---> 29\u001b[1;33m             \u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"//li[@class='_b8vexar']/a\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     30\u001b[0m             \u001b[1;31m#nxt_page.click()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m             \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element_by_xpath\u001b[1;34m(self, xpath)\u001b[0m\n\u001b[0;32m    392\u001b[0m             \u001b[0melement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'//div/td[1]'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    393\u001b[0m         \"\"\"\n\u001b[1;32m--> 394\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXPATH\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mxpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    395\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    396\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    976\u001b[0m         return self.execute(Command.FIND_ELEMENT, {\n\u001b[0;32m    977\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 978\u001b[1;33m             'value': value})['value']\n\u001b[0m\u001b[0;32m    979\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    980\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//li[@class='_b8vexar']/a\"}\n  (Session info: chrome=72.0.3626.109)\n  (Driver info: chromedriver=71.0.3578.137 (86ee722808adfe9e3c92e6e8ea746ade08423c7e),platform=Windows NT 10.0.16299 x86_64)\n"
     ]
    }
   ],
   "source": [
    "with open('AirBnbOneDay.csv','w',encoding=\"utf-8\",newline='') as csvfile:\n",
    "    fieldnames = ['Date','Title','Price','Room_type','num_of_beds','Additions','Reviewrs','Review_stars']\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    for h in range(1,30): \n",
    "        Checkin_Date = start_date+timedelta(days=1)\n",
    "        Checkout_Date = start_date+timedelta(days=2)\n",
    "        browser.get(\"https://www.airbnb.ca/s/homes?refinement_paths%5B%5D=%2Fhomes&query=New%20York&allow_override%5B%5D=&checkin=\"+Checkin_Date.strftime('%Y-%m-%d')+\"&checkout=\"+Checkout_Date.strftime('%Y-%m-%d'))\n",
    "        for page_num in range(1,18):\n",
    "            cont = browser.find_element_by_xpath(\"//body[@class='with-new-header has-epcot-header']\")\n",
    "            prods= cont.find_elements_by_class_name('_8ssblpx')\n",
    "            review_stars = browser.find_elements_by_xpath(\"//span[@class='_q27mtmr']\")\n",
    "            for prod, i in zip(prods,review_stars):\n",
    "                title = prod.find_element_by_class_name('_1424jywo').text\n",
    "                price = prod.find_element_by_class_name('_121z06r2').text\n",
    "                room_type = prod.find_element_by_class_name('_1xxanas2').text\n",
    "                num_of_beds = prod.find_element_by_class_name('_1jlnvra2').text\n",
    "                additions = prod.find_element_by_xpath(\"//div[@style='margin-top: 2px;']\").text\n",
    "                reviews_rate = i.find_element_by_css_selector('span').get_attribute(\"aria-label\")\n",
    "                num_of_reviews = prod.find_element_by_xpath(\"//span[@class='_1gvnvab']\").text\n",
    "                writer.writerow({'Date' : Checkin_Date,\n",
    "                                 'Title': title,\n",
    "                                 'Price': price,\n",
    "                                 'Room_type': room_type,\n",
    "                                 'num_of_beds': num_of_beds,\n",
    "                                 'Additions': additions,\n",
    "                                 'Reviewrs': num_of_reviews,\n",
    "                                 'Review_stars':reviews_rate})\n",
    "            browser.find_element_by_xpath(\"//li[@class='_b8vexar']/a\")\n",
    "            #nxt_page.click()\n",
    "            time.sleep(10)\n",
    "            \n",
    "        start_date=Checkin_Date\n",
    "        \n",
    "   \n",
    "           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}