Web_Scraping_Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bts\n",
    "import urllib.request as ur\n",
    "from urllib.error import HTTPError\n",
    "import os\n",
    "from multiprocessing.pool import ThreadPool\n",
    "import itertools\n",
    "import sys\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "import xlsxwriter\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "indeed = \"https://www.indeed.com\"\n",
    "fName = {'scrape':1,'addToExcel':2,'readFromExcel':3,'extractFromHome':4,'extractPages':5,'processKeywords':6}\n",
    "\n",
    "def monitor(f):\n",
    "    def f_(x):\n",
    "        countx=fName[f.__name__]\n",
    "        pool = ThreadPool(processes=2)\n",
    "        t = pool.apply_async(func=f)\n",
    "        spinner = itertools.cycle(\"|/~\\\\\")\n",
    "        while not t.ready():\n",
    "            sys.stdout.write(\"\\r[%s]: Running %s [%s of 6] %s\" % (time.ctime(), f.__name__,countx , next(spinner)))\n",
    "            sys.stdout.flush()\n",
    "            time.sleep(.5)\n",
    "        sys.stdout.write(\"\\n\")\n",
    "    return f_\n",
    "\n",
    "\n",
    "@monitor\n",
    "def scrape(value=None):\n",
    "    itr=1\n",
    "    profiles = ['business+analyst','data+scientist','data+analyst','quantitative+analyst','quantitative+researcher','product+analyst','business+analyst','business+intelligence+analyst','data+engineer','statistical+analyst']\n",
    "    locations = ['New+York','Mountain+View','Los+Angeles','San+Jose','San+Francisco','San+Diego','Sacramento','Miami','Atlanta','Chicago']\n",
    "    profiles = profiles*itr\n",
    "    locations = locations*itr\n",
    "    start = 0\n",
    "    contents = []\n",
    "    visited = []\n",
    "    for i in range(len(profiles)):\n",
    "        if profiles[i] not in visited:\n",
    "            visited.append(profiles[i])\n",
    "            start = 0\n",
    "        else:\n",
    "            start = start + 10\n",
    "        profile= profiles[i]\n",
    "        location= locations[i]\n",
    "        indeed_home = ur.urlopen('https://www.indeed.com/jobs?q='+profile+'&l='+location+'&start='+str(start))\n",
    "        soup_home = bts(indeed_home,\"html.parser\")\n",
    "        div_home_contents = soup_home.find_all('div', attrs = {'class': 'jobsearch-SerpJobCard'})\n",
    "        contents.extend(div_home_contents)\n",
    "    return contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "@monitor\n",
    "def addToExcel(contents):\n",
    "    workbook = xlsxwriter.Workbook('Rawdata.xlsx') \n",
    "    worksheet = workbook.add_worksheet()\n",
    "    row = 1\n",
    "    column = 0\n",
    "    flag = 0\n",
    "    for item in contents :\n",
    "        if flag == 0:\n",
    "            worksheet.write(0, column, 'Data')\n",
    "            flag = 1\n",
    "        worksheet.write(row, column, str(item)) \n",
    "        row += 1\n",
    "    workbook.close()\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "@monitor\n",
    "def readFromExcel(value=None):\n",
    "    df = pd.read_excel('Rawdata.xlsx', sheetname=0) # can also index sheet by name or fetch all sheets\n",
    "    mylist = df['Data'].tolist()\n",
    "    return mylist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "@monitor\n",
    "def extractFromHome(contents):\n",
    "    cols = ['Div_Id','Title','Company','Location','Summary Description','Requirements','Salary','Reviews','Date Posted','URL']\n",
    "    lst = []\n",
    "    for content in contents:\n",
    "\n",
    "        content = bts(content,\"html.parser\")\n",
    "        div_id=content.div['id']\n",
    "       \n",
    "        div_title_home = content.find('div', attrs = {'class': 'title'})\n",
    "        title=''\n",
    "        if div_title_home!=None:\n",
    "            div_title_home_title = bts(str(div_title_home))\n",
    "            title = div_title_home_title.a['title']\n",
    "        else:\n",
    "            div_title_home = content.find('h2', attrs = {'class': 'title'})\n",
    "            div_title_home_title = bts(str(div_title_home))\n",
    "            title = div_title_home_title.a['title']\n",
    "            \n",
    "        link_url = indeed+div_title_home_title.a['href']\n",
    "        if link_url == None:\n",
    "            link_url = 'Not Available'\n",
    "        \n",
    "        div_requirements_home = content.find('div', attrs= {'class': 'jobCardReqContainer'})\n",
    "        requirements = ''\n",
    "        if div_requirements_home!=None:\n",
    "            requirements_t = content.find_all('div', attrs= {'class': 'jobCardReqList'})\n",
    "            for i in requirements_t:\n",
    "                requirements = '#' + requirements\n",
    "                requirements = requirements + i.get_text().strip()\n",
    "        else: requirements = 'Not Available'\n",
    "            \n",
    "        div_company_review_home = content.find('div', attrs = {'class': 'sjcl'})\n",
    "        span_company = div_company_review_home.find('span', attrs = {'class': 'company'})\n",
    "        company = span_company.get_text().strip()\n",
    "        \n",
    "        \n",
    "        \n",
    "        l_flag = 0\n",
    "        div_location = div_company_review_home.find('span', attrs = {'class': 'location'})\n",
    "        if div_location != None:\n",
    "            l_flag = 1\n",
    "            location = div_location.get_text().strip()\n",
    "        else:\n",
    "            location = 'Not Available'\n",
    "        \n",
    "        if l_flag == 0:\n",
    "            div_location = div_company_review_home.find('div', attrs = {'class': 'location'})\n",
    "            if div_location != None:\n",
    "                location = div_location.get_text().strip()\n",
    "            else:\n",
    "                location = 'Not Available'\n",
    "            \n",
    "        span_review = div_company_review_home.find('span', attrs = {'class': 'slNoUnderline'})\n",
    "        if span_review != None:\n",
    "            review = span_review.get_text().strip()\n",
    "        else:\n",
    "            review = 'Not Available'\n",
    "        \n",
    "        div_salary_home = content.find('div', attrs = {'class': 'salarySnippet'})\n",
    "        if div_salary_home != None:\n",
    "            span_salary = div_salary_home.find('span', attrs = {'class': 'salary'})\n",
    "            salary = span_salary.get_text().strip()\n",
    "        else:\n",
    "            salary = 'Not Available'\n",
    "        \n",
    "        div_summary_description = content.find('div', attrs = {'class': 'summary'})\n",
    "        if div_summary_description != None:\n",
    "            summary_description = div_summary_description.get_text().strip()\n",
    "        else:\n",
    "            summary_description = 'Not Available'\n",
    "        \n",
    "        div_result_link_bar_container_home = content.find('div', attrs = {'class': 'result-link-bar-container'})\n",
    "        span_post_date = div_result_link_bar_container_home.find('span', attrs = {'class': 'date'})\n",
    "        if span_post_date != None:\n",
    "            post_date = span_post_date.get_text().strip()\n",
    "        else:\n",
    "            post_date = 'Not Available'\n",
    "        \n",
    "        lst.append([div_id,title,company,location,summary_description,requirements,salary,review,post_date,str(link_url)])\n",
    "    dfa = pd.DataFrame(lst, columns=cols)\n",
    "    #dfa.index = np.arange(1, len(dfa) + 1)\n",
    "    dfa.to_csv('records_home.csv',index=False)\n",
    "    os.remove('Rawdata.xlsx')\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "@monitor\n",
    "def extractPages(value=None):\n",
    "    result=pd.read_csv(\"records_home.csv\")\n",
    "    temp = []\n",
    "    for url in result['URL']:\n",
    "        page_content=None\n",
    "        if url != None:\n",
    "            try:\n",
    "                ideed_page_object = ur.urlopen(url)\n",
    "                ideed_page = bts(ideed_page_object)\n",
    "                page_content = ideed_page.find('div', attrs = {'class': 'jobsearch-JobComponent-description'})\n",
    "            except urllib.error.HTTPError as err:\n",
    "                print(err.code)\n",
    "        if page_content!=None:\n",
    "            page_content = page_content.get_text().strip()\n",
    "        else:\n",
    "            page_content = 'Page Content Not Available'\n",
    "        temp.append(page_content)\n",
    "      \n",
    "    result['Description'] = temp\n",
    "    result.to_csv('records_home.csv',index=False)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "@monitor\n",
    "def processKeywords(value=None):\n",
    "    csv_data = pd.read_csv(\"records_home.csv\")\n",
    "    job_data = csv_data['Description']\n",
    "    keywords = ['Python','Sql','Java','R','Ruby','Matlab','SAS','Hadoop','Hive','Spark','Pig','HBase','Tableau','Spotfire','Alteryx','Excel','MapReduce','DBMS','Database','Database Management','Ruby on Rails']\n",
    "    key_list = []\n",
    "    for data in job_data:\n",
    "        counts = dict()\n",
    "        for line in data.split(\"\\n\"):\n",
    "            for keyword in keywords:\n",
    "                regex = \"\\b\"+keyword+\"\\b\"\n",
    "                if keyword=='Python':\n",
    "                    reg = re.compile(r\"\\bpython\\b\",re.I)\n",
    "                elif keyword=='Sql':\n",
    "                    reg = re.compile(r\"\\bsql\\b\",re.I)\n",
    "                elif keyword=='Java':\n",
    "                    reg = re.compile(r\"\\bjava\\b\",re.I)\n",
    "                elif keyword=='R':\n",
    "                    reg = re.compile(r\"\\br\\b\",re.I)\n",
    "                elif keyword=='Ruby':\n",
    "                    reg = re.compile(r\"\\bruby\\b\",re.I)\n",
    "                elif keyword=='Ruby on Rails':\n",
    "                    reg = re.compile(r\"\\bruby on rails\\b\",re.I)\n",
    "                elif keyword=='Matlab':\n",
    "                    reg = re.compile(r\"\\bmatlab\\b\",re.I)\n",
    "                elif keyword=='Hive':\n",
    "                    reg = re.compile(r\"\\bhive\\b\",re.I)\n",
    "                elif keyword=='Spark':\n",
    "                    reg = re.compile(r\"\\bspark\\b\",re.I)\n",
    "                elif keyword=='Pig':\n",
    "                    reg = re.compile(r\"\\bpig\\b\",re.I)\n",
    "                elif keyword=='HBase':\n",
    "                    reg = re.compile(r\"\\bhbase\\b\",re.I)\n",
    "                elif keyword=='Tableau':\n",
    "                    reg = re.compile(r\"\\btableau\\b\",re.I)\n",
    "                elif keyword=='Spotfire':\n",
    "                    reg = re.compile(r\"\\bspotfire\\b\",re.I)\n",
    "                elif keyword=='Alteryx':\n",
    "                    reg = re.compile(r\"\\balteryx\\b\",re.I)\n",
    "                elif keyword=='Excel':\n",
    "                    reg = re.compile(r\"\\bexcel\\b\",re.I)\n",
    "                elif keyword=='MapReduce':\n",
    "                    reg = re.compile(r\"\\bmapreduce\\b\",re.I)\n",
    "                elif keyword=='SAS':\n",
    "                    reg = re.compile(r\"\\bsas\\b\",re.I)\n",
    "                elif keyword=='Hadoop':\n",
    "                    reg = re.compile(r\"\\bhadoop\\b\",re.I)\n",
    "                elif keyword=='DBMS':\n",
    "                    reg = re.compile(r\"\\bdbms\\b\",re.I)\n",
    "                elif keyword=='Database':\n",
    "                    reg = re.compile(r\"\\bdatabase\\b\",re.I)\n",
    "                elif keyword=='Database Management':\n",
    "                    reg = re.compile(r\"\\bdatabase management\\b\",re.I)\n",
    "                elif keyword=='R Programming':\n",
    "                    reg = re.compile(r\"\\br programming\\b\",re.I)  \n",
    "                for kw in reg.findall(line):\n",
    "                    if kw=='ruby' or kw=='ruby on rails':\n",
    "                        keyword='Ruby'\n",
    "                    if kw=='dbms' or kw=='database' or kw=='database management':\n",
    "                        keyword='DBMS'\n",
    "                    if kw=='R' or kw=='R Programming':\n",
    "                        keyword='R'\n",
    "                    counts[keyword] = counts.get(keyword,0) + 1\n",
    "        key_list.append(counts.copy())\n",
    "    csv_data['Skills'] = key_list\n",
    "    csv_data.to_csv('records_home.csv',index=False)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Wed Apr 15 16:38:57 2020]: Running scrape [1 of 6] ~\n",
      "[Wed Apr 15 16:38:57 2020]: Running addToExcel [2 of 6] |\n",
      "[Wed Apr 15 16:38:58 2020]: Running readFromExcel [3 of 6] |\n",
      "[Wed Apr 15 16:38:58 2020]: Running extractFromHome [4 of 6] |\n",
      "[Wed Apr 15 16:41:10 2020]: Running extractPages [5 of 6] \\\n",
      "[Wed Apr 15 16:41:11 2020]: Running processKeywords [6 of 6] /\n",
      "CSV File is Ready!\n"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    val=''\n",
    "    content = scrape(val)\n",
    "    addToExcel(content)\n",
    "    contents = readFromExcel(val)\n",
    "    extractFromHome(contents)\n",
    "    extractPages(val)\n",
    "    processKeywords(val)\n",
    "    print('CSV File is Ready!')\n",
    "    return\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}