web_data_miner.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ArjunRajaSekhar/python-projects/blob/master/web_data_miner.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "kdYcb82cywsI",
        "colab_type": "code",
        "outputId": "a7fc5250-0933-4f78-b9da-51bb36f035eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1516
        },
        "cellView": "both"
      },
      "cell_type": "code",
      "source": [
        "#The given scenario is to find the events described in the following webpages:\n",
        "#https://10times.com/top100/technology\n",
        "#https://blog.bizzabo.com/technology-events\n",
        "#https://www.nasscom.in/events\n",
        "\n",
        "#The first two webpage has the data in tabular for which can be found in rows under the tags <tr>\n",
        "#the last link has the data in the form of blocks\n",
        "\n",
        "# the contents of the webpage needs to be scrapped for the data and should be defined by a structure.\n",
        "\n",
        "#EVENT_NAME\n",
        "#EVENT_COUNTRY\n",
        "#EVENT_CITY\n",
        "#EVENT_DATE\n",
        "#EVENT_TIME\n",
        "#EVENT_DETAILS\n",
        "#EVENT_TYPE\n",
        "#EVENT_FOR\n",
        "\n",
        "#Show the result as each event which can be sorted by city as well as country.\n",
        "\n",
        "\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "import pandas as pd\n",
        "import os\n",
        "#import spacy\n",
        "from urllib.request import urlopen\n",
        "import lxml.html as lh\n",
        "try: \n",
        "    from googlesearch import search \n",
        "except ImportError:  \n",
        "    print(\"No module named 'google' found\") \n",
        "  \n",
        "\n",
        "'''https://10times.com/top100/technology','https://blog.bizzabo.com/technology-events','https://www.nasscom.in/events','''\n",
        "  \n",
        "urls=['http://www.psgtech.edu/aisgsc2019/index.html','https://www.eventbrite.com/d/india--bangalore/tech-conferences/']\n",
        "\n",
        "'''\n",
        "aggreagate=[\n",
        "['html', 'head', 'meta', 'link', 'title', 'script', 'body', 'p', 'a', 'header', 'div', 'img', 'ul', 'li', 'form', 'h2', 'label', \n",
        "'input', 'span', 'h4', 'select', 'option', 'fieldset', 'legend', 'i', 'strong', 'h1', 'nav', 'button', 'h5', 'h3', 'br'],\n",
        "['html', 'head', 'meta', 'title', 'link', 'script', 'style', 'body', 'header', 'nav', 'div', 'a', 'img', 'ul', 'li', 'form', 'input',\n",
        " 'i', 'noscript', 'iframe', 'section', 'h1', 'span', 'small', 'br','select', 'option', 'h2', 'strong', 'table', 'tbody', 'tr', 'th', 'td', 'ins', 'footer', 'hr', 'p', 'u'],\n",
        "['html', 'head', 'meta', 'title', 'link', 'script', 'style', 'body', 'div', 'span', 'a', 'img', 'i', 'ul', 'li', 'form', 'input', 'h3', 'h1',\n",
        "'p', 'strong', 'table', 'tbody', 'tr', 'td', 'thead', 'th', 'br', 'h4', 'gcse:searchresults-only', 'noscript', 'iframe']]\n",
        "\n",
        "aggreagate_list=[]\n",
        "\n",
        "direct=[]\n",
        "\n",
        "direct_list=[]\n",
        "'''\n",
        "def is_executable(url):\n",
        "    \"\"\"\n",
        "    Does the url contain a downloadable resource\n",
        "    \"\"\"\n",
        "    h = requests.head(url, allow_redirects=True)\n",
        "    header = h.headers\n",
        "    content_type = header.get('content-type')\n",
        "    if 'image' in content_type.lower():\n",
        "        return False\n",
        "    if 'pdf' in content_type.lower():\n",
        "        return False\n",
        "    if 'application' in content_type.lower():\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "\n",
        "def webpage_type_analyzer(url):\n",
        "    res=[]\n",
        "    req=requests.get(url)\n",
        "    print(req)\n",
        "    html = urlopen(url).read()\n",
        "    soup = BeautifulSoup(html,\"html5lib\")\n",
        "    for tag in soup.find_all():\n",
        "        if tag.name in res:\n",
        "            pass\n",
        "        else:\n",
        "            res.append(tag.name)\n",
        "    print(res)\n",
        "    print(len(res))\n",
        "    \n",
        "#    for ele in aggreagate:\n",
        "    #if(ele==res):\n",
        "#    aggreagate.append(res)\n",
        "#    aggreagate_list.append(url)\n",
        " #   webpage_content_analyzer_aggregate(url)\n",
        "#   else:\n",
        "#    direct.append(res)\n",
        "#    direct_list.append(url)\n",
        "#    webpage_content_analyzer_direct(url)\n",
        "\n",
        "def webpage_content_analyzer_direct(url):\n",
        "    if is_executable(url) is True:\n",
        "        req=requests.get(url)\n",
        "        print(req)\n",
        "        #reading the contents of the web page\n",
        "        html = urlopen(url).read()\n",
        "        ## print(html)\n",
        "\n",
        "        soup = BeautifulSoup(html,\"html5lib\")\n",
        "        ## print(soup)\n",
        "        try:\n",
        "            blocks = soup.find(\"div\",class_=\"view-content\")\n",
        "            tags=blocks.find(\"div\",class_=\"item-list\")\n",
        "            tags2=tags.find_all('li')\n",
        "            for tag in tags2:\n",
        "                print((tag.get_text()).strip(\"\"))\n",
        "        except AttributeError as ae:\n",
        "            print(ae)\n",
        "            pass\n",
        "          \n",
        "def webpage_content_analyzer_aggregate(url):\n",
        "    if is_executable(url) is True:\n",
        "        req=requests.get(url)\n",
        "        print(req)\n",
        "        #reading the contents of the web page\n",
        "        html = urlopen(url).read()\n",
        "        ## print(html)\n",
        "\n",
        "        soup = BeautifulSoup(html,\"html5lib\")\n",
        "        ## print(soup)\n",
        "        try:\n",
        "            tbl = soup.find_all('table')\n",
        "            for tb in tbl:\n",
        "            #print(tb)\n",
        "                rows = tb.find_all('tr')\n",
        "                #print(rows)\n",
        "                for row in rows:\n",
        "                    cols = row.find_all('td')\n",
        "                    #print(cols)\n",
        "                    for col in cols:\n",
        "                        hrefs = col.find_all()\n",
        "                        for href in hrefs:\n",
        "                            text=href.get_text()\n",
        "                            print(text)\n",
        "                    print('\\n')\n",
        "        except AttributeError as ae:\n",
        "            print(ae)\n",
        "            pass\n",
        "\n",
        "for url in urls: #in search(query, tld=\"co.in\", num=1, stop=1, pause=2):\n",
        "    webpage_type_analyzer(url)\n",
        "    webpage_content_analyzer_aggregate(url)\n",
        "    webpage_content_analyzer_direct(url)\n",
        "\n",
        "#print(aggreagate)\n",
        "#print(aggreagate_list)\n",
        "#print(direct)\n",
        "#print(direct_list)\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<Response [200]>\n",
            "['html', 'head', 'meta', 'title', 'script', 'style', 'link', 'body', 'div', 'a', 'ul', 'li', 'br', 'marquee', 'center', 'p', 'b', 'span', 'h3', 'font', 'i', 'h1', 'img', 'h2', 'h5', 'h4', 'strong', 'dl', 'table', 'tbody', 'tr', 'td', 'iframe', 'aside']\n",
            "34\n",
            "<Response [200]>\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "Pre conference  \n",
            "\n",
            "\n",
            "\n",
            "Conference  \n",
            "\n",
            "\n",
            "\n",
            "Both \n",
            "\n",
            "\n",
            "\n",
            "<Response [200]>\n",
            "'NoneType' object has no attribute 'find'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "H6uxgDcCzKCL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}