5.AB-Test I.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Does the experimental web page drive higher traffic than the control web page ?\n",
    "More Click-Through-Rate in the new website ? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>id</th>\n",
       "      <th>group</th>\n",
       "      <th>action</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-09-24 17:42:27.839496</td>\n",
       "      <td>804196</td>\n",
       "      <td>experiment</td>\n",
       "      <td>view</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-09-24 19:19:03.542569</td>\n",
       "      <td>434745</td>\n",
       "      <td>experiment</td>\n",
       "      <td>view</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-09-24 19:36:00.944135</td>\n",
       "      <td>507599</td>\n",
       "      <td>experiment</td>\n",
       "      <td>view</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-09-24 19:59:02.646620</td>\n",
       "      <td>671993</td>\n",
       "      <td>control</td>\n",
       "      <td>view</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-09-24 20:26:14.466886</td>\n",
       "      <td>536734</td>\n",
       "      <td>experiment</td>\n",
       "      <td>view</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    timestamp      id       group action\n",
       "0  2016-09-24 17:42:27.839496  804196  experiment   view\n",
       "1  2016-09-24 19:19:03.542569  434745  experiment   view\n",
       "2  2016-09-24 19:36:00.944135  507599  experiment   view\n",
       "3  2016-09-24 19:59:02.646620  671993     control   view\n",
       "4  2016-09-24 20:26:14.466886  536734  experiment   view"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('C:/Users/Minkun/Desktop/classes_1/NanoDeg/1.Data_AN/L4/data/homepage_actions.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8188 entries, 0 to 8187\n",
      "Data columns (total 4 columns):\n",
      "timestamp    8188 non-null object\n",
      "id           8188 non-null int64\n",
      "group        8188 non-null object\n",
      "action       8188 non-null object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 256.0+ KB\n"
     ]
    }
   ],
   "source": [
    "# total number of actions\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6328"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of unique users\n",
    "df['id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3332, 2996)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# size of control group and experiment group\n",
    "df.query('group == \"control\"').id.nunique(), df.query('group == \"experiment\"').id.nunique() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('2016-09-24 17:42:27.839496', '2017-01-18 10:24:08.629327')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# duration of this experiment\n",
    "df.timestamp.min(), df.timestamp.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['view', 'click'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# action types in this experiment\n",
    "df.action.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Why would we use click through rate instead of number of clicks to compare the performances of control and experiment pages?\n",
    "\n",
    "because... Getting the proportion of the users who click is more effective than getting the number of users who click when comparing groups of different sizes.ie..more total clicks could occur in one version, even if there is a greater percentage of clicks in the other version (simpson's paradox)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Definition of click through rate (CTR) for this experiment\n",
    "\n",
    "The No.of unique visitors who click at least once / The No.of unique visitors who view the page"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Use $CTR_{old}$ and $CTR_{new}$ in our hypotheses.\n",
    "\n",
    "$$H_0: CTR_{new} - CTR_{old} \\leq 0$$  \n",
    "$$H_1: CTR_{new} - CTR_{old} > 0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Steps to analyze the results of this A/B test.\n",
    "\n",
    "`1.` We computed the observed difference between the `metric`, **click through rate**, for the control and experiment group.\n",
    "\n",
    "`2.` We simulated the sampling distribution for the difference in proportions (or difference in click through rates).\n",
    "\n",
    "`3.` We used this sampling distribution to simulate the distribution under the null hypothesis, by creating a random normal distribution centered at 0 with the same spread and size.\n",
    "\n",
    "`4.` We computed the p-value by finding the proportion of values in the null distribution that were greater than our observed difference.\n",
    "\n",
    "`5.` We used this p-value to determine the statistical significance of our observed difference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2797118847539016"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract all the actions from control group\n",
    "control_df = df.query('group == \"control\"')\n",
    "\n",
    "# compute click through rate\n",
    "control_ctr = control_df.query('action == \"click\"').id.nunique() / control_df.query('action == \"view\"').id.nunique(); \n",
    "control_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3097463284379172"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract all actions from experiment group\n",
    "experiment_df = df.query('group == \"experiment\"')\n",
    "\n",
    "# compute click through rate\n",
    "experiment_ctr = experiment_df.query('action == \"click\"').id.nunique() / experiment_df.query('action == \"view\"').id.nunique();\n",
    "experiment_ctr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.030034443684015644"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute observed difference in CTR\n",
    "obs_diff = experiment_ctr - control_ctr; obs_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# See if this difference is significant !\n",
    "\n",
    "#BOOTSTRAPPING ! to simulate the sampling distribution for the difference in proportions - CTRs !\n",
    "\n",
    "diffs = []\n",
    "\n",
    "for _ in range(10000):\n",
    "    b_samp = df.sample(df.shape[0], replace=True)\n",
    "    control_df = b_samp.query('group == \"control\"')\n",
    "    experiment_df = b_samp.query('group == \"experiment\"')\n",
    "    control_ctr = control_df.query('action == \"click\"').id.nunique() / control_df.query('action == \"view\"').id.nunique()\n",
    "    experiment_ctr = experiment_df.query('action == \"click\"').id.nunique() / experiment_df.query('action == \"view\"').id.nunique()\n",
    "    diffs.append(experiment_ctr - control_ctr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Compute the C.I. ?????? we need t-statistics??? WTF? \n",
    "#upper = np.percentile(diffs, 95)\n",
    "#lower = np.percentile(diffs, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([   13.,   102.,   520.,  1535.,  2768.,  2858.,  1608.,   504.,\n",
       "           87.,     5.]),\n",
       " array([-0.01611488, -0.00691579,  0.00228329,  0.01148237,  0.02068145,\n",
       "         0.02988054,  0.03907962,  0.0482787 ,  0.05747778,  0.06667687,\n",
       "         0.07587595]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEfNJREFUeJzt3X+MndV95/H3p5gQSkKBZtZyjLMQrdvIIIWEkZdtq4pu\nusUL0ZqqUuRILVRNcSvYKOlW2jXtH03/sESa9IfQNuy6TRbTJkFWkxQrCbsFK1VbqYYOKcHYhMUJ\nsHjWYHerLun+QdfOt3/c4+ZmzpgZz70zdwbeL+nRPfc85zzPeY7G85nnx71OVSFJ0rDvmfQAJEmr\nj+EgSeoYDpKkjuEgSeoYDpKkjuEgSeosGA5J3pjk0SRfS3I4ya+3+suSPJTkmfZ66VCfO5McTfJ0\nkhuG6q9NcqituztJluewJEmjWMyZwyvAv66qdwLXANuSXAfsAg5U1WbgQHtPki3ADuAqYBvwiSTn\ntW3dA9wGbG7LtjEeiyRpTBYMhxr4+/b2/LYUsB3Y2+r3Aje38nbg/qp6paqeBY4CW5NsAC6uqoM1\n+OTdfUN9JEmryLrFNGp/+T8G/Avgd6vqkSTrq+p4a/IisL6VNwIHh7ofa3X/v5Xn1s+3v53AToCL\nLrro2ne84x2LOxpJEgCPPfbY31TV1FL7Lyocquo0cE2SS4AvJLl6zvpKMrbv4aiqPcAegOnp6ZqZ\nmRnXpiXpdSHJ86P0P6enlarq74CvMLhX8FK7VER7PdGazQKbhrpd3upmW3luvSRplVnM00pT7YyB\nJBcC/wb4OrAfuLU1uxV4oJX3AzuSXJDkSgY3nh9tl6BeTnJde0rplqE+kqRVZDGXlTYAe9t9h+8B\n9lXVF5P8JbAvyQeA54H3AVTV4ST7gCPAKeCOdlkK4HbgXuBC4MG2SJJWmaz2r+z2noMknbskj1XV\n9FL7+wlpSVLHcJAkdQwHSVLHcJAkdQwHSVJnUZ+QltS7YteXJrbv5+66aWL71uuDZw6SpI7hIEnq\nGA6SpI7hIEnqGA6SpI5PK2nNm+RTQ9JrlWcOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO\n4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOguGQZFOSryQ5kuRwkg+1+o8kmU3y\neFtuHOpzZ5KjSZ5OcsNQ/bVJDrV1dyfJ8hyWJGkUi/nPfk4Bv1xVX03yZuCxJA+1db9dVR8fbpxk\nC7ADuAp4K/Bwkh+oqtPAPcBtwCPAl4FtwIPjORRJ0rgseOZQVcer6qut/C3gKWDjq3TZDtxfVa9U\n1bPAUWBrkg3AxVV1sKoKuA+4eeQjkCSN3Tndc0hyBfAuBn/5A3wwyRNJPpXk0la3EXhhqNuxVrex\nlefWS5JWmUWHQ5I3AZ8DPlxVLzO4RPR24BrgOPCb4xpUkp1JZpLMnDx5clyblSQt0qLCIcn5DILh\n01X1eYCqeqmqTlfVt4HfA7a25rPApqHul7e62VaeW9+pqj1VNV1V01NTU+dyPJKkMVjM00oBPgk8\nVVW/NVS/YajZTwJPtvJ+YEeSC5JcCWwGHq2q48DLSa5r27wFeGBMxyFJGqPFPK30w8DPAIeSPN7q\nfgV4f5JrgAKeA34BoKoOJ9kHHGHwpNMd7UklgNuBe4ELGTyl5JNKkrQKLRgOVfUXwHyfR/jyq/TZ\nDeyep34GuPpcBihJWnl+QlqS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEc\nJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkd\nw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEmdBcMhyaYkX0lyJMnhJB9q9ZcleSjJM+310qE+dyY5\nmuTpJDcM1V+b5FBbd3eSLM9hSZJGsZgzh1PAL1fVFuA64I4kW4BdwIGq2gwcaO9p63YAVwHbgE8k\nOa9t6x7gNmBzW7aN8VgkSWOyYDhU1fGq+morfwt4CtgIbAf2tmZ7gZtbeTtwf1W9UlXPAkeBrUk2\nABdX1cGqKuC+oT6SpFXknO45JLkCeBfwCLC+qo63VS8C61t5I/DCULdjrW5jK8+tn28/O5PMJJk5\nefLkuQxRkjQGiw6HJG8CPgd8uKpeHl7XzgRqXIOqqj1VNV1V01NTU+ParCRpkRYVDknOZxAMn66q\nz7fql9qlItrriVY/C2wa6n55q5tt5bn1kqRVZjFPKwX4JPBUVf3W0Kr9wK2tfCvwwFD9jiQXJLmS\nwY3nR9slqJeTXNe2ectQH0nSKrJuEW1+GPgZ4FCSx1vdrwB3AfuSfAB4HngfQFUdTrIPOMLgSac7\nqup063c7cC9wIfBgWyRJq8yC4VBVfwGc7fMI7zlLn93A7nnqZ4Crz2WAkqSV5yekJUkdw0GS1DEc\nJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkd\nw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1Fk36QFIOndX7PrSRPb73F03TWS/WnmeOUiS\nOoaDJKljOEiSOoaDJKmzYDgk+VSSE0meHKr7SJLZJI+35cahdXcmOZrk6SQ3DNVfm+RQW3d3koz/\ncCRJ47CYp5XuBf4zcN+c+t+uqo8PVyTZAuwArgLeCjyc5Aeq6jRwD3Ab8AjwZWAb8OBIo9eqMamn\nZyQtjwXPHKrqz4C/XeT2tgP3V9UrVfUscBTYmmQDcHFVHayqYhA0Ny910JKk5TXKPYcPJnmiXXa6\ntNVtBF4YanOs1W1s5bn180qyM8lMkpmTJ0+OMERJ0lIsNRzuAd4OXAMcB35zbCMCqmpPVU1X1fTU\n1NQ4Ny1JWoQlhUNVvVRVp6vq28DvAVvbqllg01DTy1vdbCvPrZckrUJLCod2D+GMnwTOPMm0H9iR\n5IIkVwKbgUer6jjwcpLr2lNKtwAPjDBuSdIyWvBppSSfBa4H3pLkGPBrwPVJrgEKeA74BYCqOpxk\nH3AEOAXc0Z5UAridwZNPFzJ4SsknlSRplVowHKrq/fNUf/JV2u8Gds9TPwNcfU6jkyRNhJ+QliR1\nDAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJ\nUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdw\nkCR1FgyHJJ9KciLJk0N1lyV5KMkz7fXSoXV3Jjma5OkkNwzVX5vkUFt3d5KM/3AkSeOwmDOHe4Ft\nc+p2AQeqajNwoL0nyRZgB3BV6/OJJOe1PvcAtwGb2zJ3m5KkVWLBcKiqPwP+dk71dmBvK+8Fbh6q\nv7+qXqmqZ4GjwNYkG4CLq+pgVRVw31AfSdIqs9R7Duur6ngrvwisb+WNwAtD7Y61uo2tPLd+Xkl2\nJplJMnPy5MklDlGStFQj35BuZwI1hrEMb3NPVU1X1fTU1NQ4Ny1JWoSlhsNL7VIR7fVEq58FNg21\nu7zVzbby3HpJ0iq01HDYD9zayrcCDwzV70hyQZIrGdx4frRdgno5yXXtKaVbhvpIklaZdQs1SPJZ\n4HrgLUmOAb8G3AXsS/IB4HngfQBVdTjJPuAIcAq4o6pOt03dzuDJpwuBB9siSVqFFgyHqnr/WVa9\n5yztdwO756mfAa4+p9FJkibCT0hLkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhI\nkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqG\ngySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjojhUOS55IcSvJ4kplWd1mSh5I8014vHWp/Z5KjSZ5O\ncsOog5ckLY9xnDn8WFVdU1XT7f0u4EBVbQYOtPck2QLsAK4CtgGfSHLeGPYvSRqzdcuwze3A9a28\nF/hT4D+1+vur6hXg2SRHga3AXy7DGF63rtj1pUkPQdJrwKjhUMDDSU4D/7Wq9gDrq+p4W/8isL6V\nNwIHh/oea3WS1ohJ/fHx3F03TWS/r2ejhsOPVNVskn8GPJTk68Mrq6qS1LluNMlOYCfA2972thGH\nKEk6VyPdc6iq2fZ6AvgCg8tELyXZANBeT7Tms8Cmoe6Xt7r5trunqqaranpqamqUIUqSlmDJ4ZDk\noiRvPlMGfgJ4EtgP3Nqa3Qo80Mr7gR1JLkhyJbAZeHSp+5ckLZ9RLiutB76Q5Mx2PlNV/z3JXwH7\nknwAeB54H0BVHU6yDzgCnALuqKrTI41ekrQslhwOVfVN4J3z1P8f4D1n6bMb2L3UfUqSVoafkJYk\ndQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwH\nSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkddZNegCvRVfs+tKkhyBJ\nIzEcJK16k/yD67m7bprYvifJy0qSpI7hIEnqGA6SpM6Kh0OSbUmeTnI0ya6V3r8kaWErGg5JzgN+\nF/i3wBbg/Um2rOQYJEkLW+mnlbYCR6vqmwBJ7ge2A0eWY2c+UippVJP6PTLpp6RWOhw2Ai8MvT8G\n/Mu5jZLsBHa2t3+f5GngLcDfLPsIVz/nYcB5+A7nYuA1NQ/56JK7npmHfz7K/lfl5xyqag+wZ7gu\nyUxVTU9oSKuG8zDgPHyHczHgPAyMax5W+ob0LLBp6P3lrU6StIqsdDj8FbA5yZVJ3gDsAPav8Bgk\nSQtY0ctKVXUqyb8H/gdwHvCpqjq8yO57Fm7yuuA8DDgP3+FcDDgPA2OZh1TVOLYjSXoN8RPSkqSO\n4SBJ6qyqcEhyWZKHkjzTXi89S7t5v4IjyceSfD3JE0m+kOSSlRv96Bb6apEM3N3WP5Hk3Yvtu5Ys\ndR6SbErylSRHkhxO8qGVH/34jPLz0Nafl+Svk3xx5UY9fiP+u7gkyR+13wtPJflXKzv68RlxHn6p\n/Zt4Mslnk7xxwR1W1apZgN8AdrXyLuCj87Q5D/gG8HbgDcDXgC1t3U8A61r5o/P1X63Lqx3XUJsb\ngQeBANcBjyy271pZRpyHDcC7W/nNwP98Pc7D0Pr/AHwG+OKkj2dS8wDsBX6+ld8AXDLpY1rpeWDw\n4eNngQvb+33Azy60z1V15sDgqzT2tvJe4OZ52vzTV3BU1T8AZ76Cg6r6k6o61dodZPA5irXirMc1\nZDtwXw0cBC5JsmGRfdeKJc9DVR2vqq8CVNW3gKcY/MNYi0b5eSDJ5cBNwO+v5KCXwZLnIcn3AT8K\nfBKgqv6hqv5uJQc/RiP9PDB4MvXCJOuA7wX+90I7XG3hsL6qjrfyi8D6edrM9xUc8/0C+DkGKbpW\nLOa4ztZmsXOyFowyD/8kyRXAu4BHxj7ClTHqPPwO8B+Bby/XAFfIKPNwJXAS+G/t8trvJ7loOQe7\njJY8D1U1C3wc+F/AceD/VtWfLLTDSXxl98Ptutfc5btSsAbnP0t6zjbJrwKngE+PYchaY5K8Cfgc\n8OGqennS41lpSd4LnKiqxyY9lglbB7wbuKeq3gX8PwaXq19X2r3b7QzC8q3ARUl+eqF+K/7dSlX1\n42dbl+SlM5cH2unQiXmavepXcCT5WeC9wHtawKwVi/lqkbO1OX8RfdeKUeaBJOczCIZPV9Xnl3Gc\ny22Uefgp4N8luRF4I3Bxkj+sqgV/IaxCo8xDAceq6szZ4x+xdsNhlHn4ceDZqjoJkOTzwA8Bf/iq\ne5z0jZY5N1Q+xnffkP6NedqsA77JIAXP3Ji5qq3bxuDrv6cmfSxLOPazHtdQm5v47htOjy6271pZ\nRpyHAPcBvzPp45jkPMxpcz1r+4b0SPMA/Dnwg638EeBjkz6mlZ4HBt98fZjBvYYwuJ/7wQX3OemD\nnnNw3w8cAJ4BHgYua/VvBb481O5GBk+ifAP41aH6owyuuT3elv8y6WM6x+Pvjgv4ReAXWzkM/rOk\nbwCHgOmF5mQtLkudB+BHGPy1+MTQz8CNkz6eSfw8DG1jTYfDqPMAXAPMtJ+JPwYunfTxTGgefh34\nOvAk8AfABQvtz6/PkCR1VtvTSpKkVcBwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUucfAbPq4IWl\nPlUhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x203c50a0a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# convert to numpy ?????\n",
    "diffs = np.array(diffs)\n",
    "                 \n",
    "# plot sampling distribution\n",
    "plt.hist(diffs)\n",
    "#plt.axvline(x=upper, color='r', linewidth=2)\n",
    "#plt.axvline(x=lower, color='r', linewidth=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Simulating From the Null Hypothesis\n",
    "### $$H_0: CTR_{new} - CTR_{old} \\leq 0$$  \n",
    "#### : Simulating what we believe to be a possible under the H0, then seeing if our data in reality is consistent with that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# simulate distribution under the null hypothesis\n",
    "null_vals = np.random.normal(0, diffs.std(), diffs.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEYpJREFUeJzt3X+s3fVdx/Hny4KIbmQg19qVYpnWJYVkbDQVM/9AUamw\nWIzJ0iUOjAt1AZctLjFl+0P9ownb1BkSwaBbKHGTNNkWmg1U1syYJQK7IKNrWaUbEHpTaLfFMP9B\ny97+cT61x7vb3nPvufeeXj/PR/LN+ZzP9/P5fj/fT2/6ut8f59xUFZKkPv3IpAcgSZocQ0CSOmYI\nSFLHDAFJ6pghIEkdMwQkqWPzhkCSH0vyRJKvJzmY5E9b/SVJHk3yXHu9eKjPnUmOJDmc5Iah+muS\nHGjr7k6S5TksSdIoRjkTeA34lap6G3A1sC3JtcAuYH9VbQL2t/ck2QzsAK4EtgH3JFnTtnUvcBuw\nqS3blvBYJEkLNG8I1MB/trfnt6WA7cCeVr8HuLmVtwMPVtVrVfU8cATYmmQdcFFVPVaDT6g9MNRH\nkjQB543SqP0m/yTwc8BfVdXjSdZW1bHW5GVgbSuvBx4b6n601f13K8+uP6tLL720Nm7cOMowJem0\nw4cHr29962THMSFPPvnkd6pqar52I4VAVb0OXJ3kTcAXklw1a30lWbLvn0iyE9gJcPnllzM9Pb1U\nm5bUi+uuG7z+8z9PchQTk+TFUdot6OmgqvoP4CsMruW/0i7x0F6Pt2YzwIahbpe1uplWnl0/137u\nq6otVbVlamreIJMkLdIoTwdNtTMAklwI/BrwTWAfcGtrdivwUCvvA3YkuSDJFQxuAD/RLh29muTa\n9lTQLUN9JEkTMMrloHXAnnZf4EeAvVX1xST/CuxN8j7gReDdAFV1MMle4BBwErijXU4CuB24H7gQ\neKQtkqQJmTcEquoZ4O1z1H8XuP4MfXYDu+eonwau+uEekqRJ8BPDktQxQ0CSOmYISFLHDAFJ6pgh\nIEkdG+kTw5J+2MZdX5rYvl+466aJ7Vv/v3gmIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhS\nxwwBSeqYISBJHTMEJKljfm2EVr1Jfn2DtNp5JiBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghI\nUscMAUnqmCEgSR0zBCSpY/OGQJINSb6S5FCSg0k+2Or/JMlMkqfbcuNQnzuTHElyOMkNQ/XXJDnQ\n1t2dJMtzWJKkUYzy3UEngQ9X1VNJ3gg8meTRtu6TVfVnw42TbAZ2AFcCbwa+nOTnq+p14F7gNuBx\n4GFgG/DI0hyKJGmh5j0TqKpjVfVUK38feBZYf5Yu24EHq+q1qnoeOAJsTbIOuKiqHquqAh4Abh77\nCCRJi7agewJJNgJvZ/CbPMAHkjyT5NNJLm5164GXhrodbXXrW3l2vSRpQkYOgSRvAD4HfKiqXmVw\naectwNXAMeDPl2pQSXYmmU4yfeLEiaXarCRplpFCIMn5DALgM1X1eYCqeqWqXq+qHwB/A2xtzWeA\nDUPdL2t1M608u/6HVNV9VbWlqrZMTU0t5HgkSQswytNBAT4FPFtVfzFUv26o2W8B32jlfcCOJBck\nuQLYBDxRVceAV5Nc27Z5C/DQEh2HJGkRRnk66J3Ae4EDSZ5udR8B3pPkaqCAF4DfB6iqg0n2AocY\nPFl0R3syCOB24H7gQgZPBflkkCRN0LwhUFVfBeZ6nv/hs/TZDeyeo34auGohA5QkLR8/MSxJHTME\nJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CS\nOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKlj\nhoAkdcwQkKSOzRsCSTYk+UqSQ0kOJvlgq78kyaNJnmuvFw/1uTPJkSSHk9wwVH9NkgNt3d1JsjyH\nJUkaxShnAieBD1fVZuBa4I4km4FdwP6q2gTsb+9p63YAVwLbgHuSrGnbuhe4DdjUlm1LeCySpAWa\nNwSq6lhVPdXK3weeBdYD24E9rdke4OZW3g48WFWvVdXzwBFga5J1wEVV9VhVFfDAUB9J0gQs6J5A\nko3A24HHgbVVdaytehlY28rrgZeGuh1tdetbeXb9XPvZmWQ6yfSJEycWMkRJ0gKMHAJJ3gB8DvhQ\nVb06vK79Zl9LNaiquq+qtlTVlqmpqaXarCRplpFCIMn5DALgM1X1+Vb9SrvEQ3s93upngA1D3S9r\ndTOtPLtekjQhozwdFOBTwLNV9RdDq/YBt7byrcBDQ/U7klyQ5AoGN4CfaJeOXk1ybdvmLUN9JEkT\ncN4Ibd4JvBc4kOTpVvcR4C5gb5L3AS8C7waoqoNJ9gKHGDxZdEdVvd763Q7cD1wIPNIWSdKEzBsC\nVfVV4EzP819/hj67gd1z1E8DVy1kgJKk5eMnhiWpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ\n6pghIEkdG+UTw5LOMRt3fWki+33hrpsmsl8tH88EJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscM\nAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1bN4Q\nSPLpJMeTfGOo7k+SzCR5ui03Dq27M8mRJIeT3DBUf02SA23d3Umy9IcjSVqIUc4E7ge2zVH/yaq6\nui0PAyTZDOwArmx97kmyprW/F7gN2NSWubYpSVpB84ZAVf0L8L0Rt7cdeLCqXquq54EjwNYk64CL\nquqxqirgAeDmxQ5akrQ0xrkn8IEkz7TLRRe3uvXAS0Ntjra69a08u35OSXYmmU4yfeLEiTGGKEk6\nm8WGwL3AW4CrgWPAny/ZiICquq+qtlTVlqmpqaXctCRpyKJCoKpeqarXq+oHwN8AW9uqGWDDUNPL\nWt1MK8+ulyRN0KJCoF3jP+W3gFNPDu0DdiS5IMkVDG4AP1FVx4BXk1zbngq6BXhojHFLkpbAefM1\nSPL3wHXApUmOAn8MXJfkaqCAF4DfB6iqg0n2AoeAk8AdVfV629TtDJ40uhB4pC2SpAmaNwSq6j1z\nVH/qLO13A7vnqJ8GrlrQ6CRJy8pPDEtSxwwBSeqYISBJHZv3noA0io27vjTpIUhaBM8EJKljhoAk\ndcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLH\nDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSerYvCGQ5NNJjif5xlDdJUke\nTfJce714aN2dSY4kOZzkhqH6a5IcaOvuTpKlPxxJ0kKMciZwP7BtVt0uYH9VbQL2t/ck2QzsAK5s\nfe5Jsqb1uRe4DdjUltnblCStsHlDoKr+BfjerOrtwJ5W3gPcPFT/YFW9VlXPA0eArUnWARdV1WNV\nVcADQ30kSROy2HsCa6vqWCu/DKxt5fXAS0Ptjra69a08u35OSXYmmU4yfeLEiUUOUZI0n7FvDLff\n7GsJxjK8zfuqaktVbZmamlrKTUuShiw2BF5pl3hor8db/QywYajdZa1uppVn10uSJmixIbAPuLWV\nbwUeGqrfkeSCJFcwuAH8RLt09GqSa9tTQbcM9ZEkTch58zVI8vfAdcClSY4CfwzcBexN8j7gReDd\nAFV1MMle4BBwErijql5vm7qdwZNGFwKPtEWSNEHzhkBVvecMq64/Q/vdwO456qeBqxY0OknSsvIT\nw5LUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEg\nSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdO2/SA5C0emzc9aWJ7PeFu26a\nyH574JmAJHXMEJCkjhkCktQxQ0CSOmYISFLHxgqBJC8kOZDk6STTre6SJI8mea69XjzU/s4kR5Ic\nTnLDuIOXJI1nKc4Efrmqrq6qLe39LmB/VW0C9rf3JNkM7ACuBLYB9yRZswT7lyQt0nJcDtoO7Gnl\nPcDNQ/UPVtVrVfU8cATYugz7lySNaNwQKODLSZ5MsrPVra2qY638MrC2ldcDLw31PdrqJEkTMu4n\nhn+pqmaS/BTwaJJvDq+sqkpSC91oC5SdAJdffvmYQ5QknclYZwJVNdNejwNfYHB555Uk6wDa6/HW\nfAbYMNT9slY313bvq6otVbVlampqnCFKks5i0SGQ5CeSvPFUGfh14BvAPuDW1uxW4KFW3gfsSHJB\nkiuATcATi92/JGl841wOWgt8Icmp7Xy2qv4hydeAvUneB7wIvBugqg4m2QscAk4Cd1TV62ONXpI0\nlkWHQFV9G3jbHPXfBa4/Q5/dwO7F7lOStLT8xLAkdcwQkKSO+Udl/p+Z1B/9kLQ6eSYgSR0zBCSp\nY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpm\nCEhSxwwBSeqYISBJHTMEJKljhoAkdcy/MSzpnLeYv5394Le/C8COMf/u9gt33TRW/3OdZwKS1DFD\nQJI65uWgZbCYU1dJmgTPBCSpYyseAkm2JTmc5EiSXSu9f0nSaSsaAknWAH8F/AawGXhPks0rOQZJ\n0mkrfSawFThSVd+uqv8CHgS2r/AYJEnNSt8YXg+8NPT+KPALy7Uzb9BKGtek/h9Zqc8nnJNPByXZ\nCexsb/8zyeFJjucMLgW+M+lBnCOci9Oci4GJz8Mvnip87F2THAYsci7ysbH3+zOjNFrpEJgBNgy9\nv6zV/R9VdR9w30oNajGSTFfVlkmP41zgXJzmXAw4D6ed63Ox0vcEvgZsSnJFkh8FdgD7VngMkqRm\nRc8Equpkkj8A/hFYA3y6qg6u5BgkSaet+D2BqnoYeHil97sMzunLVSvMuTjNuRhwHk47p+ciVTXp\nMUiSJsSvjZCkjhkCZ5HkkiSPJnmuvV58hnZn/SqMJB9OUkkuXf5RL49x5yLJJ5J8M8kzSb6Q5E0r\nN/rxjfBvnCR3t/XPJHnHqH1Xm8XORZINSb6S5FCSg0k+uPKjX1rj/Fy09WuS/FuSL67cqGepKpcz\nLMDHgV2tvAv42Bxt1gDfAt4C/CjwdWDz0PoNDG6EvwhcOuljmtRcAL8OnNfKH5ur/7m6zPdv3Nrc\nCDwCBLgWeHzUvqtpGXMu1gHvaOU3Av/e61wMrf9D4LPAFyd1HJ4JnN12YE8r7wFunqPNfF+F8Ung\nj4DVfvNlrLmoqn+qqpOt3WMMPiOyWozydSfbgQdq4DHgTUnWjdh3NVn0XFTVsap6CqCqvg88y+Bb\nBFarcX4uSHIZcBPwtys56NkMgbNbW1XHWvllYO0cbeb6Koz1AEm2AzNV9fVlHeXKGGsuZvk9Br8d\nrRajHNeZ2ow6J6vFOHPxv5JsBN4OPL7kI1w5487FXzL4BfEHyzXAUZyTXxuxkpJ8GfjpOVZ9dPhN\nVVWSkX+bT/LjwEcYXAZZFZZrLmbt46PASeAzi+mv1S/JG4DPAR+qqlcnPZ5JSPIu4HhVPZnkukmO\npfsQqKpfPdO6JK+cOo1tp3DH52h2pq/C+FngCuDrSU7VP5Vka1W9vGQHsISWcS5ObeN3gXcB11e7\nILpKjPJ1J2dqc/4IfVeTceaCJOczCIDPVNXnl3GcK2Gcufht4DeT3Aj8GHBRkr+rqt9ZxvHObdI3\nV87lBfgE//dm6MfnaHMe8G0G/+Gfujl05RztXmB13xgeay6AbcAhYGrSx7KIY5/335jBtd3hG4BP\nLOTnY7UsY85FgAeAv5z0cUx6Lma1uY4J3hie+ESeywvwk8B+4Dngy8Alrf7NwMND7W5k8KTDt4CP\nnmFbqz0ExpoL4AiDa6NPt+WvJ31MCzz+Hzou4P3A+1s5DP5g0reAA8CWhfx8rKZlsXMB/BKDBySe\nGfo5uHHSxzOpn4uhbUw0BPzEsCR1zKeDJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEg\nSR37H/rJEtUmR00VAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x203c5190710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot null distribution and line at our observed differece\n",
    "plt.hist(null_vals)\n",
    "plt.axvline(x=obs_diff, color='red');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0044000000000000003"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute p-value for (H0: diff = 0)..and.. >, <... follows H1? \n",
    "(null_vals > obs_diff).mean()\n",
    "\n",
    "# So reject H0..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}