utility_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some useful stuff for copies and pastes\n",
    "\n",
    "1. Reading and Writing to `csv`\n",
    "1. Use the `logging` module\n",
    "1. Load all `csv` files in a directory into a dataframe\n",
    "1. Exclusive deduplication"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and Writing to csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other encodings include `utf-8`, `unicode` and many others. `latin_1` seems to work well in Windows. Especially if you have an excel sheet that you save as csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.403026</td>\n",
       "      <td>0.559875</td>\n",
       "      <td>0.755273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.795062</td>\n",
       "      <td>-0.195291</td>\n",
       "      <td>-0.550756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2.151132</td>\n",
       "      <td>-0.915356</td>\n",
       "      <td>0.848399</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c\n",
       "0 -0.403026  0.559875  0.755273\n",
       "1 -1.795062 -0.195291 -0.550756\n",
       "2 -2.151132 -0.915356  0.848399"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/file1.csv', encoding='latin_1')\n",
    "\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('data/file2.csv', index=False, quoting=csv.QUOTE_ALL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"a\",\"b\",\"c\"\n",
      "\"-0.4030264624911308\",\"0.5598745496930929\",\"0.7552732765837198\"\n",
      "\"-1.7950623985555558\",\"-0.19529096820893035\",\"-0.5507564378674887\"\n",
      "\"-2.151131926398676\",\"-0.9153555453197798\",\"0.8483986080862593\"\n"
     ]
    }
   ],
   "source": [
    "!head -n 4 data/file2.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use `logging`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Something failed.\n",
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-5-ff92aff4b25d>\", line 5, in <module>\n",
      "    1/0\n",
      "ZeroDivisionError: division by zero\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "\n",
    "try:\n",
    "    1/0\n",
    "except:\n",
    "    logger.exception('Something failed.')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    logging.basicConfig(level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load all `csv` files in a directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "class data_loader:\n",
    "    def __init__(self):\n",
    "        self.path = 'data/'\n",
    "        self.lookupid_col = 'CONSTITUENTLOOKUPID'\n",
    "        self.data = None\n",
    "    \n",
    "    def read_files(self, path=None):\n",
    "        if not path:\n",
    "            path = self.path\n",
    "        \n",
    "        data_frames = []\n",
    "        \n",
    "        for i in os.listdir(path):\n",
    "            _df = pd.read_csv(f'{path}/{i}', encoding='latin_1')\n",
    "            data_frames.append(_df)\n",
    "        \n",
    "        return pd.concat(data_frames, axis=0)\n",
    "    \n",
    "    def ten_digit_ids(self, df):\n",
    "        def zerominator(x):\n",
    "            new = '0000000000' + str(x)\n",
    "            return new[-10:]\n",
    "        \n",
    "        df[self.lookupid_col] = df[self.lookupid_col].apply(zerominator)\n",
    "        \n",
    "        return df\n",
    "    \n",
    "    def get(self):\n",
    "        self.data = (\n",
    "            self.read_files()\n",
    "            .pipe(self.ten_digit_ids)\n",
    "        )\n",
    "        return self.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exclusive deduplication\n",
    "\n",
    "Sometimes I need to check a list of IDs for the ones in either list that only appear in one list. Normal deduplication functions leave in one copy of the duplicate value. I want no copies of the duplicate values. I just want the values that only appear a single time in the set of values that includes both lists.\n",
    "\n",
    "Behold. In Python, the answer is always a list comprehension."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dedupe_exclusive(x, y):\n",
    "    xs = [i for i in x if (i not in y)]\n",
    "    ys = [i for i in y if (i not in x)]\n",
    "    return xs + ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4, 5, 6, 7, 8])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.arange(1, 9)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3,  4,  5,  6,  7,  8,  9, 10])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.arange(3, 11)\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 9, 10]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dedupe_exclusive(x, y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}