find all duplicate rows in a pandas dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import collections\n",
    "np.random.seed(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(9,3),\n",
    "                     index=['one', 'two', 'three', 'four', 'five', 'sex', 'seven', 'eight', 'nine'],\n",
    "                     columns=list('abc'))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def df_X(x):\n",
    "    df[x] = [0 if i % 2 == 0 else 7 for i in range(0, len(df['c']))]\n",
    "    return df[x]\n",
    "df_X('new')\n",
    "df = df.replace('N/A',np.nan) #  replace all the N/A values with NaN in columns\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>1.764052</td>\n",
       "      <td>0.400157</td>\n",
       "      <td>0.978738</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seven</th>\n",
       "      <td>0.313068</td>\n",
       "      <td>-0.854096</td>\n",
       "      <td>-2.552990</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace a few cell with another values\n",
    "df.iloc[0, 3] = 10\n",
    "df.iloc[1, 3] = 20\n",
    "df.iloc[6, 3] = 30\n",
    "df.iloc[8, 3] = 20\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find duplicates in the column\n",
    "df1 = df[df.duplicated('new',keep=False)] # or df1 = df[df.new.duplicated(keep=False)]\n",
    "df1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[20, 0, 7]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if you have not faced this task before, maybe you would solve the task this way like I did :)\n",
    "\n",
    "l1 = [item for item, count in collections.Counter(df['new']).items() if count > 1]\n",
    "l1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>two</th>\n",
       "      <td>2.240893</td>\n",
       "      <td>1.867558</td>\n",
       "      <td>-0.977278</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nine</th>\n",
       "      <td>2.269755</td>\n",
       "      <td>-1.454366</td>\n",
       "      <td>0.045759</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>three</th>\n",
       "      <td>0.950088</td>\n",
       "      <td>-0.151357</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>five</th>\n",
       "      <td>0.761038</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.443863</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>four</th>\n",
       "      <td>0.410599</td>\n",
       "      <td>0.144044</td>\n",
       "      <td>1.454274</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0.333674</td>\n",
       "      <td>1.494079</td>\n",
       "      <td>-0.205158</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eight</th>\n",
       "      <td>0.653619</td>\n",
       "      <td>0.864436</td>\n",
       "      <td>-0.742165</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "list_df = [df.loc[df['new'] == i] for i in l1] # if i in df['new'].values] # list of DataFrames\\n\",\n",
    "pd.concat(list_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "new\n0          [three, five]\n7     [four, sex, eight]\n10                 [one]\n20           [two, nine]\n30               [seven]\ndtype: object"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('new').apply(lambda x: list(x.index), )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "new\n10          [one]\n20    [two, nine]\ndtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.a > 1].groupby('new').apply(lambda x: list(x.index), )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}