stratification.html


<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>scikit-multilearn: Multi-Label Classification in Python &#8212; Multi-Label Classification for Python</title>
    <link rel="stylesheet" href="_static/" type="text/css" />
    <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
    <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
    <script type="text/javascript" src="_static/jquery.js"></script>
    <script type="text/javascript" src="_static/underscore.js"></script>
    <script type="text/javascript" src="_static/doctools.js"></script>
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="The scikit-multilearn Team" href="authors.html" />
    <link rel="prev" title="6. Multi-label embedding-based classification" href="multilabelembeddings.html" />
<meta content="True" name="HandheldFriendly">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0">
<meta name="twitter:card" content="summary">
<meta name="twitter:site" content="@scikitml">
<meta name="twitter:title" content="scikit-multilearn">
<meta name="twitter:description" content="A native Python implementation of a variety of multi-label classification algorithms. Includes a Meka, MULAN, Weka wrapper. BSD licensed.">
<meta name="keywords" content="scikit-multilearn, multi-label classification, clustering, python, machinelearning">
<meta property="og:title" content="scikit-multilearn | Multi-label classification package for python" />
<meta property="og:description" content="A native Python implementation of a variety of multi-label classification algorithms. Includes a Meka, MULAN, Weka wrapper. BSD licensed." />
<!-- Compiled and minified CSS -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0-rc.2/css/materialize.min.css">
<link rel="stylesheet" href="/_static/custom.css">
<link href="https://fonts.googleapis.com/css?family=IBM+Plex+Mono|IBM+Plex+Sans|IBM+Plex+Sans+Condensed|IBM+Plex+Serif" rel="stylesheet">
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.2.0/css/all.css" integrity="sha384-hWVjflwFxL6sNzntih27bfxkr27PmbbK/iSvJ+a4+0owXq79v+lsFkW54bOGbiDQ" crossorigin="anonymous">


<!-- Compiled and minified JavaScript -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0-rc.2/js/materialize.min.js"></script>

<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-51136636-1"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());

  gtag('config', 'UA-51136636-1');
</script>


  </head><body>
<div class="navbar-fixed">

  <nav>
    <div class="nav-wrapper container">
      <a href="index.html" class="brand-logo">scikit-multilearn</a>
      <ul id="nav-mobile" class="right hide-on-med-and-down">
        <li><a href="userguide.html">User Guide</a></li>
        <li><a href="api/skmultilearn.html">Reference</a></li>
        <li><a href="https://github.com/scikit-multilearn/scikit-multilearn">Github</a></li>
        <li><a href="https://pypi.org/project/scikit-multilearn">PyPi</a></li>
        <li id="navbar-about"><a href="authors.html">About</a></li>
      </ul>
    </div>
  </nav>
</div>


<!-- this is a replacement -->

<div class="container">
  <div class="row">
    <!-- Table of contents -->
    <div class="col hide-on-small-only m3 xl2">
      <div class="toc-wrapper">
        <div style="height: 1px;">
          <ul class="section table-of-contents">
            <ul>
<li><a class="reference internal" href="#">7. Multi-label data stratification</a></li>
</ul>

          </ul>
        </div>
      </div>
    </div>
    <div class="main-text section col s12 m8 offset-m1 xl9 offset-xl3">

      
<style>
/* CSS for nbsphinx extension */

/* remove conflicting styling from Sphinx themes */
div.nbinput,
div.nbinput div.prompt,
div.nbinput div.input_area,
div.nbinput div[class*=highlight],
div.nbinput div[class*=highlight] pre,
div.nboutput,
div.nbinput div.prompt,
div.nbinput div.output_area,
div.nboutput div[class*=highlight],
div.nboutput div[class*=highlight] pre {
    background: none;
    border: none;
    padding: 0 0;
    margin: 0;
    box-shadow: none;
}

/* avoid gaps between output lines */
div.nboutput div[class*=highlight] pre {
    line-height: normal;
}

/* input/output containers */
div.nbinput,
div.nboutput {
    display: -webkit-flex;
    display: flex;
    align-items: flex-start;
    margin: 0;
    width: 100%;
}
@media (max-width: 540px) {
    div.nbinput,
    div.nboutput {
        flex-direction: column;
    }
}

/* input container */
div.nbinput {
    padding-top: 5px;
}

/* last container */
div.nblast {
    padding-bottom: 5px;
}

/* input prompt */
div.nbinput div.prompt pre {
    color: #303F9F;
}

/* output prompt */
div.nboutput div.prompt pre {
    color: #D84315;
}

/* all prompts */
div.nbinput div.prompt,
div.nboutput div.prompt {
    min-width: 9ex;
    padding-top: 0.4em;
    padding-right: 0.4em;
    text-align: right;
    flex: 0;
}
@media (max-width: 540px) {
    div.nbinput div.prompt,
    div.nboutput div.prompt {
        text-align: left;
        padding: 0.4em;
    }
    div.nboutput div.prompt.empty {
        padding: 0;
    }
}

/* disable scrollbars on prompts */
div.nbinput div.prompt pre,
div.nboutput div.prompt pre {
    overflow: hidden;
}

/* input/output area */
div.nbinput div.input_area,
div.nboutput div.output_area {
    padding: 0.4em;
    -webkit-flex: 1;
    flex: 1;
    overflow: auto;
}
@media (max-width: 540px) {
    div.nbinput div.input_area,
    div.nboutput div.output_area {
        width: 100%;
    }
}

/* input area */
div.nbinput div.input_area {
    border: 1px solid #cfcfcf;
    border-radius: 2px;
    background: #f7f7f7;
}

/* override MathJax center alignment in output cells */
div.nboutput div[class*=MathJax] {
    text-align: left !important;
}

/* override sphinx.ext.pngmath center alignment in output cells */
div.nboutput div.math p {
    text-align: left;
}

/* standard error */
div.nboutput div.output_area.stderr {
    background: #fdd;
}

/* ANSI colors */
.ansi-black-fg { color: #3E424D; }
.ansi-black-bg { background-color: #3E424D; }
.ansi-black-intense-fg { color: #282C36; }
.ansi-black-intense-bg { background-color: #282C36; }
.ansi-red-fg { color: #E75C58; }
.ansi-red-bg { background-color: #E75C58; }
.ansi-red-intense-fg { color: #B22B31; }
.ansi-red-intense-bg { background-color: #B22B31; }
.ansi-green-fg { color: #00A250; }
.ansi-green-bg { background-color: #00A250; }
.ansi-green-intense-fg { color: #007427; }
.ansi-green-intense-bg { background-color: #007427; }
.ansi-yellow-fg { color: #DDB62B; }
.ansi-yellow-bg { background-color: #DDB62B; }
.ansi-yellow-intense-fg { color: #B27D12; }
.ansi-yellow-intense-bg { background-color: #B27D12; }
.ansi-blue-fg { color: #208FFB; }
.ansi-blue-bg { background-color: #208FFB; }
.ansi-blue-intense-fg { color: #0065CA; }
.ansi-blue-intense-bg { background-color: #0065CA; }
.ansi-magenta-fg { color: #D160C4; }
.ansi-magenta-bg { background-color: #D160C4; }
.ansi-magenta-intense-fg { color: #A03196; }
.ansi-magenta-intense-bg { background-color: #A03196; }
.ansi-cyan-fg { color: #60C6C8; }
.ansi-cyan-bg { background-color: #60C6C8; }
.ansi-cyan-intense-fg { color: #258F8F; }
.ansi-cyan-intense-bg { background-color: #258F8F; }
.ansi-white-fg { color: #C5C1B4; }
.ansi-white-bg { background-color: #C5C1B4; }
.ansi-white-intense-fg { color: #A1A6B2; }
.ansi-white-intense-bg { background-color: #A1A6B2; }

.ansi-default-inverse-fg { color: #FFFFFF; }
.ansi-default-inverse-bg { background-color: #000000; }

.ansi-bold { font-weight: bold; }
.ansi-underline { text-decoration: underline; }
</style>
<div class="section" id="Multi-label-data-stratification">
<h1>7. Multi-label data stratification<a class="headerlink" href="#Multi-label-data-stratification" title="Permalink to this headline">¶</a></h1>
<p>With the development of more complex multi-label transformation methods
the community realizes how much the quality of classification depends on
how the data is split into train/test sets or into folds for parameter
estimation. More questions appear on stackoverflow or
<a class="reference external" href="https://datascience.stackexchange.com/questions/33076/how-can-i-perform-stratified-sampling-for-multi-label-multi-class-classification">crossvalidated</a>
concerning methods for multi-label stratification.</p>
<p>For many reasons, described
<a class="reference external" href="http://lpis.csd.auth.gr/publications/sechidis-ecmlpkdd-2011.pdf">here</a>
and <a class="reference external" href="http://proceedings.mlr.press/v74/szyma%C5%84ski17a.html">here</a>
traditional single-label approaches to stratifying data fail to provide
balanced data set divisions which prevents classifiers from generalizing
information.</p>
<p>Some train/test splits don’t include evidence for a given label at all
in the train set. others disproportionately put even as much as 70% of
label pair evidence in the test set, leaving the train set without
proper evidence for generalizing conditional probabilities for label
relations.</p>
<p>You can also watch a great video presentation from ECML 2011 which
explains this in depth:</p>
  <blockquote>

On the Stratification of Multi-Label Data Grigorios Tsoumakas</blockquote><p>Scikit-multilearn provides an implementation of iterative stratification
which aims to provide well-balanced distribution of evidence of label
relations up to a given order. To see what it means, let’s load up some
data. We’ll be using the scene data set, both in divided and undivided
variants, to illustrate the problem.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [263]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">skmultilearn.dataset</span> <span class="kn">import</span> <span class="n">load_dataset</span>
<span class="n">X</span><span class="p">,</span><span class="n">y</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">load_dataset</span><span class="p">(</span><span class="s1">&#39;scene&#39;</span><span class="p">,</span> <span class="s1">&#39;undivided&#39;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
scene:undivided - exists, not redownloading
</pre></div></div>
</div>
<p>Let’s look at how many examples are available per label combination:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [264]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">skmultilearn.model_selection.measures</span> <span class="kn">import</span> <span class="n">get_combination_wise_output_matrix</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [265]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">Counter</span><span class="p">(</span><span class="n">combination</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">get_combination_wise_output_matrix</span><span class="p">(</span><span class="n">y</span><span class="o">.</span><span class="n">A</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="k">for</span> <span class="n">combination</span> <span class="ow">in</span> <span class="n">row</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>Out[265]:
</pre></div>
</div>
<div class="output_area highlight-none notranslate"><div class="highlight"><pre>
<span></span>Counter({(0, 0): 427,
         (0, 3): 1,
         (0, 4): 38,
         (0, 5): 19,
         (1, 1): 364,
         (2, 2): 397,
         (2, 3): 24,
         (2, 4): 14,
         (3, 3): 433,
         (3, 4): 76,
         (3, 5): 6,
         (4, 4): 533,
         (4, 5): 1,
         (5, 5): 431})
</pre></div>
</div>
</div>
<p>Let’s load up the original division, to see how the set was split into
train/test data in 2004, before multi-label stratification methods
appeared.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [266]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">_</span><span class="p">,</span> <span class="n">original_y_train</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">load_dataset</span><span class="p">(</span><span class="s1">&#39;scene&#39;</span><span class="p">,</span> <span class="s1">&#39;train&#39;</span><span class="p">)</span>
<span class="n">_</span><span class="p">,</span> <span class="n">original_y_test</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">load_dataset</span><span class="p">(</span><span class="s1">&#39;scene&#39;</span><span class="p">,</span> <span class="s1">&#39;test&#39;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
scene:train - exists, not redownloading
scene:test - exists, not redownloading
</pre></div></div>
</div>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [267]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [268]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span>
    <span class="s1">&#39;train&#39;</span><span class="p">:</span> <span class="n">Counter</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">combination</span><span class="p">)</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">get_combination_wise_output_matrix</span><span class="p">(</span><span class="n">original_y_train</span><span class="o">.</span><span class="n">A</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="k">for</span> <span class="n">combination</span> <span class="ow">in</span> <span class="n">row</span><span class="p">),</span>
    <span class="s1">&#39;test&#39;</span> <span class="p">:</span> <span class="n">Counter</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">combination</span><span class="p">)</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">get_combination_wise_output_matrix</span><span class="p">(</span><span class="n">original_y_test</span><span class="o">.</span><span class="n">A</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="k">for</span> <span class="n">combination</span> <span class="ow">in</span> <span class="n">row</span><span class="p">)</span>
<span class="p">})</span><span class="o">.</span><span class="n">T</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mf">0.0</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>Out[268]:
</pre></div>
</div>
<div class="output_area docutils container">
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>(0, 0)</th>
      <th>(0, 3)</th>
      <th>(0, 4)</th>
      <th>(0, 5)</th>
      <th>(1, 1)</th>
      <th>(2, 2)</th>
      <th>(2, 3)</th>
      <th>(2, 4)</th>
      <th>(3, 3)</th>
      <th>(3, 4)</th>
      <th>(3, 5)</th>
      <th>(4, 4)</th>
      <th>(4, 5)</th>
      <th>(5, 5)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>test</th>
      <td>200.0</td>
      <td>1.0</td>
      <td>17.0</td>
      <td>7.0</td>
      <td>199.0</td>
      <td>200.0</td>
      <td>16.0</td>
      <td>8.0</td>
      <td>237.0</td>
      <td>49.0</td>
      <td>5.0</td>
      <td>256.0</td>
      <td>0.0</td>
      <td>207.0</td>
    </tr>
    <tr>
      <th>train</th>
      <td>227.0</td>
      <td>0.0</td>
      <td>21.0</td>
      <td>12.0</td>
      <td>165.0</td>
      <td>197.0</td>
      <td>8.0</td>
      <td>6.0</td>
      <td>196.0</td>
      <td>27.0</td>
      <td>1.0</td>
      <td>277.0</td>
      <td>1.0</td>
      <td>224.0</td>
    </tr>
  </tbody>
</table>
</div></div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [269]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">original_y_train</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">original_y_test</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>Out[269]:
</pre></div>
</div>
<div class="output_area highlight-none notranslate"><div class="highlight"><pre>
<span></span>(1211, 1196)
</pre></div>
</div>
</div>
<p>We can see that the split size is nearly identical, yet some label
combination evidence is well balanced between the splits. While this is
a toy case on a small data set, such phenomena are common in larger
datasets. We would like to fix this.</p>
<p>Let’s load the iterative stratifier and divided the set again.</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [270]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">skmultilearn.model_selection</span> <span class="kn">import</span> <span class="n">iterative_train_test_split</span>
</pre></div>
</div>
</div>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [278]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">iterative_train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [279]:
</pre></div>
</div>
<div class="input_area highlight-ipython2 notranslate"><div class="highlight"><pre>
<span></span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span>
    <span class="s1">&#39;train&#39;</span><span class="p">:</span> <span class="n">Counter</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">combination</span><span class="p">)</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">get_combination_wise_output_matrix</span><span class="p">(</span><span class="n">y_train</span><span class="o">.</span><span class="n">A</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="k">for</span> <span class="n">combination</span> <span class="ow">in</span> <span class="n">row</span><span class="p">),</span>
    <span class="s1">&#39;test&#39;</span> <span class="p">:</span> <span class="n">Counter</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">combination</span><span class="p">)</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">get_combination_wise_output_matrix</span><span class="p">(</span><span class="n">y_test</span><span class="o">.</span><span class="n">A</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="k">for</span> <span class="n">combination</span> <span class="ow">in</span> <span class="n">row</span><span class="p">)</span>
<span class="p">})</span><span class="o">.</span><span class="n">T</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mf">0.0</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>Out[279]:
</pre></div>
</div>
<div class="output_area docutils container">
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>(0, 0)</th>
      <th>(0, 3)</th>
      <th>(0, 4)</th>
      <th>(0, 5)</th>
      <th>(1, 1)</th>
      <th>(2, 2)</th>
      <th>(2, 3)</th>
      <th>(2, 4)</th>
      <th>(3, 3)</th>
      <th>(3, 4)</th>
      <th>(3, 5)</th>
      <th>(4, 4)</th>
      <th>(4, 5)</th>
      <th>(5, 5)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>test</th>
      <td>213.0</td>
      <td>0.0</td>
      <td>19.0</td>
      <td>9.0</td>
      <td>182.0</td>
      <td>199.0</td>
      <td>12.0</td>
      <td>7.0</td>
      <td>217.0</td>
      <td>38.0</td>
      <td>3.0</td>
      <td>267.0</td>
      <td>1.0</td>
      <td>215.0</td>
    </tr>
    <tr>
      <th>train</th>
      <td>214.0</td>
      <td>1.0</td>
      <td>19.0</td>
      <td>10.0</td>
      <td>182.0</td>
      <td>198.0</td>
      <td>12.0</td>
      <td>7.0</td>
      <td>216.0</td>
      <td>38.0</td>
      <td>3.0</td>
      <td>266.0</td>
      <td>0.0</td>
      <td>216.0</td>
    </tr>
  </tbody>
</table>
</div></div>
</div>
<p>We can see that the new division is much more balanced.</p>
</div>


    </div>
  </div>

</div>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="right" >
          <a href="authors.html" title="The scikit-multilearn Team"
             accesskey="N">next</a> |</li>
        <li class="right" >
          <a href="multilabelembeddings.html" title="6. Multi-label embedding-based classification"
             accesskey="P">previous</a> |</li>
        <li class="nav-item nav-item-0"><a href="index.html">scikit-multilearn</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="userguide.html" accesskey="U">User Guide</a> &#187;</li> 
      </ul>
    </div>
<footer class="page-footer blue-grey darken-4">
  <div class="container">
    <div class="row  ">
      <div class="col l6 s12">
        <h5 class="white-text">Cite US!</h5>
        <p>If you use scikit-multilearn in your research and publish it, please consider citing us, it will help us get funding for making the library better. The paper is available on <a href="https://arxiv.org/abs/1702.01460">arXiv</a>, to cite it try the Bibtex code on the right.</p>
      </div>
      <div class="col l4 s12">
        <pre><code>
        
        @ARTICLE{2017arXiv170201460S,
          author = {{Szyma{\'n}ski}, P. and {Kajdanowicz}, T.},
          title = "{A scikit-based Python environment for performing multi-label classification}",
          journal = {ArXiv e-prints},
          archivePrefix = "arXiv",
          eprint = {1702.01460},
          primaryClass = "cs.LG",
          keywords = {Computer Science - Learning, Computer Science - Mathematical Software},
          year = 2017,
          month = feb,
        }
        
      </code></pre>
      </div>
    </div>
  </div>
  <div class="footer-copyright blue-grey darken-4">
    <div class="container">
        Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.8.2.
      <span style="padding-left: 5ex;">
        <a href="_sources/stratification.ipynb.txt"
         rel="nofollow">Show this page source</a>
      </span>
    </div>
  </div>
</footer>
<!-- Place this tag in your head or just before your close body tag. -->
<script async defer src="https://buttons.github.io/buttons.js"></script>
  </body>
</html>