{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Important three techniques to improve machine learning model performance with imbalance datasets\n",
    "\n",
    "Author: Sabber Ahamed\n",
    "\n",
    "Objective: The primary goal of this code is to develop a machine learning model that can classify the labels properly and address dataset imbalance. The dataset has three classes (1, 2, and 3). There are three problems in the challenge. In each problem, there is some specific task that needs to be done. In the following subsections, I describe three techniques I used to overcome the data imbalance problem.\n",
    "\n",
    "Codes and libraries: This project requires Python 3.\n",
    "\n",
    "The following Python libraries are also required:\n",
    "\n",
    "- matplotlib\n",
    "- NumPy\n",
    "- Pandas\n",
    "- scikit-learn\n",
    "- seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'seaborn'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-1d78f3134027>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m## Plotting libraries\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'seaborn'"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "\n",
    "## Plotting libraries\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "## Sklearn Libraries\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import f1_score, confusion_matrix, roc_curve, auc, \\\n",
    "            classification_report, recall_score, precision_recall_curve\n",
    "\n",
    "# Define random state\n",
    "random_state = 2018\n",
    "np.random.seed(random_state)\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# latex parameter\n",
    "font = {\n",
    "    'family': 'serif', \n",
    "    'serif': ['Computer Modern Roman'],\n",
    "    'weight' : 'regular',\n",
    "    'size'   : 14\n",
    "    }\n",
    "\n",
    "plt.rc('font', **font)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./train.csv')\n",
    "test = pd.read_csv('./test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploratory data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38829, 17)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16641, 16)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>y1</th>\n",
       "      <th>y2</th>\n",
       "      <th>y3</th>\n",
       "      <th>y4</th>\n",
       "      <th>y5</th>\n",
       "      <th>y6</th>\n",
       "      <th>z1</th>\n",
       "      <th>z2</th>\n",
       "      <th>z3</th>\n",
       "      <th>z4</th>\n",
       "      <th>z5</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.874424</td>\n",
       "      <td>0.759918</td>\n",
       "      <td>0.959276</td>\n",
       "      <td>0.821806</td>\n",
       "      <td>0.722757</td>\n",
       "      <td>0.064458</td>\n",
       "      <td>0.399405</td>\n",
       "      <td>0.400940</td>\n",
       "      <td>0.665386</td>\n",
       "      <td>0.749402</td>\n",
       "      <td>0.962733</td>\n",
       "      <td>0.059650</td>\n",
       "      <td>0.392512</td>\n",
       "      <td>0.899956</td>\n",
       "      <td>0.377613</td>\n",
       "      <td>0.157010</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.169331</td>\n",
       "      <td>0.111202</td>\n",
       "      <td>0.604827</td>\n",
       "      <td>0.175265</td>\n",
       "      <td>0.122169</td>\n",
       "      <td>0.327532</td>\n",
       "      <td>0.023786</td>\n",
       "      <td>0.161728</td>\n",
       "      <td>0.539730</td>\n",
       "      <td>0.487391</td>\n",
       "      <td>0.663999</td>\n",
       "      <td>0.838974</td>\n",
       "      <td>0.258899</td>\n",
       "      <td>0.056143</td>\n",
       "      <td>0.002095</td>\n",
       "      <td>0.000144</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.329992</td>\n",
       "      <td>0.173176</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.353518</td>\n",
       "      <td>0.156055</td>\n",
       "      <td>0.260328</td>\n",
       "      <td>0.091179</td>\n",
       "      <td>0.375606</td>\n",
       "      <td>0.455247</td>\n",
       "      <td>0.500023</td>\n",
       "      <td>0.728747</td>\n",
       "      <td>0.484457</td>\n",
       "      <td>0.633725</td>\n",
       "      <td>0.348351</td>\n",
       "      <td>0.036536</td>\n",
       "      <td>0.004947</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.176104</td>\n",
       "      <td>0.108428</td>\n",
       "      <td>0.579186</td>\n",
       "      <td>0.215494</td>\n",
       "      <td>0.110754</td>\n",
       "      <td>0.415192</td>\n",
       "      <td>0.015857</td>\n",
       "      <td>0.154121</td>\n",
       "      <td>0.526295</td>\n",
       "      <td>0.434409</td>\n",
       "      <td>0.592285</td>\n",
       "      <td>0.823823</td>\n",
       "      <td>0.280367</td>\n",
       "      <td>0.064162</td>\n",
       "      <td>0.002547</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.971552</td>\n",
       "      <td>0.703186</td>\n",
       "      <td>0.957768</td>\n",
       "      <td>0.818220</td>\n",
       "      <td>0.680667</td>\n",
       "      <td>0.046714</td>\n",
       "      <td>0.329039</td>\n",
       "      <td>0.304118</td>\n",
       "      <td>0.697514</td>\n",
       "      <td>0.660573</td>\n",
       "      <td>0.979338</td>\n",
       "      <td>0.082224</td>\n",
       "      <td>0.459216</td>\n",
       "      <td>0.896192</td>\n",
       "      <td>0.325817</td>\n",
       "      <td>0.118351</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         x1        x2        x3        x4        x5        y1        y2  \\\n",
       "0  0.874424  0.759918  0.959276  0.821806  0.722757  0.064458  0.399405   \n",
       "1  0.169331  0.111202  0.604827  0.175265  0.122169  0.327532  0.023786   \n",
       "2  0.329992  0.173176  0.692308  0.353518  0.156055  0.260328  0.091179   \n",
       "3  0.176104  0.108428  0.579186  0.215494  0.110754  0.415192  0.015857   \n",
       "4  0.971552  0.703186  0.957768  0.818220  0.680667  0.046714  0.329039   \n",
       "\n",
       "         y3        y4        y5        y6        z1        z2        z3  \\\n",
       "0  0.400940  0.665386  0.749402  0.962733  0.059650  0.392512  0.899956   \n",
       "1  0.161728  0.539730  0.487391  0.663999  0.838974  0.258899  0.056143   \n",
       "2  0.375606  0.455247  0.500023  0.728747  0.484457  0.633725  0.348351   \n",
       "3  0.154121  0.526295  0.434409  0.592285  0.823823  0.280367  0.064162   \n",
       "4  0.304118  0.697514  0.660573  0.979338  0.082224  0.459216  0.896192   \n",
       "\n",
       "         z4        z5  label  \n",
       "0  0.377613  0.157010      1  \n",
       "1  0.002095  0.000144      1  \n",
       "2  0.036536  0.004947      1  \n",
       "3  0.002547  0.000182      1  \n",
       "4  0.325817  0.118351      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>y1</th>\n",
       "      <th>y2</th>\n",
       "      <th>y3</th>\n",
       "      <th>y4</th>\n",
       "      <th>y5</th>\n",
       "      <th>y6</th>\n",
       "      <th>z1</th>\n",
       "      <th>z2</th>\n",
       "      <th>z3</th>\n",
       "      <th>z4</th>\n",
       "      <th>z5</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>38829.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.452564</td>\n",
       "      <td>0.324257</td>\n",
       "      <td>0.753640</td>\n",
       "      <td>0.425769</td>\n",
       "      <td>0.311064</td>\n",
       "      <td>0.199044</td>\n",
       "      <td>0.163545</td>\n",
       "      <td>0.283753</td>\n",
       "      <td>0.559216</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.794910</td>\n",
       "      <td>0.420935</td>\n",
       "      <td>0.508013</td>\n",
       "      <td>4.703503e-01</td>\n",
       "      <td>1.284139e-01</td>\n",
       "      <td>3.757896e-02</td>\n",
       "      <td>1.054186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.243708</td>\n",
       "      <td>0.219627</td>\n",
       "      <td>0.156440</td>\n",
       "      <td>0.208532</td>\n",
       "      <td>0.209755</td>\n",
       "      <td>0.133914</td>\n",
       "      <td>0.129485</td>\n",
       "      <td>0.112639</td>\n",
       "      <td>0.094967</td>\n",
       "      <td>0.078400</td>\n",
       "      <td>0.146847</td>\n",
       "      <td>0.299677</td>\n",
       "      <td>0.206335</td>\n",
       "      <td>3.245707e-01</td>\n",
       "      <td>1.509215e-01</td>\n",
       "      <td>6.229685e-02</td>\n",
       "      <td>0.262132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.015714</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>0.152338</td>\n",
       "      <td>0.014287</td>\n",
       "      <td>0.010255</td>\n",
       "      <td>0.000374</td>\n",
       "      <td>0.002973</td>\n",
       "      <td>0.029058</td>\n",
       "      <td>0.254826</td>\n",
       "      <td>0.273939</td>\n",
       "      <td>0.386060</td>\n",
       "      <td>0.000813</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>9.265071e-09</td>\n",
       "      <td>1.756340e-12</td>\n",
       "      <td>4.764865e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.220130</td>\n",
       "      <td>0.123741</td>\n",
       "      <td>0.616893</td>\n",
       "      <td>0.243828</td>\n",
       "      <td>0.119672</td>\n",
       "      <td>0.079274</td>\n",
       "      <td>0.044599</td>\n",
       "      <td>0.212762</td>\n",
       "      <td>0.493038</td>\n",
       "      <td>0.494189</td>\n",
       "      <td>0.679581</td>\n",
       "      <td>0.152327</td>\n",
       "      <td>0.342202</td>\n",
       "      <td>1.132663e-01</td>\n",
       "      <td>5.486555e-03</td>\n",
       "      <td>4.072433e-04</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.423327</td>\n",
       "      <td>0.253957</td>\n",
       "      <td>0.779789</td>\n",
       "      <td>0.386694</td>\n",
       "      <td>0.255930</td>\n",
       "      <td>0.177742</td>\n",
       "      <td>0.135778</td>\n",
       "      <td>0.261159</td>\n",
       "      <td>0.541170</td>\n",
       "      <td>0.537088</td>\n",
       "      <td>0.805458</td>\n",
       "      <td>0.333305</td>\n",
       "      <td>0.554853</td>\n",
       "      <td>4.999816e-01</td>\n",
       "      <td>7.025290e-02</td>\n",
       "      <td>1.001907e-02</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.654701</td>\n",
       "      <td>0.486845</td>\n",
       "      <td>0.889894</td>\n",
       "      <td>0.601650</td>\n",
       "      <td>0.481184</td>\n",
       "      <td>0.295147</td>\n",
       "      <td>0.256690</td>\n",
       "      <td>0.331268</td>\n",
       "      <td>0.638947</td>\n",
       "      <td>0.576464</td>\n",
       "      <td>0.937122</td>\n",
       "      <td>0.724802</td>\n",
       "      <td>0.665629</td>\n",
       "      <td>7.853695e-01</td>\n",
       "      <td>2.037514e-01</td>\n",
       "      <td>4.835476e-02</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971634</td>\n",
       "      <td>0.989442</td>\n",
       "      <td>0.950625</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.592082</td>\n",
       "      <td>0.764123</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.766188</td>\n",
       "      <td>0.823532</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.947246</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>8.771210e-01</td>\n",
       "      <td>8.727469e-01</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 x1            x2            x3            x4            x5  \\\n",
       "count  38829.000000  38829.000000  38829.000000  38829.000000  38829.000000   \n",
       "mean       0.452564      0.324257      0.753640      0.425769      0.311064   \n",
       "std        0.243708      0.219627      0.156440      0.208532      0.209755   \n",
       "min        0.015714      0.014286      0.152338      0.014287      0.010255   \n",
       "25%        0.220130      0.123741      0.616893      0.243828      0.119672   \n",
       "50%        0.423327      0.253957      0.779789      0.386694      0.255930   \n",
       "75%        0.654701      0.486845      0.889894      0.601650      0.481184   \n",
       "max        1.000000      0.971634      0.989442      0.950625      1.000000   \n",
       "\n",
       "                 y1            y2            y3            y4            y5  \\\n",
       "count  38829.000000  38829.000000  38829.000000  38829.000000  38829.000000   \n",
       "mean       0.199044      0.163545      0.283753      0.559216      0.540541   \n",
       "std        0.133914      0.129485      0.112639      0.094967      0.078400   \n",
       "min        0.000374      0.002973      0.029058      0.254826      0.273939   \n",
       "25%        0.079274      0.044599      0.212762      0.493038      0.494189   \n",
       "50%        0.177742      0.135778      0.261159      0.541170      0.537088   \n",
       "75%        0.295147      0.256690      0.331268      0.638947      0.576464   \n",
       "max        0.592082      0.764123      1.000000      0.766188      0.823532   \n",
       "\n",
       "                 y6            z1            z2            z3            z4  \\\n",
       "count  38829.000000  38829.000000  38829.000000  3.882900e+04  3.882900e+04   \n",
       "mean       0.794910      0.420935      0.508013  4.703503e-01  1.284139e-01   \n",
       "std        0.146847      0.299677      0.206335  3.245707e-01  1.509215e-01   \n",
       "min        0.386060      0.000813      0.000005  9.265071e-09  1.756340e-12   \n",
       "25%        0.679581      0.152327      0.342202  1.132663e-01  5.486555e-03   \n",
       "50%        0.805458      0.333305      0.554853  4.999816e-01  7.025290e-02   \n",
       "75%        0.937122      0.724802      0.665629  7.853695e-01  2.037514e-01   \n",
       "max        0.999997      0.999998      0.947246  1.000000e+00  8.771210e-01   \n",
       "\n",
       "                 z5         label  \n",
       "count  3.882900e+04  38829.000000  \n",
       "mean   3.757896e-02      1.054186  \n",
       "std    6.229685e-02      0.262132  \n",
       "min    4.764865e-15      1.000000  \n",
       "25%    4.072433e-04      1.000000  \n",
       "50%    1.001907e-02      1.000000  \n",
       "75%    4.835476e-02      1.000000  \n",
       "max    8.727469e-01      3.000000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_count = train.groupby('label', as_index = False).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_name = list(label_count['label'])\n",
    "count = list(label_count['x1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[37064, 1426, 339]"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAFuCAYAAAAYrMtzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtcVHXi//H3YUZTGRBZWRNR8paFii6RboV20cLaXK3F\nvJTtqmklarpfFUVBDVctTSuvaZctbctAMy37WlFqml9sNVMxK1s1CCTIG4wXgjm/P3o4P1lBR2KY\n8Lyej4ePB3zmc+a8h4fKez7nzDmGaZqmAACA5fj5OgAAAPANSgAAABZFCQAAwKIoAQAAWBQlAAAA\ni6IEAABgUZQAoAYqLS3VK6+8ovvvv1+9evXSPffco9mzZ6u4uFiSNGHCBL300kvVlufo0aNq06aN\nJCk9PV3Tp0+/6PyNGzfqueeeK/ex87cfOHCg/vd///eyshQWFurhhx92f9+rVy+dPHnysp4DsAq7\nrwMAuHxTp07ViRMn9OqrryogIECnTp3S2LFjNWnSJM2ePdun2bp166Zu3bpddM6ePXt04sSJSm9/\nMSdOnNCePXvc37/zzjuVfi7gSkcJAGqYrKwsrVu3Tlu2bJHD4ZAk1atXT9OmTdMXX3xxwfy0tDSt\nXLlSP//8s06cOKGhQ4dqwIABys/PV0JCgo4dOyZJuvXWWzV69OgKx//bBx98oHnz5qlu3bpq166d\ne3z16tXasGGDXnjhBX3wwQdavHixDMOQzWbT+PHjVbt2bb355psqLS1VQECAwsPDlZaWptOnT8vh\ncOi+++5zby9JH374oZYuXaozZ86oZ8+eevzxx5Wdna2ePXu6X+/530+cOFFnzpxRr169tHr1akVE\nRGjbtm0KDg7WwoUL9d5778lms6l58+ZKSkpSSEiIBg4cqI4dO2rnzp3Kzc3VDTfcoKeeekp+fiyW\n4srG33Cghtm3b59atWrlLgDnhISE6K677ioz5nQ6lZqaqqVLl2rNmjWaN2+ee6XgrbfeUlhYmN5+\n+229/vrrOnz4sAoLCyscP19BQYESExM1f/58rV69Wk2aNCk369NPP60pU6Zo9erVeuKJJ5SRkaEO\nHTqoX79+uueeezRmzBhJ0oEDB7R8+XItX778gudwOp1666239NZbb2nt2rXatGnTRX8+M2fOVJ06\ndfTOO+/IZrO5x1etWqVPP/1UaWlpWrdunVq3bq0JEya4H//++++1fPlyrV27Vv/3f/+n7du3X3Q/\nwJWAlQCghvHz85PL5fJorr+/v5YsWaJNmzbp0KFD2r9/v06dOiVJ6tKli4YNG6bc3FzdfPPN+p//\n+R8FBARUOH6+HTt26Nprr1WrVq0kSX379tXcuXMv2P+f/vQnjRgxQrfeeqtuueUWDR06tNycbdq0\nuaDUnBMXFye73S6Hw6HY2Fh99tlnatmypUev/3ybN2/W/fffr3r16kmSHn74YS1ZssR9HsXtt98u\nPz8/ORwOhYeHV3i4AriSsBIA1DCRkZH6z3/+o6KiojLjeXl5GjZsmM6cOeMeO3LkiHr37q0ffvhB\nN9xwQ5ll/cjISKWnp6tv37764Ycf1KdPH+3cubPC8fMZhqHzbztit5f/fmLMmDF644031K5dO61e\nvVp9+/Ytt8Cc+8VcnvPfzZumKbvdfsH+f/755wq3P3/b87lcLpWUlLi/r1Onjvvr/35+4EpFCQBq\nmEaNGqlnz55KTEx0F4GioiJNnTpVQUFBZX6Z7d27V8HBwRo+fLi6dOmiTz75RNIvny6YM2eOFi1a\npO7du2vSpElq1aqVDh06VOH4+aKjo3XgwAHt379f0i/nAfy3kpIS3XHHHTp16pT69++vKVOm6Lvv\nvlNJSYlsNluZX8AXs2bNGpmmqRMnTuj9999X165dFRgYqJ9//lkHDhyQ9Mt5A+fY7XaVlpZe8Es8\nJiZGq1evdq+ELF++XDfeeKNq167tUQ7gSsThAKAGmjJlihYtWqR+/frJZrOpuLhY3bt318iRI8vM\nu+WWW5SWlqYePXqobt26ioyMVHBwsA4fPqy//vWvmjBhgu69917Vrl1bbdq00b333qsTJ06UO36+\n4OBgzZkzR2PHjlWtWrV04403XpDRbrcrMTFRY8eOdb97nzFjhmrXrq2bbrpJI0eOVK1atdS2bduL\nvtaAgADdf//9OnPmjB566CF17txZkjRu3DgNHTpUwcHB6tGjh3t+SEiIIiIidPfdd+uNN95wj8fF\nxSk3N1d9+vSRy+VSeHi45syZc9k/e+BKYnArYQAArInDAQAAWBQlAAAAi6IEAABgUZQAAAAsihIA\nAIBFWe4jgvn5hZeeBADAFSIkJKDCx1gJAADAoigBAABYFCUAAACLogQAAGBRlAAAACyKEgAAgEVR\nAgAAsChKAAAAFkUJAADAoigBAABYFCUAAACLogQAAGBRlruBUGXFxw9VXl6ur2PAixo1aqyFC5f5\nOgYAVBtKgIfy8nLldDp9HQNeRMkDYDWUgMvkZ79KVzlCfB0DVehsUb5cJWd9HQMAqh0l4DJd5QhR\n06g+vo6BKpS1M1Wnj2f7OgYAVDtODAQAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsihIAAIBFUQIA\nALAoSgAAABZFCQAAwKIoAQAAWFS1XTa4tLRUkydP1sGDB2UYhqZNm6aSkhI9+uijuuaaayRJ/fv3\n1z333KMFCxZo48aNstvtSkxMVGRkpA4fPqwJEybIMAy1bt1aU6ZMkZ+fX7lzAQDApVVbCfjkk08k\nSW+++aYyMjI0b9483XHHHRo0aJAGDx7snpeZmant27crNTVVubm5GjlypFatWqWZM2dq9OjR6ty5\ns5KTk5Wenq7Q0NBy5wIAgEurthLQvXt33XbbbZKknJwcBQYGau/evTp48KDS09MVHh6uxMRE7dix\nQzExMTIMQ6GhoSotLdXRo0eVmZmpTp06SZK6du2qrVu3qnnz5uXODQ4Orq6XBQBAjVWtdxG02+1K\nSEjQhx9+qOeff155eXnq06eP2rVrp8WLF2vhwoUKCAhQUFCQext/f38VFhbKNE0ZhlFmrKioqNy5\nFysBDRrUk91uu+zs5/aNK5dhGAoJCfB1DACoNtV+K+GnnnpKY8eO1QMPPKA333xTjRo1kiTdeeed\nSklJUbdu3eR0Ot3znU6nAgIC5OfnV2YsMDBQDoej3LkXc+zYqUrlNk2zUtuh5jBNU/n5hb6OAQBV\n6mJvbqrt0wFr1qzRCy+8IEmqW7euDMPQiBEjtHv3bknStm3b1LZtW0VFRWnLli1yuVzKycmRy+VS\ncHCwIiIilJGRIUnavHmzoqOjK5wLAAAurdpWAu666y5NnDhRDz74oEpKSpSYmKjGjRsrJSVFtWrV\nUsOGDZWSkiKHw6Ho6Gj17dtXLpdLycnJkqSEhAQlJSVp7ty5atGihWJjY2Wz2cqdCwAALs0wLbbO\nXdnl3ri4e+V0OlU3KExNo/pUcSr4UtbOVJ0+ni1/f3+lpb3r6zgAUKV+E4cDAADAbwslAAAAi6IE\nAABgUZQAAAAsihIAAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsihIA\nAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsihIAAIBFUQIAALAoSgAA\nABZFCQAAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsihIAAIBFUQIAALAoe3XtqLS0VJMnT9bBgwdl\nGIamTZumq666ShMmTJBhGGrdurWmTJkiPz8/LViwQBs3bpTdbldiYqIiIyN1+PBhj+cCAIBLq7YS\n8Mknn0iS3nzzTWVkZGjevHkyTVOjR49W586dlZycrPT0dIWGhmr79u1KTU1Vbm6uRo4cqVWrVmnm\nzJkezwUAAJdWbSWge/fuuu222yRJOTk5CgwM1GeffaZOnTpJkrp27aqtW7eqefPmiomJkWEYCg0N\nVWlpqY4eParMzEyP5wYHB1fXywIAoMaqthIgSXa7XQkJCfrwww/1/PPPa+vWrTIMQ5Lk7++vwsJC\nFRUVKSgoyL3NuXHTND2ee7ES0KBBPdnttsvOfm7fuHIZhqGQkABfxwCAalOtJUCSnnrqKY0dO1YP\nPPCAzp496x53Op0KDAyUw+GQ0+ksMx4QECA/Pz+P517MsWOnKpXbNM1KbYeawzRN5ecX+joGAFSp\ni725qbZPB6xZs0YvvPCCJKlu3boyDEPt2rVTRkaGJGnz5s2Kjo5WVFSUtmzZIpfLpZycHLlcLgUH\nBysiIsLjuQAA4NKqbSXgrrvu0sSJE/Xggw+qpKREiYmJatmypZKSkjR37ly1aNFCsbGxstlsio6O\nVt++feVyuZScnCxJSkhI8HguAAC4NMO02Dp3ZZd74+LuldPpVN2gMDWN6lPFqeBLWTtTdfp4tvz9\n/ZWW9q6v4wBAlfpNHA4AAAC/LZQAAAAsihIAAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAA\ni6IEAABgUZQAAAAsihIAAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAs\nihIAAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsyu7pxP379+ubb76R\ny+WSJJmmqeLiYu3Zs0fTp0/3WkAAAOAdHpWAl156SbNnz5afn59M05RhGHK5XDIMQ507d/Z2RgAA\n4AUeHQ54/fXXFR8fr927dys4OFiffPKJ1q9fr2uvvVZdu3b1dkYAAOAFHpWAH3/8Ub1795bdbtd1\n112n3bt3q0WLFpowYYJWrVrl7YwAAMALPCoBDodDZ8+elSRdc801+uabbyRJ4eHhysnJ8V46AADg\nNR6VgE6dOumZZ57Rjz/+qPbt22vDhg0qLCzUxx9/rPr1619y+59//lnjxo3TgAEDFBcXp/T0dO3b\nt09dunTRwIEDNXDgQK1fv16StGDBAsXFxalfv37avXu3JOnw4cPq37+/BgwYoClTprhPTixvLgAA\n8IxHJwaOHz9ejz/+uNavX68BAwbon//8pzp16iRJGjdu3CW3X7t2rYKCgjR79mwdP35cvXv3Vnx8\nvAYNGqTBgwe752VmZmr79u1KTU1Vbm6uRo4cqVWrVmnmzJkaPXq0OnfurOTkZKWnpys0NLTcuQAA\nwDMelYCwsDCtW7dOZ8+eVe3atfWvf/1LW7Zs0dVXX63IyMhLbt+jRw/FxsZK+uWjhTabTXv37tXB\ngweVnp6u8PBwJSYmaseOHYqJiZFhGAoNDVVpaamOHj2qzMxMd+no2rWrtm7dqubNm5c7Nzg4+Ff8\nOAAAsA6PSkC3bt20atUqBQUFSZLq1aunu+66Sz/++KNuuukmbdu27aLb+/v7S5KKioo0atQojR49\nWsXFxerTp4/atWunxYsXa+HChQoICHDv49x2hYWF7o8lnj9WVFRU7txLlYAGDerJbrd58rLLOLd/\nXLkMw1BISICvYwBAtamwBGzatEl79uyRJP3www9aunSp6tWrV2bOoUOHVFpa6tGOcnNzFR8frwED\nBqhnz546efKkAgMDJUl33nmnUlJS1K1bNzmdTvc2TqdTAQEB8vPzKzMWGBgoh8NR7txLOXbslEd5\n/5tpmpXaDjWHaZrKzy/0dQwAqFIXe3NT4YmB5w4BrF27VpK0YcMGrV271v1n3bp1+s9//qPJkydf\nMkBBQYEGDx6scePGKS4uTpI0ZMgQ98l827ZtU9u2bRUVFaUtW7bI5XIpJydHLpdLwcHBioiIUEZG\nhiRp8+bNio6OrnAuAADwTIUrAS1bttSGDRskSQMHDtSCBQs8+iRAeZYsWaKTJ09q0aJFWrRokSRp\nwoQJmjFjhmrVqqWGDRsqJSVFDodD0dHR6tu3r1wul5KTkyVJCQkJSkpK0ty5c9WiRQvFxsbKZrOV\nOxcAAHjGMH/FOve5ewfccMMNVZnJqyq73BsXd6+cTqfqBoWpaVSfKk4FX8ramarTx7Pl7++vtLR3\nfR0HAKrUxQ4HeHRi4N69e5WUlFTmBkLn++qrryqfDgAA+IRHFwv6xz/+oauuukpPPvmkatWqpalT\np+qRRx5R7dq1NW/ePG9nBAAAXuDRSsBXX32lFStWqF27dnrrrbfUvHlz9e3bV7///e/1xhtvqEeP\nHt7OCQAAqphHKwGmabrPvA8PD3ffO+D222/X/v37vZcOAAB4jUcloHXr1tq0aZMkqVWrVtq5c6ck\n6aeffir3HAEAAPDb59HhgKFDh2rMmDGy2Wz605/+pAULFmj48OHav3+/Onfu7O2MAADACzxaCYiN\njdXKlSsVGRmpJk2a6IUXXpDL5dKtt96qlJQUb2cEAABe4NFKgCS1b9/e/fVNN92km266ySuBAABA\n9aiwBEycONHjJ5k5c2aVhAEAANWnwhJw5MgR99elpaXavn27GjVqpIiICNWqVUtfffWVcnJy1L17\n92oJCgAAqlaFJeCVV15xfz1r1iw1adLEfbEg6ZePDT755JM6ffq091MCAIAq59GJgampqRo6dKi7\nAEi/3Hv94Ycfdt9kCAAA1CwelYC6devqu+++u2B89+7dCgoKqvJQAADA+zz6dMADDzygSZMm6bvv\nvlNERIRM09QXX3yh1157TaNGjfJ2RgAA4AUelYCRI0fKZrNpxYoVKigokCQ1atRITzzxhP761796\nNSAAAPAOj0qAYRiKj49XfHy8jh49KsMw1KBBA29nAwAAXuTxxYLOOXcjIQAAULN5dGIgAAC48lAC\nAACwqApLwNNPP60TJ05IknJycmSaZrWFAgAA3ldhCVixYoUKCwslSd26ddOxY8eqLRQAAPC+Ck8M\nDAsL04gRI3T99dfLNE1Nnz5dV111VblzuYEQAAA1T4UlYM6cOVq6dKny8vJkGIZ+/PHHMpcNBgAA\nNVuFJSAiIkLPPvusJOmOO+7Q/PnzuTYAAABXEI+uE/Dxxx/LNE1t2rRJ3377rex2u1q3bq0//vGP\nstls3s4IAAC8wKMScPz4cQ0ePFj79u1TgwYN5HK5dOLECUVEROjll1/mJkIAANRAHl0nYObMmSot\nLdV7772nbdu2KSMjQ++++65M09ScOXO8nREAAHiBRyVg48aNSk5OVsuWLd1jrVq10qRJk5Senu61\ncAAAwHs8KgGmaap+/foXjAcFBen06dNVHgoAAHifRyWgY8eOWrZsmUpLS91jpaWlWrp0qSIjI70W\nDgAAeI9HJwaOHTtWAwYM0J133qn27dtLkvbs2aOioiK9/PLLl9z+559/VmJion744QcVFxfr8ccf\nV6tWrTRhwgQZhqHWrVtrypQp8vPz04IFC7Rx40bZ7XYlJiYqMjJShw8f9nguAADwjEcl4Nprr9U7\n77yj119/XQcOHFCdOnXUq1cvPfTQQ/rd7353ye3Xrl2roKAgzZ49W8ePH1fv3r113XXXafTo0erc\nubOSk5OVnp6u0NBQbd++XampqcrNzdXIkSO1atUqzZw50+O5AADAMx6VAElq0qSJxo8fX6md9OjR\nQ7GxsZJ+Ob/AZrMpMzNTnTp1kiR17dpVW7duVfPmzRUTEyPDMBQaGqrS0lIdPXr0suYGBwdXKiMA\nAFbjcQn4Nfz9/SVJRUVFGjVqlEaPHq2nnnpKhmG4Hy8sLFRRUVGZaw6cGzdN0+O5lyoBDRrUk91+\n+Rc4Ord/XLkMw1BISICvYwBAtamWEiBJubm5io+P14ABA9SzZ0/Nnj3b/ZjT6VRgYKAcDoecTmeZ\n8YCAAPn5+Xk891KOHTtVqfzcSvnKZ5qm8vMLfR0DAKrUxd7cePTpgF+roKBAgwcP1rhx4xQXFyfp\nl3sTZGRkSJI2b96s6OhoRUVFacuWLXK5XMrJyZHL5VJwcPBlzQUAAJ7xaCVgwoQJevTRR9W8efNK\n7WTJkiU6efKkFi1apEWLFkmSJk2apOnTp2vu3Llq0aKFYmNjZbPZFB0drb59+8rlcik5OVmSlJCQ\noKSkJI/mAgAAzximB+vc0dHRWrNmjcLCwqojk1dVdrk3Lu5eOZ1O1Q0KU9OoPlWcCr6UtTNVp49n\ny9/fX2lp7/o6DgBUqV99OKBnz556/vnndfjwYZWUlFRZMAAA4DseHQ7Ytm2bDh06pHXr1skwjDIn\n6knS3r17vRIOAAB4j0cl4NFHH/V2DgAAUM08KgH33Xeft3MAAIBq5vFHBD///HM98sgjuuOOO/TD\nDz9o/vz5WrNmjTezAQAAL/KoBGzatEmPPPKIGjdurIKCArlcLhmGoUmTJnG9fgAAaiiPSsCCBQs0\nfvx4paSkyGb75ZK7I0aMUEJCgkd3EQQAAL89HpWAAwcOqGvXrheM33777crKyqryUAAAwPs8KgEN\nGjQo95f93r171bBhwyoPBQAAvM+jEvDAAw9o2rRp2rRpkyTp+++/V1pamlJSUvjkAAAANZTH1wko\nLCzUyJEjVVxcrCFDhshut2vQoEGKj4/3dkYAAOAFHpUAwzA0btw4xcfH67vvvlOtWrV0zTXXqE6d\nOt7OBwAAvMSjEiBJZ86c0fr16/Xtt9+qdu3aat26te655x7Z7R4/BQAA+A3x6Dd4Zmamhg0bpjNn\nzqhFixZyuVxasWKFFi5cqBdffFFNmzb1dk4AAFDFPDoxcPr06brhhhu0efNmpaamatWqVfrkk0/U\ntGlTTZs2zdsZAQCAF3hUAjIzM/XEE0/I39/fPRYUFKRx48bp888/91o4AADgPR6VgKZNm+rw4cMX\njOfl5enqq6+u8lAAAMD7KjwnYOfOne6v//znP2vSpEkaM2aMOnbsKJvNpn379unpp5/mI4IAANRQ\nFZaAAQMGyDAMmabpHktOTr5g3rRp09SvXz/vpAMAAF5TYQlIT0+vzhwAAKCaVVgCmjRpUp05AABA\nNfPoOgFZWVmaN2+evv32WxUXF1/w+IYNG6o8GAAA8C6PSkBCQoLy8vJ09913c6lgAACuEB6VgH37\n9un1119X27ZtvZ0HAABUE4+uExAeHq7Tp097OwsAAKhGHq0EJCUlKSUlRYMGDVJYWJj8/Mp2h6io\nKK+EAwAA3uNRCTh48KC+++47TZgw4YLHDMPQV199VeXBAACAd3lUAp5//nnFxcXpoYceUt26db2d\nCQAAVAOPSkBRUZEeeeQRhYWFeTsPAACoJh6dGHjHHXfoo48+8nYWAABQjTxaCQgNDdUzzzyjDz74\nQOHh4bLby26WkpLi0c6+/PJLzZkzR8uXL9e+ffv06KOP6pprrpEk9e/fX/fcc48WLFigjRs3ym63\nKzExUZGRkTp8+LAmTJggwzDUunVrTZkyRX5+fuXOBQAAnvGoBOzatUsdO3aUJGVnZ1dqR8uWLdPa\ntWvd5xRkZmZq0KBBGjx4sHtOZmamtm/frtTUVOXm5mrkyJFatWqVZs6cqdGjR6tz585KTk5Wenq6\nQkNDy50LAAA841EJWL58+a/eUbNmzTR//nyNHz9ekrR3714dPHhQ6enpCg8PV2Jionbs2KGYmBgZ\nhqHQ0FCVlpbq6NGjyszMVKdOnSRJXbt21datW9W8efNy5wYHB//qrAAAWIFHJWDnzp0XfdyT6wTE\nxsaWWUWIjIxUnz591K5dOy1evFgLFy5UQECAgoKC3HP8/f1VWFgo0zRlGEaZsaKionLnUgIAAPCM\nRyVgwIABMgxDpmm6xwzDkGEY8vPz0969ey97x3feeacCAwPdX6ekpKhbt25yOp3uOU6nUwEBAWUu\nTuR0OhUYGCiHw1Hu3Etp0KCe7HbbZec9V0Jw5TIMQyEhl/47BABXCo9KQHp6epnvS0tLdfDgQT33\n3HMaO3ZspXY8ZMgQJSUlKTIyUtu2bVPbtm0VFRWl2bNna8iQITpy5IhcLpeCg4MVERGhjIwMde7c\nWZs3b9Yf//hHNWvWrNy5l3Ls2KlK5T2/AOHKZJqm8vMLfR0DAKrUxd7ceFQCmjRpcsFYs2bN5O/v\nr2nTpmndunWXHWrq1KlKSUlRrVq11LBhQ6WkpMjhcCg6Olp9+/aVy+VScnKypF/uYpiUlKS5c+eq\nRYsWio2Nlc1mK3cuAADwjGH+ire4Bw8eVK9evbR79+6qzORVlX2nFxd3r5xOp+oGhalpVJ8qTgVf\nytqZqtPHs+Xv76+0tHd9HQcAqtSvXgko78TAoqIivfrqq2rdunXlkwEAAJ+p9ImB0i+HCWbPnu2V\nYAAAwLsqdWKgJNWqVUu///3vqzwQAACoHpU+MRAAANRsFZaApKQkj57AMAw9+eSTVRYIAABUjwpL\nwKFDhy66YXZ2tnJzc2W32ykBAADUQBWWgIruF1BSUqIlS5boiy++0HXXXaeZM2d6LRwAAPAej84J\nOGffvn2aOHGiDh48qOHDh2vYsGEX3FYYAADUDB79Bi8uLtaCBQv00ksvqW3btlq9erVatWrl7WwA\nAMCLLlkCdu3apUmTJik7O1t///vfNWjQoDI39AEAADVThSXg7Nmzmjt3rlasWKE//OEPWrRokcLD\nw6szGwAA8KIKS0DPnj2VlZWlpk2b6pZbbtH7779f4ZM89thjXgkHAAC8p8ISUFJSosaNG6ukpESp\nqakVPoFhGJQAAABqoApLwMcff1ydOQAAQDXjDD8AACyKEgAAgEVRAgAAsChKAAAAFkUJAADAoigB\nAABYFCUAAACLogQAAGBRlAAAACyKEgAAgEVRAgAAsChKAAAAFkUJAADAoigBAABYFCUAAACLogQA\nAGBRlAAAACyqWkvAl19+qYEDB0qSDh8+rP79+2vAgAGaMmWKXC6XJGnBggWKi4tTv379tHv37sue\nCwAAPFNtJWDZsmWaPHmyzp49K0maOXOmRo8erX/9618yTVPp6enKzMzU9u3blZqaqrlz52ratGmX\nPRcAAHim2kpAs2bNNH/+fPf3mZmZ6tSpkySpa9eu+uyzz7Rjxw7FxMTIMAyFhoaqtLRUR48evay5\nAADAM/bq2lFsbKyys7Pd35umKcMwJEn+/v4qLCxUUVGRgoKC3HPOjV/O3ODg4IvmaNCgnux222Xn\nP7d/XLndnyZbAAAM+klEQVQMw1BISICvYwBAtam2EvDf/Pz+/yKE0+lUYGCgHA6HnE5nmfGAgIDL\nmnspx46dqlRe0zQrtR1qDtM0lZ9f6OsYAFClLvbmxmefDoiIiFBGRoYkafPmzYqOjlZUVJS2bNki\nl8ulnJwcuVwuBQcHX9ZcAADgGZ+tBCQkJCgpKUlz585VixYtFBsbK5vNpujoaPXt21cul0vJycmX\nPRcAAHjGMC22zl3Z5d64uHvldDpVNyhMTaP6VHEq+FLWzlSdPp4tf39/paW96+s4AFClfpOHAwAA\ngG9RAgAAsChKAAAAFkUJAADAoigBAABYFCUAAACLogQAAGBRlAAAACyKEgAAgEVRAgAAsChKAAAA\nFkUJAADAoigBAABYFCUAAACLogQAAGBRlAAAACyKEgAAgEVRAgAAsChKAAAAFkUJAADAoigBAABY\nFCUAAACLogQAAGBRlAAAACyKEgAAgEVRAgAAsChKAAAAFkUJAADAoigBAABYFCUAAACLsvs6wH33\n3SeHwyFJCgsLU9++ffWPf/xDNptNMTExGjFihFwul6ZOnaqvv/5atWvX1vTp0xUeHq5du3ZdMBcA\nAHjGpyXg7NmzMk1Ty5cvd4/16tVL8+fPV9OmTTVs2DDt27dP2dnZKi4u1sqVK7Vr1y7NmjVLixcv\n1pQpUy6YGxER4cNXBABAzeHTErB//36dPn1agwcPVklJiUaOHKni4mI1a9ZMkhQTE6PPPvtM+fn5\n6tKliySpY8eO2rt3r4qKisqdSwkAAMAzPi0BderU0ZAhQ9SnTx8dOnRIQ4cOVWBgoPtxf39/ZWVl\nqaioyH3IQJJsNtsFY+fmXkqDBvVkt9suO6thGJe9DWoWwzAUEhLg6xgAUG18WgKaN2+u8PBwGYah\n5s2bKyAgQMePH3c/7nQ6FRgYqDNnzsjpdLrHXS6XHA5HmbFzcy/l2LFTlcpqmmaltkPNYZqm8vML\nfR0DAKrUxd7c+PTTAWlpaZo1a5YkKS8vT6dPn1a9evX0/fffyzRNbdmyRdHR0YqKitLmzZslSbt2\n7dK1114rh8OhWrVqXTAXAAB4xqcrAXFxcZo4caL69+8vwzA0Y8YM+fn5aezYsSotLVVMTIw6dOig\n9u3ba+vWrerXr59M09SMGTMkSdOmTbtgLgAA8IxhWmydu7LLvXFx98rpdKpuUJiaRvWp4lTwpayd\nqTp9PFv+/v5KS3vX13EAoEr9Zg8HAAAA36EEAABgUZQAAAAsihIAAIBFUQIAALAoSgAAABZFCQAA\nwKIoAQAAWBQlAAAAi6IEAABgUZQAAAAsihIAAIBFUQIAALAoSgAAABZFCQAAwKIoAQAAWBQlAAAA\ni6IEAABgUZQAAAAsihIAAIBFUQIAALAoSgAAABZl93UAwKri44cqLy/X1zHgRY0aNdbChct8HQOo\nECUA8JG8vFw5nU5fx4AXUfLwW0cJAHzMr7ZNdRr6+zoGqtCZAqdcxaW+jgFcEiUA8LE6Df3V4r72\nvo6BKvSft/foVM5JX8cALokTAwEAsChKAAAAFkUJAADAoigBAABYFCcGAsAVgOtOXPm8cd2JGl8C\nXC6Xpk6dqq+//lq1a9fW9OnTFR4e7utYAFCtuO7Elc8bJa/Gl4CPPvpIxcXFWrlypXbt2qVZs2Zp\n8eLFvo4FAD5Rx89PV9ep6+sYqEJHzpzWGZfLK89d40vAjh071KVLF0lSx44dtXfvXq/u72xRvrJ2\npnp1H6heZ4vyfbr/MwVO/eftPT7NgKp1poB35KgZanwJKCoqksPhcH9vs9lUUlIiu738lxYSElCp\n/WzatKlS2wEV4e8UqhJ/n1AZNf7TAQ6Ho8xxMJfLVWEBAAAA/1+NLwFRUVHavHmzJGnXrl269tpr\nfZwIAICawTBN0/R1iF/j3KcDvvnmG5mmqRkzZqhly5a+jgUAwG9ejS8BAACgcmr84QAAAFA5lAAA\nACyKEoAKJScna9KkSb6OgRqsoKBACQkJiomJUXR0tIYMGaJvvvnG17FQgx05ckSjRo1Sp06dFB0d\nrTFjxigvL8/XsWosSgAuYJqmnnvuOa1cudLXUVCDuVwujRgxQocOHdKiRYv05ptvyuFw6G9/+5uO\nHTvm63iogUzT1LBhw3Ty5Em99tprWrFihfLz8/X444/7OlqNRQlAGVlZWXr44Yf1xhtvKDQ01Ndx\nUIPt379fX3zxhWbMmKHIyEi1atVKs2fP1qlTp7iwDSqloKBALVu21PTp03Xdddfpuuuu09/+9jdl\nZmbqxIkTvo5XI1ECUMbOnTvVuHFjrVu3TmFhYb6OgxqscePGeuGFF9S8eXP3mGEYksR/2KiUkJAQ\nzZs3z/1/05EjR7Ry5Uq1b99e9evX93G6molL66GMXr16qVevXr6OgStAgwYNdNttt5UZW758uc6c\nOaOYmBjfhMIVY/jw4UpPT1f9+vX12muv+TpOjcVKAIBqkZ6errlz52rQoEFc0Au/2hNPPKHU1FRF\nRUVp0KBBnBxYSZQAAF63evVqjRo1SnfffbfGjRvn6zi4ArRp00aRkZGaN2+eXC6X3n77bV9HqpEo\nAQC8avHixZo4caL69eunp59+Wn5+/LeDyikoKNB7771XZqxu3bpq2rQpKwGVxL9GAF6zbNkyPfvs\nsxo1apSSkpLcJwYClZGTk6O///3v2rNnj3ussLBQBw8eVKtWrXyYrOaiBADwiv3792vevHn6y1/+\nogceeED5+fnuP6dOnfJ1PNRA7dq1U3R0tCZPnqzdu3dr3759Gj16tIKDg9W7d29fx6uRKAEAvGL9\n+vUqLS3VqlWrFBMTU+bPP//5T1/HQw3k5+en+fPn6/rrr9ejjz6qhx56SP7+/lqxYoX8/f19Ha9G\n4i6CAABYFCsBAABYFCUAAACLogQAAGBRlAAAACyKEgAAgEVRAgAAsChKAACPFBcX68UXX1Tv3r31\nhz/8QTfffLMee+wx99XbsrOz1aZNG/373//2cVIAnuJWwgAu6fTp03r44Yd17NgxjRo1Sh06dJDT\n6dRrr72mBx98UEuXLnXf4x1AzUEJAHBJzz77rA4dOqR3331XjRo1co/PmjVLP/30k1JSUrRkyRIf\nJgRQGZQAABdVXFys1atXKy4urkwBOCc5OVlOp/OCmwMdP35cTz31lD799FMdO3ZMDRo0UM+ePTVu\n3Dj5+fmpoKBAU6dO1fbt21VcXKyOHTsqISFB119/vaRfbj+8bNkyZWVlKSQkRPfdd59GjBjBXQiB\nKkQJAHBRWVlZOnnypDp06FDu402bNpX0yzkB50tISNCxY8e0ePFiBQUFafPmzUpJSdENN9yg7t27\na9q0aSopKdEbb7whwzD0zDPPaOTIkfroo4+0f/9+JScna+7cuWrXrp0yMzM1duxYNWvWjBvFAFWI\nEgDgok6ePClJCgwMvKztunTpos6dO6t169aSpAcffFAvvviivv76a3Xv3l2HDx9WmzZtFBYWpquu\nukpPPvmkDhw4IJfLpaysLBmGodDQUPefV155RVdffXWVvz7AyigBAC6qQYMGkn5Z3r8c/fv3V3p6\nulJTU3Xo0CF9/fXXOnLkiFwulyRp+PDhSkhI0AcffKAbb7xRXbt2Vc+ePeXn56cuXbqoQ4cO+stf\n/qLw8HDFxMSoR48eCg0NrfLXB1gZB9cAXFSzZs30u9/9Tl9++WW5j2dkZOixxx5Tfn6+e8zlcmnY\nsGGaNWuW6tatq169emnFihVq0qSJe06PHj306aefavr06QoJCdGiRYt07733qqCgQHXq1NGKFSuU\nlpamXr16ad++fRo4cKAWLFjg9dcLWAkrAQAuys/PT/fdd59WrlypwYMHlzk50DRNLV26VNnZ2WrY\nsKF7fN++fdqyZYtWr16ttm3bSpKKioqUn58v0zRVUlKiZ555Rn/+85/Vs2dP9ezZUz/99JNuvvlm\nbd++XfXr19euXbsUHx+v9u3bKz4+XlOnTtX69es1YsSIav8ZAFcqSgCASxo+fLi2bt2qAQMGaMyY\nMerQoYMKCgr08ssv6/PPP9fLL79c5tMBISEhstvtev/991W/fn3l5+dr3rx5Ki4uVnFxsex2uzIz\nM/Xvf/9bkydPVnBwsNatW6datWqpbdu2ysvL08KFCxUQEKDbb79dBQUFysjIUMeOHX34UwCuPIZp\nmqavQwD47SsqKtKyZcu0YcMG5ebmKiAgQB06dNCIESN0/fXXKzs7W926ddPrr7+u6OhovfPOO5o/\nf77y8vLUqFEj3X333Tpy5Ih+/PFHvfrqq8rPz9eMGTO0bds2OZ1OtW7dWk888YRuvfVWSdKaNWv0\n4osv6vvvv5fD4VD37t01fvx4ORwOH/8kgCsHJQAAAIvixEAAACyKEgAAgEVRAgAAsChKAAAAFkUJ\nAADAoigBAABYFCUAAACLogQAAGBRlAAAACzq/wG10WlZQpZ0pwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108ebfac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "index = np.arange(len(class_name))\n",
    "# plt.bar(class_name, count)\n",
    "sns.barplot(x=class_name, y=count, linewidth=2.5, errcolor=\".2\", edgecolor=\".2\")\n",
    "plt.xlabel('Class', fontsize=15)\n",
    "plt.ylabel('Number of data', fontsize=15)\n",
    "plt.xticks(index, class_name, fontsize=15)\n",
    "plt.title('Class distribution')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped_df = train.dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>y1</th>\n",
       "      <th>y2</th>\n",
       "      <th>y3</th>\n",
       "      <th>y4</th>\n",
       "      <th>y5</th>\n",
       "      <th>y6</th>\n",
       "      <th>z1</th>\n",
       "      <th>z2</th>\n",
       "      <th>z3</th>\n",
       "      <th>z4</th>\n",
       "      <th>z5</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>38829.000000</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>3.882900e+04</td>\n",
       "      <td>38829.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.452564</td>\n",
       "      <td>0.324257</td>\n",
       "      <td>0.753640</td>\n",
       "      <td>0.425769</td>\n",
       "      <td>0.311064</td>\n",
       "      <td>0.199044</td>\n",
       "      <td>0.163545</td>\n",
       "      <td>0.283753</td>\n",
       "      <td>0.559216</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.794910</td>\n",
       "      <td>0.420935</td>\n",
       "      <td>0.508013</td>\n",
       "      <td>4.703503e-01</td>\n",
       "      <td>1.284139e-01</td>\n",
       "      <td>3.757896e-02</td>\n",
       "      <td>1.054186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.243708</td>\n",
       "      <td>0.219627</td>\n",
       "      <td>0.156440</td>\n",
       "      <td>0.208532</td>\n",
       "      <td>0.209755</td>\n",
       "      <td>0.133914</td>\n",
       "      <td>0.129485</td>\n",
       "      <td>0.112639</td>\n",
       "      <td>0.094967</td>\n",
       "      <td>0.078400</td>\n",
       "      <td>0.146847</td>\n",
       "      <td>0.299677</td>\n",
       "      <td>0.206335</td>\n",
       "      <td>3.245707e-01</td>\n",
       "      <td>1.509215e-01</td>\n",
       "      <td>6.229685e-02</td>\n",
       "      <td>0.262132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.015714</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>0.152338</td>\n",
       "      <td>0.014287</td>\n",
       "      <td>0.010255</td>\n",
       "      <td>0.000374</td>\n",
       "      <td>0.002973</td>\n",
       "      <td>0.029058</td>\n",
       "      <td>0.254826</td>\n",
       "      <td>0.273939</td>\n",
       "      <td>0.386060</td>\n",
       "      <td>0.000813</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>9.265071e-09</td>\n",
       "      <td>1.756340e-12</td>\n",
       "      <td>4.764865e-15</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.220130</td>\n",
       "      <td>0.123741</td>\n",
       "      <td>0.616893</td>\n",
       "      <td>0.243828</td>\n",
       "      <td>0.119672</td>\n",
       "      <td>0.079274</td>\n",
       "      <td>0.044599</td>\n",
       "      <td>0.212762</td>\n",
       "      <td>0.493038</td>\n",
       "      <td>0.494189</td>\n",
       "      <td>0.679581</td>\n",
       "      <td>0.152327</td>\n",
       "      <td>0.342202</td>\n",
       "      <td>1.132663e-01</td>\n",
       "      <td>5.486555e-03</td>\n",
       "      <td>4.072433e-04</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.423327</td>\n",
       "      <td>0.253957</td>\n",
       "      <td>0.779789</td>\n",
       "      <td>0.386694</td>\n",
       "      <td>0.255930</td>\n",
       "      <td>0.177742</td>\n",
       "      <td>0.135778</td>\n",
       "      <td>0.261159</td>\n",
       "      <td>0.541170</td>\n",
       "      <td>0.537088</td>\n",
       "      <td>0.805458</td>\n",
       "      <td>0.333305</td>\n",
       "      <td>0.554853</td>\n",
       "      <td>4.999816e-01</td>\n",
       "      <td>7.025290e-02</td>\n",
       "      <td>1.001907e-02</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.654701</td>\n",
       "      <td>0.486845</td>\n",
       "      <td>0.889894</td>\n",
       "      <td>0.601650</td>\n",
       "      <td>0.481184</td>\n",
       "      <td>0.295147</td>\n",
       "      <td>0.256690</td>\n",
       "      <td>0.331268</td>\n",
       "      <td>0.638947</td>\n",
       "      <td>0.576464</td>\n",
       "      <td>0.937122</td>\n",
       "      <td>0.724802</td>\n",
       "      <td>0.665629</td>\n",
       "      <td>7.853695e-01</td>\n",
       "      <td>2.037514e-01</td>\n",
       "      <td>4.835476e-02</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971634</td>\n",
       "      <td>0.989442</td>\n",
       "      <td>0.950625</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.592082</td>\n",
       "      <td>0.764123</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.766188</td>\n",
       "      <td>0.823532</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.947246</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>8.771210e-01</td>\n",
       "      <td>8.727469e-01</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 x1            x2            x3            x4            x5  \\\n",
       "count  38829.000000  38829.000000  38829.000000  38829.000000  38829.000000   \n",
       "mean       0.452564      0.324257      0.753640      0.425769      0.311064   \n",
       "std        0.243708      0.219627      0.156440      0.208532      0.209755   \n",
       "min        0.015714      0.014286      0.152338      0.014287      0.010255   \n",
       "25%        0.220130      0.123741      0.616893      0.243828      0.119672   \n",
       "50%        0.423327      0.253957      0.779789      0.386694      0.255930   \n",
       "75%        0.654701      0.486845      0.889894      0.601650      0.481184   \n",
       "max        1.000000      0.971634      0.989442      0.950625      1.000000   \n",
       "\n",
       "                 y1            y2            y3            y4            y5  \\\n",
       "count  38829.000000  38829.000000  38829.000000  38829.000000  38829.000000   \n",
       "mean       0.199044      0.163545      0.283753      0.559216      0.540541   \n",
       "std        0.133914      0.129485      0.112639      0.094967      0.078400   \n",
       "min        0.000374      0.002973      0.029058      0.254826      0.273939   \n",
       "25%        0.079274      0.044599      0.212762      0.493038      0.494189   \n",
       "50%        0.177742      0.135778      0.261159      0.541170      0.537088   \n",
       "75%        0.295147      0.256690      0.331268      0.638947      0.576464   \n",
       "max        0.592082      0.764123      1.000000      0.766188      0.823532   \n",
       "\n",
       "                 y6            z1            z2            z3            z4  \\\n",
       "count  38829.000000  38829.000000  38829.000000  3.882900e+04  3.882900e+04   \n",
       "mean       0.794910      0.420935      0.508013  4.703503e-01  1.284139e-01   \n",
       "std        0.146847      0.299677      0.206335  3.245707e-01  1.509215e-01   \n",
       "min        0.386060      0.000813      0.000005  9.265071e-09  1.756340e-12   \n",
       "25%        0.679581      0.152327      0.342202  1.132663e-01  5.486555e-03   \n",
       "50%        0.805458      0.333305      0.554853  4.999816e-01  7.025290e-02   \n",
       "75%        0.937122      0.724802      0.665629  7.853695e-01  2.037514e-01   \n",
       "max        0.999997      0.999998      0.947246  1.000000e+00  8.771210e-01   \n",
       "\n",
       "                 z5         label  \n",
       "count  3.882900e+04  38829.000000  \n",
       "mean   3.757896e-02      1.054186  \n",
       "std    6.229685e-02      0.262132  \n",
       "min    4.764865e-15      1.000000  \n",
       "25%    4.072433e-04      1.000000  \n",
       "50%    1.001907e-02      1.000000  \n",
       "75%    4.835476e-02      1.000000  \n",
       "max    8.727469e-01      3.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dropped_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0sAAAI+CAYAAABt3JGwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+UVeV9L/7PmYEZFBQVhSDKj2AwWaXBkFW9t0YTaqiR\nQpAJqAMSQy9ETRCWYlDAaCQWkQRNVYzIVa4SFYKondvY2hoT7VW/3l4Ff7RqKqgRCT+EpvycAebs\n7x9ZmQQyw9kS5uw9zOu11lnrzD7D3u8zw8Pwmc/zPLuQJEkSAAAA7KMi6wAAAAB5pFgCAABohmIJ\nAACgGYolAACAZiiWAAAAmqFYAgAAaEaH1jz5ZYW+rXn6Q+aE6sqsI6TSsVDIOkIq/7mnMesIqXTp\n0DZ+V/A/R1yZdYRUnthTl3WEVP7kf96XdYRUnj5jRNYRUjnuE8dmHSGVp//l/awjpDLnz2qyjpDK\nL8e2jZ+b86csyzpCKmf2OjrrCKkcf+pxWUdI5bR77846Qioden4i6wgfWTn/b3938m7ZrnUgbeN/\niwAAAGWmWAIAAGhGq07DAwAADg+VbWNFyCGlswQAANAMnSUAAKCkyjay2dihpLMEAADQDJ0lAACg\nJGuWAAAAiAidJQAAIAVrlgAAAIgInSUAACAFa5YAAACICJ0lAAAgBWuWAAAAiAjFEgAAQLNanIY3\nfvz42LNnzz7HkiSJQqEQS5cubfVgAABAfrTHDR5aLJauvvrquO6662LBggVRWVlZzkwAAACZa7FY\nGjRoUIwcOTLeeuutGDp0aDkzAQAAOWODh/1MnDgxPvzww6aP9+7dG7fcckurhwIAAMhayQ0eXnnl\nlZg2bVq88cYbMW7cuOjcuXM5cgEAADlSUcZHXpTMMnfu3KiqqoqampoYPXp0TJ48uRy5AAAAMlWy\nWJo2bVps27Ytli5dGo888kgsXLiwHLkAAIAcqSwUyvbIi5LF0uDBg+POO++MQYMGxZIlS2LLli3l\nyAUAAJCpFnfD+61x48Y1Pa+qqooZM2a0aiAAACB/2uN9lvK0fgoAACA3SnaWAAAA8rSWqFx0lgAA\nAJqhswQAAJRkzRIAAAARobMEAACkYM0SAAAAEaFYAgAAaFarTsM7obqyNU9/yGxqaMw6QioDj67O\nOkIq9cUk6wipbNndNr7vnY79WNYRUum4o2PWEVLZfWS3rCOk8l/bGrKOkEphza+zjpBK7yPbxt/P\nziecnHWEVI74eOesI6TSo7ptrDaorGobv7sutpGf75EUs05w2LLBAwAAABFhgwcAACAFGzwAAAAQ\nETpLAABACtYsAQAAEBE6SwAAQAo6SwAAAESEzhIAAJCC3fAAAACICJ0lAAAgBWuWAAAAiAidJQAA\nIAVrllLYvXt3a+QAAADIlRaLpaeffjqGDBkSQ4cOjSeeeKLp+MSJE8sSDAAAyI/KQvkeedHiNLy7\n7747Hn/88SgWizF16tRoaGiIUaNGRZIk5cwHAACQiRaLpY4dO0bXrl0jIuKuu+6KSy65JHr27BmF\ndjhXEQAAaH9anIbXq1evuPnmm2Pnzp3RpUuXuPPOO2P27NmxZs2acuYDAAByoLJQKNsjL1oslubM\nmROnnnpqPProoxER0bNnz1iyZEmcddZZZQsHAACwv2KxGNdff31ceOGFMX78+Hjvvff2ef2+++6L\nmpqa+MpXvhL//M//fNDXabFY6tChQ9TU1MTrr78e06ZNizfeeCO+8Y1vRK9evQ76YgAAQNuUpw0e\nnnrqqdi9e3csW7Yspk2bFnPnzm16bevWrfHAAw/E0qVL47777os5c+Yc9HsuuXX43Llzo6qqKmpq\namL06NExefLkg74YAADAH+ull15qmvF22mmnxeuvv9702hFHHBEnnnhi7Nq1K3bt2vVH7blQ8qa0\n06ZNi4aGhli6dGnMmTMntmzZEpdeeulBXxAAAGh78rSWaPv27dGlS5emjysrK2Pv3r3RocNvypue\nPXvGX/3VX0VjY+MfVbuU7CwNHjw47rzzzhg0aFAsWbIktmzZctAXAwAA+GN16dIlduzY0fRxsVhs\nKpSeffbZ2LhxY/z0pz+Nn//85/HUU0/Fq6++elDXKVksjRs3rul5VVVVzJgx46AuBAAAtF0VhULZ\nHqUMHjw4nn322YiIWLVqVQwYMKDpta5du0anTp2iqqoqqqur46ijjoqtW7ce1HsuOQ0PAAAgT4YO\nHRrPPfdcXHTRRZEkScyZMycWL14cvXv3jnPOOSeef/75uOCCC6KioiIGDx4cZ5555kFdR7EEAACU\nVEizTV2ZVFRUxOzZs/c51r9//6bnU6ZMiSlTpvzx1/mjzwAAAHAY0lkCAABKqshRZ6lcdJYAAACa\nobMEAACUVKhsf32W9veOAQAAUtBZAgAASsrTbnjl0qrFUscUN5TKg4FHV2cdIZXXtzZkHSGVEzu1\njRq8R5e28X1/IusAKVUffUTWEVKprGgb/y4dc0ynrCOk0ufs3llHSOWFB1/POkIqOzevyzpCKu/+\n+M2sI6SyqzHJOkIqXXp2yTpCKt1O7Z51hHSSYtYJOIyYhgcAANCMttECAAAAMmXrcAAAACJCZwkA\nAEihUNH++izt7x0DAACkoLMEAACUZM0SAAAAEaGzBAAApNAeb0qrswQAANAMnSUAAKCkQmX767O0\nv3cMAACQgs4SAABQkt3wDqC+vj52797dmlkAAAByo8XO0ttvvx233nprdO3aNUaMGBHXXXddVFRU\nxKxZs2LIkCHlzAgAAGSsUNH+OkstFks33HBDTJ06NT744IOYMmVKPPnkk1FdXR0TJ05ULAEAAIe9\nFoulYrEYp59+ekREvPjii9GtW7ff/IEOljkBAEB7U2E3vN/p169fzJo1K4rFYsydOzciIu655544\n4YQTyhYOAAAgKy0WSzfddFMMGTIkli1b1nSse/fu0aNHj7IEAwAAyFKLxVJFRUV88YtfjFdeeSWm\nTZsWb7zxRjz88MPRpUuXcuYDAAByoFBZKNsjL0pOPJw7d25UVVVFTU1NjB49OiZPnlyOXAAAAJkq\nWSxNmzYttm3bFkuXLo1HHnkkFi5cWI5cAABAjugsNWPw4MFx5513xqBBg2LJkiWxZcuWcuQCAADI\nVMl9wMeNG9f0vKqqKmbMmNGqgQAAgPyxdTgAAAARkaKzBAAAkKe1ROWiswQAANAMnSUAAKCkigqd\nJQAAAEJnCQAASKFgNzwAAAAidJYAAIAUKtrhbnitWiz9557G1jz9IVNfTLKOkMqJndpGbbuufm/W\nEVJpK9/3xt27so6Qyoe/3JR1hFT67Pp11hFSKe5uG/9+bl27NesIqXRuI1NHKjp0zDpCKj3/+8Cs\nI6Sy+0evZR0hlR0bdmYdIZWtR2/JOkIqSWVV1hE4jLSN/30DAACZcp8lAAAAIkJnCQAASMFueAAA\nAESEYgkAAKBZpuEBAAAltcetw3WWAAAAmqGzBAAAlFSo0FkCAAAgdJYAAIAUKmwdDgAAQITOEgAA\nkELBbngt27x5c2vmAAAAyJUWO0vvvPPOPh9fc801ccstt0RERL9+/Vo3FQAAkCuFdrhmqcViacKE\nCdGpU6fo3r17JEkS77zzTlx//fVRKBTigQceKGdGAACAsmuxWFqxYkXccMMNUVtbG2eeeWaMHz8+\nlixZUs5sAABAThQqdJaadOvWLX7wgx/ELbfcEq+99lo5MwEAAGTugOVhhw4dol+/ftGjR49IkiT2\n7t3btG4JAABoPyoqK8r2yIuSW4e/+uqrsWfPnpg1a1aMGzcuzjrrrHLkAgAAyFTJsm3u3LlRVVUV\nNTU1MXr06Jg8eXI5cgEAADlSqKwo2yMvSiaZNm1abNu2LZYuXRqPPPJILFy4sBy5AAAAMlWyWBo8\neHDceeedMWjQoFiyZEls2bKlHLkAAAAyVXLN0rhx45qeV1VVxYwZM1o1EAAAkD95mh5XLu3vHQMA\nAKRQsrMEAADQHm9K2/7eMQAAQAo6SwAAQEmFysqsI5SdzhIAAEAzdJYAAICS7IYHAABAROgsAQAA\nKVS0w93wWrVY6tKhbXxBt+xuzDpCKj26VGcdIZX6YpJ1hFTayve9Y6cjs46QSpcenbOOkMrOjkdn\nHSGVD7fvzjpCKtXrtmcdIZVeR7SN3w126npC1hFSqT65bYz3rh3bxv9D2opiY9v4+V5IillH4DDS\nNn56AAAAmbJmCQAAgIjQWQIAAFLQWQIAACAidJYAAIAUCu1wN7z2944BAABSUCwBAAA0wzQ8AACg\nJBs8AAAAEBE6SwAAQAo6SwAAAESEzhIAAJBChc5S84rFYmzYsCGKxWJr5wEAAMiFFoulmTNnRkTE\nK6+8Eueee25Mnjw5hg8fHqtWrSpbOAAAIB8KFRVle+RFi9Pw1q5dGxERt912WyxatCj69u0bGzZs\niGnTpsWPfvSjsgUEAADIQsk1S5WVldG3b9+IiOjRo4epeAAA0A7ZDe/3bN++PWpqamLdunWxfPny\naGhoiBtvvDFOPPHEcuYDAADIRIudpUcffTQaGhpi0aJFcdppp0WhUIh+/frFr371q3LmAwAAckBn\naT/V1dWxdu3auPvuu2P16tXxk5/8JDp37lyubAAAAJkpWR7OnTs3qqqqoqamJkaPHh2TJ08uRy4A\nACBH2uNueCWTTJs2LbZt2xZLly6NRx55JBYuXFiOXAAAAJkquRve4MGDY9y4cRERsWTJkpg/f36r\nhwIAAPKlorIy6whlV7Kz9NtCKSKiqqoqZsyY0aqBAAAA8iA/EwIBAABypOQ0PAAAAFuHAwAAEBE6\nSwAAQAo6SwAAAESEzhIAAJBCnm4WWy7t7x0DAACkoFgCAABKKlRWlO1RSrFYjOuvvz4uvPDCGD9+\nfLz33nvNfs7EiRPj4YcfPuj33KrT8P7niCtb8/SHTKdjP5Z1hFSeyDpASo27d2UdIZWOnY7MOkIq\nQxdOyzpCKjffvzzrCKlct31v1hFSufvqBVlHSKWispB1hFRe/+efZx0hlW//4+1ZR0hlwogHs46Q\nylN/cWnWEVL5xBmnZR0hlU/2PTbrCKlc17Ft/L+uX9YB2rinnnoqdu/eHcuWLYtVq1bF3Llz44c/\n/OE+n/ODH/wgtm7d+kddx5olAACgpDzthvfSSy/FWWedFRERp512Wrz++uv7vP6P//iPUSgUmj7n\nYOXnHQMAAKSwffv26NKlS9PHlZWVsXfvb2aP/OIXv4i///u/j6lTp/7R19FZAgAASsrTbnhdunSJ\nHTt2NH1cLBajQ4fflDaPP/54bNiwIS655JL44IMPomPHjtGrV684++yzP/J1FEsAAECbMnjw4PjZ\nz34Ww4YNi1WrVsWAAQOaXps+fXrT8zvuuCOOP/74gyqUIhRLAABACoWKyqwjNBk6dGg899xzcdFF\nF0WSJDFnzpxYvHhx9O7dO84555xDdh3FEgAA0KZUVFTE7Nmz9znWv3//P/i8K6644o+6jmIJAAAo\nLUedpXLJzyotAACAHFEsAQAANMM0PAAAoLQcbR1eLu3vHQMAAKSgswQAAJRUqLTBAwAAAKGzBAAA\npNEOtw5vsVgaP3587NmzZ59jSZJEoVCIpUuXtnowAACALLVYLF199dVx3XXXxYIFC6KyHc5PBAAA\nfo/O0u8MGjQoRo4cGW+99VYMHTq0nJkAAAAyd8A1SxMnToxLL700CoVCDBkyRIcJAADaqYL7LP2h\n6dOnx8svvxw1NTXxve99L959990yxAIAAMhWyWKpf//+MX369Fi8eHGsX78+hg8fHhMmTIiVK1eW\nIx8AAJAHFZXle+REya3Dn3nmmXjsscdi9erVMXLkyJg5c2bs3bs3Jk2aFHV1deXICAAAUHYli6W6\nurqora2NM844Y5/jV1xxRauFAgAAciZHHZ9yKVkszZ8/v9njdsgDAAAOZyWLJQAAALvhAQAAEBGK\nJQAAgGaZhgcAAJTWDjd40FkCAABohs4SAABQms4SAAAAETpLAABACoXK9tdZatVi6Yk9da15+kOm\n446OWUdIpfroI7KOkMqHv9yUdYRUuvTonHWEVG6+f3nWEVI5+pIxWUdI5aT7vpp1hFSePLlt/C5r\n7476rCOk8n63VVlHSGXm/PuzjpDKY3+yMesIqbzzwvNZR0jlmOr3s46QSvWuo7KOkEp1p2lZR+Aw\n0jZ+GgMAANlyU1oAAAAidJYAAIA07IYHAABAhM4SAACQQkFnCQAAgAidJQAAIA274QEAABChswQA\nAKRgzRIAAAARoVgCAABolml4AABAaabhlbZ79+7WyAEAAJArLRZLTz/9dAwZMiSGDh0aTzzxRNPx\niRMnliUYAACQIxUV5XvkRIvT8O6+++54/PHHo1gsxtSpU6OhoSFGjRoVSZKUMx8AAEAmWiyWOnbs\nGF27do2IiLvuuisuueSS6NmzZxQKhbKFAwAA8qFQac1Sk169esXNN98cO3fujC5dusSdd94Zs2fP\njjVr1pQzHwAAQCZaLJbmzJkTp556anzzm9+Mp556Krp37x5LliyJ8847r5z5AACAPKioLN8jJ1qc\nhtehQ4eoqamJQYMGxYoVK+KOO+6Iz33uczF27Nhy5gMAAMhEya0m+vfvH9OnT4/FixfH+vXr48tf\n/nJMmDAhVq5cWY58AABAHugs/aFnnnkmHnvssVi9enWMHDkyZs6cGXv37o1JkyZFXV1dOTICAACU\nXcliqa6uLmpra+OMM87Y5/gVV1zRaqEAAIB8KeTo/kflUrJYmj9/frPHhw4desjDAAAA5EXJYgkA\nACBPa4nKpf310gAAAFLQWQIAAEortL8+S/t7xwAAACkolgAAAJphGh4AAFCaaXgAAABE6CwBAAAp\nJO2ws1RIkiRprZPv/nBta536kNp9ZLesI6RSWVHIOkIqlbt+nXWEVHZ2PDrrCKms27436wipnPQP\n3886QirT//qBrCOkcvwT/5B1hFS+ffZJWUdIpXLr+qwjpPJ+x49lHSGVPv/1RtYRUmk8qnvWEVIp\nHnFM1hEOK/UV1VlHSKVr5yOyjvCRNb7zctmuVdlvcNmudSA6SwAAQGntsLPU/t4xAABACjpLAABA\naYW2sSTkUNJZAgAAaIbOEgAAUFpF++uztL93DAAAkILOEgAAUFJ7vM9S+3vHAAAAKegsAQAApeks\nAQAAEKGzBAAApKGzBAAAQMRHKJbq6+tj9+7drZkFAAAgN1qchvf222/HrbfeGl27do0RI0bEdddd\nFxUVFTFr1qwYMmRIOTMCAABZa4fT8Foslm644YaYOnVqfPDBBzFlypR48skno7q6OiZOnKhYAgAA\nDnstFkvFYjFOP/30iIh48cUXo1u3br/5Ax3sCQEAAO2Nm9L+nn79+sWsWbOiWCzG3LlzIyLinnvu\nieOPP75s4QAAALLSYrF00003xZAhQ+Ib3/hGPPXUU9HY2Bg9evSIm2++uZz5AACAPChUlO+REy0m\nqaioiC9+8YvxrW99K15++eWoqamJX/ziF7Fhw4Zy5gMAAMhEybKtf//+MX369Fi8eHGsX78+hg8f\nHhMmTIiVK1eWIx8AAJAHhUL5HjlRcreGZ555Jh577LFYvXp1jBw5MmbOnBl79+6NSZMmRV1dXTky\nAgAAlF3JYqmuri5qa2vjjDPO2Of4FVdc0WqhAACAnMnRWqJyKVkszZ8/v9njQ4cOPeRhAAAA8sJN\nkwAAgJLcZwkAAICI0FkCAADSqGh/fZb2944BAABS0FkCAABKs2YJAACACMUSAABAs0zDAwAASmuH\n0/BatVh6+owRrXn6Q+a/tjVkHSGVY47plHWEVIq7G7OOkMqH23dnHSGVu69ekHWEVJ48uW387uX4\nJ/4h6wipfDjsvKwjpPL68E9kHSGVrWu3Zh0hlennfzfrCKn8fxcekXWEVF4YMTbrCKkcd8qxWUdI\npctJx2cdIZVeV30n6wjpdO6XdQJSaBv/uwEAALLVDjtL7e8dAwAApKCzBAAAlJToLAEAABChswQA\nAKShswQAAECEzhIAAJBGoZB1grLTWQIAAGiGzhIAAFCaNUsAAABE6CwBAAAp5Ok+S8ViMb7zne/E\nW2+9FVVVVXHTTTdFnz59ml7/8Y9/HEuXLo0OHTrE5ZdfHkOGDDmo6yiWAACANuWpp56K3bt3x7Jl\ny2LVqlUxd+7c+OEPfxgREZs2bYolS5bEihUroqGhIcaOHRtnnnlmVFVVfeTrpC4PN2/e/JFPDgAA\nHCYKFeV7lPDSSy/FWWedFRERp512Wrz++utNr7366qvxmc98JqqqquKoo46K3r17x5tvvnlQb7nF\nJO+8884+j8svv7zpOQAAQFa2b98eXbp0afq4srIy9u7d2/TaUUcd1fRa586dY/v27Qd1nRan4U2Y\nMCE6deoU3bt3jyRJ4p133onrr78+CoVCPPDAAwd1MQAAoG1KcnSfpS5dusSOHTuaPi4Wi9GhQ4dm\nX9uxY8c+xdNH0WJnacWKFXHKKafEpZdeGkuWLIlPfvKTsWTJEoUSAACQqcGDB8ezzz4bERGrVq2K\nAQMGNL326U9/Ol566aVoaGiIbdu2xerVq/d5/aNosbPUrVu3+MEPfhDz5s2L11577aBODgAAcKgN\nHTo0nnvuubjooosiSZKYM2dOLF68OHr37h3nnHNOjB8/PsaOHRtJksSVV14Z1dXVB3WdA+6G16FD\nh3jvvffixBNPjD179hzUBQAAgLYvSbJO8DsVFRUxe/bsfY7179+/6fkFF1wQF1xwwR9/nVKfMH36\n9Ni4cWPs2rUrvve978W77777R18UAAAg70oWS/3794/p06fH4sWLY/369TF8+PCYMGFCrFy5shz5\nAACAHCgmSdkeeVHyprTPPPNMPPbYY7F69eoYOXJkzJw5M/bu3RuTJk2Kurq6cmQEAAAou5LFUl1d\nXdTW1sYZZ5yxz/Errrii1UIBAAD5kp9+T/mULJbmz5/f7PGhQ4ce8jAAAAB5UbJYAgAAKLbD1lLJ\nDR4AAADaI50lAACgpCRHu9SVi84SAABAM3SWAACAkqxZAgAAICJ0lgAAgBTaYWOpdYul4z5xbGue\n/pAprPl11hFS6XN276wjpLJ17dasI6RSvW571hFSqagsZB0hlb076rOOkMq3zz8p6wipvD78E1lH\nSGXR3/9H1hFSuXLS4KwjpHJkl6qsIxxWuvbpmnWEVI44/qisI6RS2dHv2Gl//K0HAABKsmYJAACA\niFAsAQAANMs0PAAAoCQ3pQUAACAidJYAAIAUilkHyIDOEgAAQDN0lgAAgJLa4ZIlnSUAAIDm6CwB\nAAAluSktAAAAEaGzBAAApOA+SwAAAEREymKpWCzGhg0bolhsj7urAwAAxTI+8qLFYmnmzJkREfHK\nK6/EueeeG5MnT47hw4fHqlWryhYOAAAgKy2uWVq7dm1ERNx2222xaNGi6Nu3b2zYsCGmTZsWP/rR\nj8oWEAAAyF47XLJUehpeZWVl9O3bNyIievToYSoeAADQLrRYLG3fvj1qampi3bp1sXz58mhoaIgb\nb7wxevbsWc58AABADhSTpGyPvGhxGt6jjz4aDQ0NMXny5Ni4cWNUVFTEgAEDYvTo0eXMBwAAkIkD\nTsOrrq6Oa6+9Nnbs2BGjR4+OtWvXxgcffFCubAAAAJkpuWapf//+MX369Fi8eHGsX78+hg8fHhMm\nTIiVK1eWIx8AAJADSRkfedHiNLzfeuaZZ+Kxxx6L1atXx8iRI2PmzJmxd+/emDRpUtTV1ZUjIwAA\nQNmVLJbq6uqitrY2zjjjjH2OX3HFFa0WCgAAyJdinlo+ZVKyWJo/f36zx4cOHXrIwwAAAORFyWIJ\nAAAgRzt6l03JDR4AAADaI50lAACgpGKu9qkrD50lAACAZugsAQAAJVmzBAAAQEToLAEAACm4z9Ih\n9vS/vN+apz9keh/ZMesIqbzw4OtZR0ilc2XbaFj2OqJt/K7g9X/+edYRUnm/26qsI6TyqRHrs46Q\nyta1W7OOkMqVkwZnHSGV2xa9nHWEVF4556msI6Ty9lv/nnWEVP7pidVZR0jl8587KesIqZwwsFfW\nEdJJilkn4DDSNv63CAAAZMqaJQAAACJCZwkAAEjBfZYAAACICMUSAABAs0zDAwAASrLBAwAAABGh\nswQAAKRQbIetJZ0lAACAZugsAQAAJTUWs05QfjpLAAAAzdBZAgAASrJmCQAAgIj4CJ2lLVu2xLHH\nHhuFQqE18wAAADnU2A47Sy0WSytWrIhf/epXMWTIkJg2bVpUV1dHfX193HDDDfHnf/7n5cwIAABQ\ndi0WSw899FAsWbIkLr/88vjhD38Y/fr1iw0bNsQ3vvENxRIAALQz1iz9no4dO8aRRx4ZnTt3jpNP\nPjkiInr06GEaHgAA0C602Fn6i7/4i7j88stjwIABcemll8ZZZ50Vzz77bPy3//bfypkPAADIgfZ4\nn6UWi6Wvf/3r8X//7/+NRYsWxdq1a2PDhg1xySWXxOc///ly5gMAAMjEAXfDO/3006Nbt27xyCOP\nxPPPPx8VFRXRp0+f6Nu3b5niAQAAeWDNUjP69+8f11xzTSxevDjWr18fw4cPjwkTJsTKlSvLkQ8A\nACATJe+z9Mwzz8Rjjz0Wq1evjpEjR8bMmTNj7969MWnSpKirqytHRgAAgLIrWSzV1dVFbW1tnHHG\nGfscv+KKK1otFAAAkC9uStuM+fPnN3t86NChhzwMAABAXpQslgAAAIrtr7FUeoMHAACA9khnCQAA\nKKmxHbaWdJYAAACaobMEAACU5Ka0AAAARITOEgAAkEJj+2sstW6xNOfPalrz9IdM5xNOzjpCKjs3\nr8s6QioVHTpmHSGVTl1PyDpCKt/+x9uzjpDKzPn3Zx0hlds7fizrCKlMP/+7WUdI5cguVVlHSOWV\nc57KOkIqF/30R1lHSGVKzYNZR0jl2VceyjpCKn1O/cusI6TS86Rjs46Qyr0dumcdIZWPZx2AVHSW\nAACAkqxZAgAAICJ0lgAAgBTcZwkAAICI0FkCAABSsGYJAACAiFAsAQAANMs0PAAAoKT2eFNanSUA\nAIBm6Cy/z66KAAAY4UlEQVQBAAAl2eABAACAiNBZAgAAUii6KS0AAAAROksAAEAKdsMDAAAgIg7Q\nWRo/fnzs2bNnn2NJkkShUIilS5e2ejAAACA/2uNueC0WS1dffXVcd911sWDBgqisrCxnJgAAgMy1\nWCwNGjQoRo4cGW+99VYMHTq0nJkAAICcaWyHnaUDrlmaOHFi/OxnP4tt27Y1Hbv22mtbPRQAAEDW\nSm7w8Nxzz8XXv/712LRpU0REfPDBB60eCgAAyJdiMSnbIy9KFku9e/eOWbNmxWWXXRZr1qyxfgkA\nAGgXUt1naeDAgTFv3ry46qqror6+vrUzAQAAOeM+S80YNmxYRET0798/FixYEH369Gn1UAAAAFkr\nWSzV1tY2Pe/Vq1csXLiwVQMBAADkQcliCQAAoJgkZXscjPr6+rjiiiti7NixMWnSpNiyZUuzn7dr\n164YOXJkPPvssyXPqVgCAADavIcffjgGDBgQDz30UJx//vlx1113Nft5s2fPjkKhkOqciiUAAKCk\nxiQp2+NgvPTSS3HWWWdFRMTZZ58dL7zwwh98zr333huf+cxn4pOf/GSqc6baDQ8AACAvli9fHvff\nf/8+x7p16xZHHXVURER07tw5tm3bts/rL7zwQrz33nsxe/bsePnll1NdR7EEAACU1Jijm8WOGTMm\nxowZs8+xyZMnx44dOyIiYseOHXH00Ufv8/ojjzwSH3zwQYwfPz7WrFkT//Zv/xYnnHBCfOpTn2rx\nOoolAACgzRs8eHA888wz8elPfzqeffbZ+OxnP7vP6/Pnz296fu2118awYcMOWChFWLMEAACk0FhM\nyvY4GLW1tfEf//EfUVtbG8uWLYvJkydHRMS8efPi1VdfPahzFpLkIFdQpfDre2a21qkPqSM+/oms\nI6Ty7o//d9YRUun53wdmHSGV6pP7ZR0hlQnr2sbX84E/2Zh1hFQqjjkh6wipJHsaso5wWHn71tuy\njpDKlNOmZR0hlX7fHJd1hFSuu35o1hFSOe7aO7KOkMr723ZnHSGVfh22Zx0hlarjTsw6wkd267+s\nLtu1rjqrf9mudSCm4QEAACXlac1SuZiGBwAA0AydJQAAoCSdJQAAACJCZwkAAEhBZwkAAICI0FkC\nAABS0FkCAAAgIhRLAAAAzTINDwAAKMk0PAAAACJCZwkAAEhBZwkAAICIOIjO0u7du6Oqqqo1sgAA\nADmls/R7nn766RgyZEgMHTo0nnjiiabjEydOLEswAACALLXYWbr77rvj8ccfj2KxGFOnTo2GhoYY\nNWpUJEn7qygBAKC9a4+dpRaLpY4dO0bXrl0jIuKuu+6KSy65JHr27BmFQqFs4QAAALLSYrHUq1ev\nuPnmm2Pq1KnRpUuXuPPOO+N//I//EVu3bi1nPgAAIAf2tsPOUotrlubMmROnnnpq3HTTTbFt27bo\n2bNnPPDAAzFw4MBy5gMAAMhEi8VShw4doqamJp577rn4+te/Hps2bYrjjz8+du7cWc58AABADjQW\nk7I98qLkfZZ69+4ds2bNissuuyzWrFkTFRVuzQQAABz+Ut1naeDAgTFv3ry46qqror6+vrUzAQAA\nOZOnjk+5lGwTDRs2LCIi+vfvHwsWLIg+ffq0eigAAICslews1dbWNj3v1atXLFy4sFUDAQAA+dPY\nDu+3agESAABAMxRLAAAAzUi1wQMAANC+2eABAACAiNBZAgAAUtBZAgAAICJ0lgAAgBTaY2epVYul\n+VOWtebpD5ke1W2jZtzV2Db+gu7+0WtZR0ila8e20Vh96i8uzTpCKu+88HzWEVLpe+P3s46Qygsj\nxmYdIZWufbpmHSGVf3piddYRUnn2lYeyjpDKvdcPzTpCKjfN/uesI6Ry0U+GZB0hld5n9cs6QioV\n37oh6wgcRtpGlQAAAGSqsVjMOkLZtY1frQMAAJSZzhIAAFBSe1yzpLMEAADQDJ0lAACgJJ0lAAAA\nIkJnCQAASGGvzhIAAAAROksAAEAK1iwBAAAQEYolAACAZpmGBwAAlGQaHgAAABHxETpL9fX1UVFR\nEVVVVa2ZBwAAyCGdpd/z9ttvxze+8Y2YMWNGPP/88zFs2LAYNmxY/OxnPytnPgAAgEy02Fm64YYb\nYurUqfHBBx/ElClT4sknn4zq6uqYOHFiDBkypJwZAQCAjLXHzlKLxVKxWIzTTz89IiJefPHF6Nat\n22/+QAd7QgAAAIe/Fqfh9evXL2bNmhXFYjHmzp0bERH33HNPnHDCCWULBwAA5ENjMSnbIy9aLJZu\nuummGDJkSFx33XWxffv2iIjo0aNHJEl+wgMAALSWFoulioqK+OIXvxjPPfdcTJo0KTZt2hQjR46M\nLVu2lDMfAACQA0kxKdsjL0reZ6l3794xa9asuOyyy2LNmjVRUeHWTAAAwOEv1W4NAwcOjHnz5sVV\nV10V9fX1rZ0JAADImWKOOj7lUrJNNGzYsIiI6N+/fyxYsCD69OnT6qEAAACyVrKzVFtb2/S8V69e\nsXDhwlYNBAAA5E973OjNAiQAAIBmuMMsAABQUp52qSsXnSUAAIBm6CwBAAAl2Q0PAACAiFAsAQAA\nNMs0PAAAoKSkmHWC8mvVYunMXke35ukPmcqqttFg69KzS9YRUtmxYWfWEQ4rnzjjtKwjpHJM9ftZ\nR0ileMQxWUdI5bhTjs06QipHHH9U1hFS+fznTso6Qip9Tv3LrCOkcty152YdIZWLfjIk6wipLH3p\nV1lHSKXvGx9mHSGVaddWZR2Bw4jOEgAAUJKb0gIAABAROksAAEAKtg4HAAAgInSWAACAFBKdJQAA\nACJ0lgAAgBR0lgAAAIgInSUAACCFovssAQAAEKGzBAAApGDNEgAAABGhswQAAKSgs3QAmzdvbs0c\nAAAAudJisfTOO+/s87j88subngMAABzuWpyGN2HChOjUqVN07949kiSJd955J66//vooFArxwAMP\nlDMjAACQsaJpeL+zYsWKOOWUU+LSSy+NJUuWxCc/+clYsmSJQgkAAGgXWuwsdevWLX7wgx/EvHnz\n4rXXXitnJgAAIGcSN6XdV4cOHWLbtm3RqVOnpi/OtddeW5ZgAAAAWSq5dfjzzz8f7777bvzt3/5t\nRER88MEHrR4KAADIl6SYdYLyK7l1eO/evWPWrFlx+eWXx5o1a6KysrIcuQAAADKV6qa0AwcOjHnz\n5sVVV10V9fX1rZ0JAADIGbvhNWPYsGEREdG/f/9YsGBB9OnTp9VDAQAAZK1kZ6m2trbpea9evWLh\nwoWtGggAAMifRGcJAACAiJRrlgAAgPZNZwkAAICI0FkCAABSKCY6SwAAAITOEgAAkII1SwAAAERE\nK3eWjj/1uNY8/SHTVu5G3O3U7llHSGXr0VuyjpBKsbFtfN8/2ffYrCOkUr3rqKwjHFa6nHR81hFS\nqezYNiYonDCwV9YRUul5UtsY7+9v2511hFR6n9Uv6wip9H3jw6wjpPLuzj1ZR0glqeyYdQQOI23j\npxwAAJAp0/AAAACICJ0lAAAghbaydOVQ0lkCAABohs4SAABQUtIOb0qrWAIAANq8+vr6+Na3vhWb\nN2+Ozp07xy233BLHHbfv7tw333xzvPTSS1FRURHXXHNNfPaznz3gOU3DAwAASkqKSdkeB+Phhx+O\nAQMGxEMPPRTnn39+3HXXXfu8/uabb8bKlStj+fLlMW/evPibv/mbkudULAEAAG3eSy+9FGeddVZE\nRJx99tnxwgsv7PN69+7do1OnTrF79+7Yvn17dOhQepKdaXgAAEBJedoNb/ny5XH//ffvc6xbt25x\n1FFHRURE586dY9u2bfu83qFDh6ioqIjzzjsvtm3bFt/97ndLXkexBAAAtCljxoyJMWPG7HNs8uTJ\nsWPHjoiI2LFjRxx99NH7vP7444/H8ccfH/fee2/s2LEjxo4dG6eddlp87GMfa/E6iiUAAKCkpNiY\ndYQDGjx4cDzzzDPx6U9/Op599tk/2Lzh6KOPjiOPPDIqKyujc+fOUVVVFTt37jzgORVLAABAm1db\nWxvXXHNN1NbWRseOHWP+/PkRETFv3rz40pe+FCNGjIiXX345LrroomhsbIwRI0bExz/+8QOeU7EE\nAACUlPfO0hFHHBG33377HxyfPn160/PZs2d/pHOm2g2vWCzGhg0bolgsfqSTAwAAtFUtFkszZ86M\niIhXXnklzj333Jg8eXIMHz48Vq1aVbZwAABAPiTFxrI98qLFaXhr166NiIjbbrstFi1aFH379o0N\nGzbEtGnT4kc/+lHZAgIAAGSh5DS8ysrK6Nu3b0RE9OjRw1Q8AACgXWixs7R9+/aoqamJXbt2xfLl\ny+PLX/5yzJ07N0488cRy5gMAAHIgaczP9LhyabGz9Oijj8bDDz8cJ5xwQnz84x+PQqEQn/jEJ8qZ\nDQAAIDMHnIZXXV0d7733Xnz/+9+P//qv/4qxY8fGxo0by5UNAADIifa4wUPJNUu9e/eOWbNmxWWX\nXRZr1qyJiopUu40DAAC0aaluSjtw4MCYN29eXHXVVVFfX9/amQAAgJzJU8enXEq2iYYNGxYREf37\n948FCxZEnz59Wj0UAABA1kp2lmpra5ue9+rVKxYuXNiqgQAAgPzRWQIAACAiUq5ZAgAA2jedJQAA\nACJCZwkAAEhBZwkAAICI0FkCAABSKOosAQAAEBFRSJIkaa2T7/3Vf7TWqQ+tpJh1gnTaSM6ksirr\nCKkU2sjX8/2OH8s6QiondmobX8/6Qtv4+3nk9vVZRzi8tJXx3qF71hFSOaliW9YRUqmobxs528rP\nzaSyY9YRUplywueyjpDK3cm7WUf4yE4Y+b2yXWvT332rbNc6EJ0lAACAZiiWAAAAmmGDBwAAoCRb\nhwMAABAROksAAEAKSaPOEgAAAKGzBAAApGDNEgAAABGhswQAAKSgswQAAEBE6CwBAAAp6CwBAAAQ\nETpLAABACkmxmHWEsjtgsbRs2bIWX7vwwgsPeRgAAIC8OGCxtGnTpnLlAAAAcqw9rlk6YLE0efLk\npufPP/98vP/++zFo0KDo169fqwcDAADIUqo1S7feemusX78+Vq9eHVVVVXHPPffErbfe2trZAACA\nnGiPnaVUu+G99NJLMW/evDjyyCNj1KhRsXbt2tbOBQAAkKlUxVJjY2M0NDREoVCIxsbGqKiw4zgA\nAHB4SzUN75JLLomamprYsmVLjBkzJr72ta+1ciwAACBPiu1wGl6qYum8886LP//zP49f/vKXcdJJ\nJ8Wxxx7b2rkAAAAylapYeu211+KGG26IDz/8ME488cS48cYb49RTT23tbAAAQE4kjTpLzfqbv/mb\nmDdvXpxyyinx1ltvxY033hgPPfRQa2cDAADITKpiqbq6Ok455ZSIiDj11FOjY8eOrRoKAADIl/a4\ndfgBi6Vly5b95pM6dIjvfOc78Wd/9mfx6quvRpcuXcoSDgAAICsHLJY2bdoUERGf+cxnIiLinXfe\niaOOOio+9alPtX4yAAAgN3SW9jN58uSm5xs3boy9e/dGkiSxcePGVg8GAACQpVRrlmbOnBmrVq2K\nXbt2RX19fZx88snx4x//uLWzAQAAOdEeO0sVaT7pzTffjJ/85Cfxuc99Ln7yk59EdXV1a+cCAADI\nVKrO0jHHHBOFQiF27twZxx13XGtnAgAAckZnqQUDBw6Me++9N7p37x5XXXVV1NfXt3YuAACATBWS\nJElaenH+/PlRKBQiSZLYs2dPFAqFePHFF+PTn/50fOc73yljTAAAgPI64DS8j3/8439wbMCAAa0W\nBgAAIC8O2FkCAABor1KtWQIAAGhvFEsAAADNUCwBAAA0Q7EEAADQjDZZLL3yyisxfvz4rGO0aM+e\nPfGtb30rxo4dG6NHj46f/vSnWUdqVmNjY8yYMSMuuuiiqK2tjV/84hdZRzqgzZs3x+c///lYvXp1\n1lFaNGrUqBg/fnyMHz8+ZsyYkXWcAzKODg3j6NBrK+No4cKFceGFF0ZNTU0sX7486zgHZLwfGsb7\noZf1eH/00Ufj+9//frOv3XHHHfHwww+nOs9H+VzalgNuHZ5HixYtirq6ujjiiCOyjtKiurq6OOaY\nY+J73/te/PrXv47zzz8/zjnnnKxj/YGf/exnERGxdOnSePHFF+O2226LH/7whxmnat6ePXvi+uuv\nj06dOmUdpUUNDQ2RJEksWbIk6yglGUeHjnF0aLWVcfTiiy/GypUr4+GHH45du3bFfffdl3WkFhnv\nh47xfmi1lfFO+5brztKDDz4YV111VUREXHPNNfHggw9G796944477sg42b72z7ljx46YOnVqREQk\nSRKVlZVZxmuyf84NGzbEd7/73YiIWLduXRx99NFZxmvS3Pf9lltuiYsuuii6d++ecbrf2T/nokWL\nYteuXfHXf/3X8dWvfjVWrVqVccLfmDZtWvz85z+PiIjVq1fH17/+9VyOo/1z1tXV5XIc7Z/zxz/+\ncS7HUXPf9zyOo/1zfvOb32wT42jRokUxYMCA+OY3vxmXXXZZfOELX8g032899NBDTb+lP/vss2PG\njBm5HO/75/x//+//5XK875/zpz/9aS7He3Pf9zyO9/1zTpw4MTfjff78+TFhwoQYNWrUPh2up556\nKr761a/GBRdcEK+++mpERPzDP/xDXHjhhVFbW9tiV4rDSJJzl19+eXLNNdckV155ZdOx999/Pxkz\nZkyGqf5Qczm3bduWXHzxxUldXV2GyfbVXM7p06cnn/nMZ5J/+Zd/yTDZvn4/54oVK5IFCxYkSZIk\nF198cfL2229nnO53fj/nm2++mSxbtiwpFovJmjVrknPOOSfZs2dP1hGTF154IZkyZUqSJEkyd+7c\n5Mknn0ySJH/jqKWceRtHLeXM2zjaP+d9992Xy3G0f8477rijTYyjoUOHJhMmTEgaGhqS1atXJ3/5\nl3+ZFIvFjFP+zquvvppceOGFya9//eskSfI33n9r/5x5G++/tX/OvI333/ptzsWLF+dyvP/Wb3P+\n67/+a+bjfcWKFcmcOXOSe+65J0mSJGlsbEy+9KUvJevXr09uv/325Nvf/naSJEnyi1/8Ijn//POT\n//zP/0zOO++8ZOfOnUmSJMnVV1+d/J//83+S22+/PXnooYfKmp3yyH2xtHLlymTAgAHJ66+/3nQs\nj//o759z3bp1yahRo5Lly5dnnGxfzX09kyRJNm7cmHzhC19IduzYkVGyff1+zrFjxybjxo1LLr74\n4uSzn/1s8pWvfCXZuHFj1hGTJNk3Z0NDQ7Jr166m177yla8k69atyzDdbxSLxeSv/uqvks2bNycj\nRoxIdu/enSRJ/sZRcznzOI5a+nomSb7G0f45L7jgglyOo/1ztpVxdPPNNyf33ntv0+sjRoxIPvzw\nwwwT/s7bb7+djBo1Klm/fn3TsbyN9yT5w5x5HO9J0vzXM0nyNd6TZN+cef65+fs58zDeV6xYkdx8\n883J3LlzkyuvvDKZNWtWcvbZZyfvv/9+cvvtt+9TuJ955pnJK6+8kpxxxhnJxRdfnFx88cXJqFGj\nkoceekixdBjLdbHU0NCQjBkzJlm6dGkyZsyYpKGhIUmS/P2jv3/OTZs2JV/60peS559/Puto+9g/\n56OPPprcfffdSZL85rd5Q4YM2ecfray09H1Pknz9hmz/nP/rf/2v5IYbbkiSJEnWr1+fnHvuubn4\njXiSJMnChQuTK6+8Mpk7d27TsbyNoyTZN2dex1GS7Jvzsccey+U4SpLmv+9Jkq9xlCT75nzwwQfb\nxDh6+umnk6997WtJsVhM1q9fn3zxi19M9u7dm3XEZO3atcmXv/zlP/j+5m28758zr+N9/5x5He8t\nfd+TJF/jff+ceRjvK1asSM4///xk6tSpSZIkyebNm5PTTz89+eUvf5ncfvvtyU033ZQkSZK8+eab\nyZgxY5LNmzcn559/ftMvylasWJH8+7//u2LpMJbrDR6+//3vxxe+8IW48MILY+PGjTF//vxc7ozU\nXM6tW7fGXXfdFXfddVdE/GaBbdaLLPfP+cYbb8SGDRti3LhxsXfv3pg5c2bmGZvL2Va+7+vWrYtt\n27ZFbW1tFAqFmDNnTnTokI8hVlNTE1/4whfi7/7u77KOckC/n/Puu+/O5TiK2Ddnz549Y8aMGbkb\nRxFt8/t+8sknx4wZM3I/jvr37x//+q//GqNHj44kSeL666/PxTqbG2+8Merr6+PGG2+MJEmiZ8+e\nMW/evKxj/YH9c3bu3DmX433/nMcee2wUCoXcjfe2+n3/7XqqrMf7n/7pn8a//du/xbhx46JQKMTJ\nJ58cGzdujIiItWvXxle/+tXYvXt3zJ49O4477rj42te+FuPHj4/Gxsbo1atXnHfeeWXPTPkUkiRJ\nsg4BtK4NGzbE9OnT4/777886ygHJeWjJeWi1lZwAHDq53g0P+OP90z/9U0ycODGmTJmSdZQDkvPQ\nkvPQais5ATi0dJYAAACaobMEAADQDMUSAABAMxRLAAAAzVAsAQAANEOxBAAA0AzFEgAAQDP+f/a+\n5f3YXknSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b6ae9b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "cor = train.corr()\n",
    "plt.figure(figsize=(16,10))\n",
    "sns.heatmap(cor)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To handle data imbalance issue, I have used the following three techniques :\n",
    "### A. Create ensembel class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Create_ensemble(object):\n",
    "    def __init__(self, n_splits, base_models):\n",
    "        self.n_splits = n_splits\n",
    "        self.base_models = base_models\n",
    "\n",
    "    def predict(self, X, y, T):\n",
    "        X = np.array(X)\n",
    "        y = np.array(y)\n",
    "        T = np.array(T)\n",
    "\n",
    "        folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, \n",
    "                                     random_state = random_state).split(X, y))\n",
    "\n",
    "        train_pred = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        test_pred = np.zeros((T.shape[0], len(self.base_models)* self.n_splits))\n",
    "        f1_scores = np.zeros((len(self.base_models), self.n_splits))\n",
    "        recall_scores = np.zeros((len(self.base_models), self.n_splits))\n",
    "        \n",
    "        test_col = 0\n",
    "        for i, clf in enumerate(self.base_models):\n",
    "            \n",
    "            for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "                \n",
    "                X_train = X[train_idx]\n",
    "                Y_train = y[train_idx]\n",
    "                X_valid = X[valid_idx]\n",
    "                Y_valid = y[valid_idx]\n",
    "                \n",
    "                clf.fit(X_train, Y_train)\n",
    "                \n",
    "                valid_pred = clf.predict(X_valid)\n",
    "                recall  = recall_score(Y_valid, valid_pred, average='macro')\n",
    "                f1 = f1_score(Y_valid, valid_pred, average='macro')\n",
    "                \n",
    "                recall_scores[i][j] = recall\n",
    "                f1_scores[i][j] = f1\n",
    "                \n",
    "                train_pred[valid_idx, i] = valid_pred\n",
    "                test_pred[:, test_col] = clf.predict(T)\n",
    "                test_col += 1\n",
    "                \n",
    "                print( \"Model- {} and CV- {} recall: {}, f1_score: {}\".format(i, j, recall, f1))\n",
    "            \n",
    "        return train_pred, test_pred, recall_scores, f1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 108 candidates, totalling 540 fits\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  23.2s\n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9548088064889919, total=  23.3s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  23.5s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9551892866340458, total=  23.5s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9556929417825863, total=  25.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9548088064889919, total=  33.0s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  32.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  33.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9553180530517641, total=  33.6s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9556929417825863, total=  34.4s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9548088064889919, total=  43.3s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total=  43.2s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9548088064889919, total=  30.7s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  48.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9553180530517641, total=  48.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9558217413704276, total=  49.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  27.6s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:  2.8min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  27.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  27.7s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9556929417825863, total=  26.5s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9548088064889919, total=  36.3s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  36.3s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  37.7s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9551892866340458, total=  38.7s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9556929417825863, total=  37.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9548088064889919, total=  47.2s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total=  46.5s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total=  45.4s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9548088064889919, total=  26.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9551892866340458, total=  45.2s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9556929417825863, total=  43.6s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  25.7s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  26.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9551892866340458, total=  26.9s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9560793405461102, total=  27.2s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9548088064889919, total=  36.3s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  37.1s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  37.2s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9551892866340458, total=  36.9s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9559505409582689, total=  36.8s\n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  43.9s\n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9548088064889919, total=  44.2s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV] max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total=  43.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9551892866340458, total=  43.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  26.0s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9549317537986093, total=  24.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9559505409582689, total=  42.2s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  23.7s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9551892866340458, total=  23.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9556929417825863, total=  24.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9546800566499292, total=  32.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9549317537986093, total=  32.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  34.2s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9551892866340458, total=  35.1s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9556929417825863, total=  34.5s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9546800566499292, total=  41.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9549317537986093, total=  40.1s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  39.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  24.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9551892866340458, total=  40.7s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9556929417825863, total=  41.3s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9549317537986093, total=  25.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  26.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9551892866340458, total=  26.6s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9556929417825863, total=  26.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9546800566499292, total=  35.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9549317537986093, total=  34.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  35.2s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9551892866340458, total=  35.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9556929417825863, total=  35.6s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9546800566499292, total=  44.5s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9549317537986093, total=  44.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total=  45.0s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9551892866340458, total=  45.1s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  27.7s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9549317537986093, total=  28.7s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9556929417825863, total=  46.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  28.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9551892866340458, total=  27.4s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9556929417825863, total=  27.0s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9546800566499292, total=  36.8s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9549317537986093, total=  36.9s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  37.1s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9551892866340458, total=  38.5s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9556929417825863, total=  36.5s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9546800566499292, total=  45.1s\n",
      "[CV] max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9549317537986093, total=  47.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total=  46.2s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9551892866340458, total=  46.2s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9545513068108665, total=  24.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  90 tasks      | elapsed: 14.3min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9549317537986093, total=  24.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9556929417825863, total=  42.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  25.4s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9550605202163276, total=  25.4s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.955564142194745, total=  25.9s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9544225569718038, total=  34.3s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9549317537986093, total=  33.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  34.4s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9550605202163276, total=  34.3s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.955564142194745, total=  33.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9545513068108665, total=  41.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9549317537986093, total=  41.9s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  42.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9550605202163276, total=  43.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9545513068108665, total=  26.1s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9549317537986093, total=  24.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.955564142194745, total=  41.6s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  23.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9550605202163276, total=  22.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.955564142194745, total=  22.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9544225569718038, total=  31.6s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9549317537986093, total=  32.7s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  34.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9550605202163276, total=  35.4s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.955564142194745, total=  34.4s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9545513068108665, total=  41.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9549317537986093, total=  40.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total=  40.3s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9550605202163276, total=  40.9s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9545513068108665, total=  25.6s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.955564142194745, total=  42.3s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9549317537986093, total=  26.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  25.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9550605202163276, total=  25.6s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.955564142194745, total=  27.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9544225569718038, total=  35.9s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9549317537986093, total=  36.0s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  36.6s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9550605202163276, total=  35.8s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.955564142194745, total=  36.5s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9545513068108665, total=  44.2s\n",
      "[CV] max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9549317537986093, total=  43.1s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total=  43.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9550605202163276, total=  43.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9549375563280545, total=  31.2s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=6, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.955564142194745, total=  42.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9558331187226371, total=  30.4s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9558331187226371, total=  31.6s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9553180530517641, total=  31.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9559505409582689, total=  30.2s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9549375563280545, total=  40.5s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9558331187226371, total=  40.1s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  41.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9553180530517641, total=  43.6s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9562081401339516, total=  44.1s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9549375563280545, total=  54.6s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9558331187226371, total=  52.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  50.7s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9553180530517641, total=  49.7s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9549375563280545, total=  30.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9560793405461102, total=  49.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9558331187226371, total=  31.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9558331187226371, total=  31.1s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  31.4s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9562081401339516, total=  30.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9549375563280545, total=  43.1s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9557043523049189, total=  43.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9557043523049189, total=  43.7s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9553180530517641, total=  43.5s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9563369397217929, total=  41.2s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9549375563280545, total=  51.8s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9558331187226371, total=  50.7s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9557043523049189, total=  51.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9553180530517641, total=  50.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9549375563280545, total=  29.4s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  30.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9563369397217929, total=  49.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9557043523049189, total=  31.8s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  32.8s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9563369397217929, total=  32.5s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9549375563280545, total=  43.8s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  43.6s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  44.3s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  43.4s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9563369397217929, total=  42.0s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9549375563280545, total=  51.9s\n",
      "[CV] max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total=  51.3s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  52.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  53.4s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9548088064889919, total=  31.8s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9550605202163276, total=  30.8s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9563369397217929, total=  52.8s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  30.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  30.8s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9559505409582689, total=  30.2s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9548088064889919, total=  40.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9550605202163276, total=  38.8s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  38.9s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  38.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9560793405461102, total=  38.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9548088064889919, total=  50.1s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9550605202163276, total=  50.4s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total=  51.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total=  52.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9548088064889919, total=  31.5s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9550605202163276, total=  31.1s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9560793405461102, total=  52.1s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  31.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  31.2s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9559505409582689, total=  31.3s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9548088064889919, total=  41.3s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9550605202163276, total=  40.5s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  40.4s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  39.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9560793405461102, total=  39.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9548088064889919, total=  49.5s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9550605202163276, total=  48.9s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total=  49.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total=  50.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9548088064889919, total=  29.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9550605202163276, total=  29.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9560793405461102, total=  49.1s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  29.6s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 213 tasks      | elapsed: 35.1min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  29.7s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9559505409582689, total=  30.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9548088064889919, total=  40.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9550605202163276, total=  40.1s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  40.9s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  41.3s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9560793405461102, total=  41.0s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9548088064889919, total=  51.5s\n",
      "[CV] max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9550605202163276, total=  51.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  51.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  51.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  29.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9549317537986093, total=  29.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9560793405461102, total=  50.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  30.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9553180530517641, total=  30.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9559505409582689, total=  29.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9546800566499292, total=  38.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9549317537986093, total=  37.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  38.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9551892866340458, total=  39.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9559505409582689, total=  40.1s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9546800566499292, total=  51.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9549317537986093, total=  53.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  53.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9551892866340458, total=  54.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  30.7s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9549317537986093, total=  30.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9560793405461102, total=  50.9s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  30.5s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  30.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9559505409582689, total=  30.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9546800566499292, total=  40.1s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9549317537986093, total=  39.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  39.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9551892866340458, total=  38.5s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9559505409582689, total=  37.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9546800566499292, total=  47.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9549317537986093, total=  47.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total=  47.5s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9551892866340458, total=  48.0s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  29.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9549317537986093, total=  29.7s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9560793405461102, total=  48.9s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  30.6s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9553180530517641, total=  30.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9559505409582689, total=  30.3s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9546800566499292, total=  40.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9549317537986093, total=  39.8s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  40.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9551892866340458, total=  39.4s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9559505409582689, total=  38.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9546800566499292, total=  48.2s\n",
      "[CV] max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9549317537986093, total=  47.9s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total=  49.5s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9551892866340458, total=  48.7s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9549375563280545, total=  34.0s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=8, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9560793405461102, total=  49.1s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  34.3s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  35.6s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  35.1s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.955564142194745, total=  35.0s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9550663061671173, total=  46.8s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9557043523049189, total=  45.5s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  46.4s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  46.0s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.955564142194745, total=  45.9s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9550663061671173, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9558331187226371, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9549375563280545, total=  36.4s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9557043523049189, total=  35.4s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9554353426069037, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  36.3s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  36.9s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9556929417825863, total=  36.0s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9549375563280545, total=  47.6s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9558331187226371, total=  46.7s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  45.6s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  45.8s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.955564142194745, total=  45.3s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9549375563280545, total=  58.2s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9558331187226371, total=  58.6s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9548088064889919, total=  36.8s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9558331187226371, total=  35.6s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.955564142194745, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  33.9s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  35.3s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9556929417825863, total=  35.4s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9548088064889919, total=  47.9s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9558331187226371, total=  48.1s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  48.7s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  48.8s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9556929417825863, total=  47.8s\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9548088064889919, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9558331187226371, total=  59.7s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  35.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9550605202163276, total=  35.1s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9556929417825863, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  36.9s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  36.8s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.955564142194745, total=  36.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9546800566499292, total=  49.0s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9550605202163276, total=  48.3s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  48.9s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  48.1s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.955564142194745, total=  47.2s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9546800566499292, total=  57.8s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9550605202163276, total=  57.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total=  59.0s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  37.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9550605202163276, total=  37.1s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.955564142194745, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  37.0s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  38.1s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.955564142194745, total=  38.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9546800566499292, total=  50.1s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9550605202163276, total=  50.9s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  49.7s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  50.8s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.955564142194745, total=  50.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9546800566499292, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9550605202163276, total=  58.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total=  58.5s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  36.2s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9550605202163276, total=  35.2s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.955564142194745, total=  59.2s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  35.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  35.9s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.955564142194745, total=  35.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9546800566499292, total=  49.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9550605202163276, total=  47.6s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  46.9s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  49.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.955564142194745, total=  47.4s\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9546800566499292, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9550605202163276, total= 1.0min\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  60.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total=  59.6s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  35.8s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9550605202163276, total=  35.5s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.955564142194745, total=  60.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  35.8s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9551892866340458, total=  35.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9554353426069037, total=  35.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9546800566499292, total=  48.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9550605202163276, total=  48.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  49.9s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9551892866340458, total=  49.1s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9554353426069037, total=  47.2s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9546800566499292, total=  57.6s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9550605202163276, total=  57.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total=  58.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9551892866340458, total=  59.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  36.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9550605202163276, total=  35.2s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.955564142194745, total=  59.7s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9555755858872006, total=  35.5s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9551892866340458, total=  36.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9554353426069037, total=  36.1s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9546800566499292, total=  47.7s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9550605202163276, total=  47.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  48.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9551892866340458, total=  47.8s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 384 tasks      | elapsed: 69.1min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9554353426069037, total=  47.7s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9546800566499292, total=  59.2s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9550605202163276, total=  57.9s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total=  58.2s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9551892866340458, total=  58.5s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  34.9s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9550605202163276, total=  35.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.955564142194745, total=  58.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  37.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9551892866340458, total=  36.9s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9554353426069037, total=  36.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9546800566499292, total=  49.0s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9550605202163276, total=  46.6s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  51.3s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9551892866340458, total=  52.4s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9554353426069037, total=  53.8s\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9546800566499292, total= 1.1min\n",
      "[CV] max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9550605202163276, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9551892866340458, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=10, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.955564142194745, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.954293807132741, total=  50.3s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  48.1s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9551892866340458, total=  48.2s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9555755858872006, total=  45.7s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=300, score=0.9540185471406492, total=  44.4s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9540363074546157, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  58.9s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9551892866340458, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=400, score=0.9538897475528079, total=  58.0s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9541650572936784, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9551892866340458, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9541650572936784, total=  40.5s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  39.4s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=5, n_estimators=500, score=0.9540185471406492, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9545454545454546, total=  41.0s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  40.5s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=300, score=0.9545337454920144, total=  39.7s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9541650572936784, total=  57.0s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9553180530517641, total=  56.8s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  59.1s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9550605202163276, total=  59.3s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=400, score=0.9544049459041731, total=  55.3s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9541650572936784, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9553180530517641, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9551892866340458, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9553180530517641, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9549375563280545, total=  40.2s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9555755858872006, total=  38.8s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=10, n_estimators=500, score=0.9542761463163318, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  39.9s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  40.1s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=300, score=0.9558217413704276, total=  39.4s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9549375563280545, total=  53.6s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  52.5s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  54.0s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  54.2s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=400, score=0.9556929417825863, total=  53.4s\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9549375563280545, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9557043523049189, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  38.9s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9548029873808911, total=  37.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=4, min_samples_split=15, n_estimators=500, score=0.9556929417825863, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9553180530517641, total=  39.2s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9558217413704276, total=  38.0s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=300, score=0.9554468194694824, total=  38.8s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9548088064889919, total=  52.5s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9546742209631728, total=  50.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9553180530517641, total=  51.8s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9554468194694824, total=  51.8s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=400, score=0.9559505409582689, total=  50.1s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9545513068108665, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9548029873808911, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9554468194694824, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  41.7s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9548029873808911, total=  40.2s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=5, n_estimators=500, score=0.9559505409582689, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  46.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9558217413704276, total=  44.2s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=300, score=0.9554468194694824, total=  45.5s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9548088064889919, total= 1.0min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9546742209631728, total=  54.9s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9553180530517641, total=  56.3s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9554468194694824, total=  56.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=400, score=0.9559505409582689, total=  56.7s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9545513068108665, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9548029873808911, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9554468194694824, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  41.9s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9548029873808911, total=  40.5s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=10, n_estimators=500, score=0.9559505409582689, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9553180530517641, total=  42.6s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9554468194694824, total=  43.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=300, score=0.9558217413704276, total=  42.6s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9548088064889919, total=  59.0s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9546742209631728, total=  56.4s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9553180530517641, total=  58.2s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9554468194694824, total=  58.3s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=400, score=0.9559505409582689, total=  59.2s\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9545513068108665, total= 1.3min\n",
      "[CV] max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9548029873808911, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total= 1.3min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9554468194694824, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9546800566499292, total=  42.3s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9551892866340458, total=  40.7s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=8, min_samples_split=15, n_estimators=500, score=0.9559505409582689, total= 1.2min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9557043523049189, total=  41.9s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9553180530517641, total=  41.7s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=300, score=0.9554353426069037, total=  40.8s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9548088064889919, total=  55.2s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9551892866340458, total=  52.1s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9555755858872006, total=  53.0s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9553180530517641, total=  53.3s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=400, score=0.9554353426069037, total=  52.5s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9546800566499292, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9550605202163276, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9555755858872006, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9553180530517641, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9546800566499292, total=  39.9s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9551892866340458, total=  38.3s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=5, n_estimators=500, score=0.9554353426069037, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9557043523049189, total=  40.0s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9553180530517641, total=  40.7s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=300, score=0.9554353426069037, total=  39.8s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9548088064889919, total=  54.0s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9551892866340458, total=  51.6s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9555755858872006, total=  53.3s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9553180530517641, total=  53.2s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=400, score=0.9554353426069037, total=  52.5s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9546800566499292, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9550605202163276, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9555755858872006, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9553180530517641, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9546800566499292, total=  40.4s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9551892866340458, total=  38.7s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=10, n_estimators=500, score=0.9554353426069037, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9557043523049189, total=  40.1s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9553180530517641, total=  40.0s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=300, score=0.9554353426069037, total=  39.3s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9548088064889919, total=  53.1s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9551892866340458, total=  51.4s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9555755858872006, total=  53.1s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9553180530517641, total=  53.0s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=400, score=0.9554353426069037, total=  52.1s\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9546800566499292, total= 1.1min\n",
      "[CV] max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500 \n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9550605202163276, total= 1.1min\n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9555755858872006, total= 1.1min\n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9553180530517641, total=  58.0s\n",
      "[CV]  max_depth=20, min_samples_leaf=12, min_samples_split=15, n_estimators=500, score=0.9554353426069037, total=  44.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 540 out of 540 | elapsed: 106.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5,\n",
       "        random_state=<mtrand.RandomState object at 0x112987c60>,\n",
       "        shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False,\n",
       "            random_state=<mtrand.RandomState object at 0x112987c60>,\n",
       "            verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [6, 8, 10, 20], 'min_samples_split': [5, 10, 15], 'min_samples_leaf': [4, 8, 12], 'n_estimators': [300, 400, 500]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=4)"
      ]
     },
     "execution_count": 464,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "cv = StratifiedKFold(n_splits = 5, shuffle=True, random_state = random_state)\n",
    "\n",
    "rdf = RandomForestClassifier(random_state = random_state) \n",
    "scoring = {'Recall': make_scorer(recall_score),\n",
    "           'f1_score': make_scorer(f1_score)\n",
    "          }\n",
    "\n",
    "params = {'max_depth': [6, 8, 10, 20], \n",
    "              'min_samples_split': [5, 10, 15],\n",
    "              'min_samples_leaf' : [4, 8, 12],\n",
    "              'n_estimators' : [300, 400, 500]\n",
    "             }\n",
    "\n",
    "grid_clf = GridSearchCV(estimator = rdf, param_grid = params, cv = cv, n_jobs=-1, verbose=4)\n",
    "grid_clf.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=8, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=4, min_samples_split=10,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=300, n_jobs=1,\n",
      "            oob_score=False,\n",
      "            random_state=<mtrand.RandomState object at 0x113c978b8>,\n",
      "            verbose=0, warm_start=False)\n",
      "{'max_depth': 8, 'min_samples_leaf': 4, 'min_samples_split': 10, 'n_estimators': 300}\n"
     ]
    }
   ],
   "source": [
    "print(grid_clf.best_estimator_)\n",
    "print(grid_clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [],
   "source": [
    "rdf = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
    "            max_depth=8, max_features='auto', max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=4, min_samples_split=10,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=300, n_jobs=-1,\n",
    "            oob_score=False,\n",
    "            random_state=random_state,\n",
    "            verbose=0, warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_models = [rdf1]\n",
    "n_splits = 5\n",
    "lgb_stack = Create_ensemble(n_splits = n_splits, base_models = base_models)        \n",
    "\n",
    "xtrain = train.drop(['label'], axis=1)\n",
    "ytrain = train['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model- 0 and CV- 0 recall: 0.34028137427570854, f1_score: 0.33922009556527327\n",
      "Model- 0 and CV- 1 recall: 0.34372475097091176, f1_score: 0.3458628316430164\n",
      "Model- 0 and CV- 2 recall: 0.3475920192638459, f1_score: 0.35352234374773833\n",
      "Model- 0 and CV- 3 recall: 0.34035087719298246, f1_score: 0.33952025951020165\n",
      "Model- 0 and CV- 4 recall: 0.34385964912280703, f1_score: 0.34606151198079443\n"
     ]
    }
   ],
   "source": [
    "train_pred, test_pred, recall_scores, f1_scores = lgb_stack.predict(xtrain, ytrain, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. The F-1 score of the model 0.3448836391344175\n",
      "\n",
      "2. The recall score of the model 0.343163299137863\n",
      "\n",
      "3. Classification report \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.96      1.00      0.98     37064\n",
      "          2       0.76      0.03      0.05      1426\n",
      "          3       0.20      0.00      0.01       339\n",
      "\n",
      "avg / total       0.94      0.96      0.93     38829\n",
      " \n",
      "\n",
      "4. Confusion matrix \n",
      " [[37060     4     0]\n",
      " [ 1384    38     4]\n",
      " [  330     8     1]] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('1. The F-1 score of the model {}\\n'.format(f1_score(ytrain, train_pred, average='macro')))\n",
    "print('2. The recall score of the model {}\\n'.format(recall_score(ytrain, train_pred, average='macro')))\n",
    "print('3. Classification report \\n {} \\n'.format(classification_report(ytrain, train_pred)))\n",
    "print('4. Confusion matrix \\n {} \\n'.format(confusion_matrix(ytrain, train_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 573,
   "metadata": {},
   "outputs": [],
   "source": [
    "tpred = pd.DataFrame(test_pred)\n",
    "final_tpred = tpred.mode(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  2.,  3.])"
      ]
     },
     "execution_count": 576,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(final_tpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_tpred.to_csv('predicted_labels_1.csv', index=False, header= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The datasets used in problem-1 are highly unbalance. Therefore, all the evaluation metrics shows the expected random performance. Due to the unbalance data, the probabilities for minor classes (class-2 and 3) are inaccurate. But we can still get good predictions by choosing a more appropriate probability cutoff. In the problem-3 section, I will imporve the model performance by choosing a cutoff by ovserving the minor class probability distribution and ROC curve and by setting unequal importance of the class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modified Ensemble"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this problem the labels have unequal importance, in the sense that we want to penalize the model most if it misclassified label 3, a little less for 2 and the least for label 1. Additionally, in case of a misclassification, it is preferable to over-predict a label than under-predict (i.e. misclassifying label 3 as 2 is worse than misclassifying label 2 as 3). To implement the above constraints I would rebuild problem-1 model in following steps:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<li> <b>Step 1: </b>Predict probabilities instead of actual prediction.\n",
    "<li> <b>Step 2: </b>Set the class weight.\n",
    "<li> <b>Step 3: </b>Get probability distribution of minor class.\n",
    "<li> <b>Step 4: </b>From the ROC curve and probability distribution obtain probability thresholds for classes.\n",
    "<li> <b>Step 5: </b>Finally use the threshold to over-predict a label than under-predict."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Create_ensemble(object):\n",
    "    def __init__(self, n_splits, base_models):\n",
    "        self.n_splits = n_splits\n",
    "        self.base_models = base_models\n",
    "\n",
    "    def predict(self, X, y, T):\n",
    "        X = np.array(X)\n",
    "        y = np.array(y)\n",
    "        T = np.array(T)\n",
    "        no_class = len(np.unique(y))\n",
    "\n",
    "        folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, \n",
    "                                     random_state = random_state).split(X, y))\n",
    "\n",
    "        train_proba = np.zeros((X.shape[0], no_class))\n",
    "        test_proba = np.zeros((T.shape[0], no_class))\n",
    "        \n",
    "        train_pred = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        test_pred = np.zeros((T.shape[0], len(self.base_models)* self.n_splits))\n",
    "        f1_scores = np.zeros((len(self.base_models), self.n_splits))\n",
    "        recall_scores = np.zeros((len(self.base_models), self.n_splits))\n",
    "        \n",
    "        test_col = 0\n",
    "        for i, clf in enumerate(self.base_models):\n",
    "            \n",
    "            for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "                \n",
    "                X_train = X[train_idx]\n",
    "                Y_train = y[train_idx]\n",
    "                X_valid = X[valid_idx]\n",
    "                Y_valid = y[valid_idx]\n",
    "                \n",
    "                clf.fit(X_train, Y_train)\n",
    "                \n",
    "                valid_pred = clf.predict(X_valid)\n",
    "                recall  = recall_score(Y_valid, valid_pred, average='macro')\n",
    "                f1 = f1_score(Y_valid, valid_pred, average='macro')\n",
    "                \n",
    "                recall_scores[i][j] = recall\n",
    "                f1_scores[i][j] = f1\n",
    "                \n",
    "                train_pred[valid_idx, i] = valid_pred\n",
    "                test_pred[:, test_col] = clf.predict(T)\n",
    "                test_col += 1\n",
    "                \n",
    "                ## Probabilities\n",
    "                valid_proba = clf.predict_proba(X_valid)\n",
    "                train_proba[valid_idx, :] = valid_proba\n",
    "                test_proba  += clf.predict_proba(T)\n",
    "                \n",
    "                print( \"Model- {} and CV- {} recall: {}, f1_score: {}\".format(i, j, recall, f1))\n",
    "                \n",
    "            test_proba /= self.n_splits\n",
    "            \n",
    "        return train_proba, test_proba, train_pred, test_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### B. Set class weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "class_weight = dict({1:1.9, 2:35, 3:180})\n",
    "\n",
    "rdf = RandomForestClassifier(bootstrap=True, class_weight=class_weight, criterion='gini',\n",
    "            max_depth=8, max_features='auto', max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=4, min_samples_split=10,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=300, n_jobs=-1,\n",
    "            oob_score=False,\n",
    "            random_state=random_state,\n",
    "            verbose=0, warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_models = [rdf]\n",
    "n_splits = 5\n",
    "lgb_stack = Create_ensemble(n_splits = n_splits, base_models = base_models)        \n",
    "\n",
    "xtrain = train.drop(['label'], axis=1)\n",
    "ytrain = train['label'].values\n",
    "# ytrain = label_binarize(Y, classes=[0, 1, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model- 0 and CV- 0 recall: 0.5750114509904545, f1_score: 0.3881526328257434\n",
      "Model- 0 and CV- 1 recall: 0.5889211725642496, f1_score: 0.38437185414981717\n",
      "Model- 0 and CV- 2 recall: 0.5623819310538191, f1_score: 0.3893834041357433\n",
      "Model- 0 and CV- 3 recall: 0.5834766817616167, f1_score: 0.38175751254315576\n",
      "Model- 0 and CV- 4 recall: 0.5679677139058309, f1_score: 0.3888236572862818\n"
     ]
    }
   ],
   "source": [
    "train_proba, test_proba, train_pred, test_pred = lgb_stack.predict(xtrain, ytrain, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. The F-1 score of the model 0.3864621253731387\n",
      "\n",
      "2. The recall score of the model 0.5756000234914537\n",
      "\n",
      "3. Classification report \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.98      0.79      0.88     37064\n",
      "          2       0.12      0.49      0.19      1426\n",
      "          3       0.05      0.44      0.09       339\n",
      "\n",
      "avg / total       0.94      0.78      0.85     38829\n",
      " \n",
      "\n",
      "4. Confusion matrix \n",
      " [[29434  4980  2650]\n",
      " [  399   699   328]\n",
      " [   90    99   150]] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('1. The F-1 score of the model {}\\n'.format(f1_score(ytrain, train_pred, average='macro')))\n",
    "print('2. The recall score of the model {}\\n'.format(recall_score(ytrain, train_pred, average='macro')))\n",
    "print('3. Classification report \\n {} \\n'.format(classification_report(ytrain, train_pred)))\n",
    "print('4. Confusion matrix \\n {} \\n'.format(confusion_matrix(ytrain, train_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Probability distribution for all classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 584,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1QAAAEUCAYAAAAspncYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlY1WX+//EXi6ACLoxYmdFgaWqOuVOJlJaDpmaigjKR\nppVZF6XjFLhhZmpm0aKZoy3zHdQSt8yxbAoXDBONSouyxd9Ig0uSuAAqIuf+/dHlGZFFOJ6FA8/H\ndXVdnPvc5/N5nwO+O+/PfX/u28MYYwQAAAAAqDZPVwcAAAAAAO6KggoAAAAAbERBBQAAAAA2oqAC\nAAAAABtRUAEAAACAjSioAAAAAMBGFFR2lpOTo3bt2mnw4MHW/+69916tXr36io89btw4rV27VpI0\nePBgnTp1qsK++fn5euCBB6p9jk2bNik2NrbK/TMyMjRw4MBqn8dWq1ev1qOPPurQc8TGxmrTpk3V\nes2CBQv07LPPlvvcww8/rJ9//rnUZ/Xqq6/q/ffflyQtXLhQn376aZl24HLIN46xd+9ejRgxQoMH\nD9agQYO0fv16h50rISFBb731VrVes3btWo0bN67c56ZOnaodO3YoJydHnTt3liS9++67WrJkiSRp\n1apVWr58eZl2oDLkGsfYuXOnIiMjde+99yoqKkp79+512Ln4buNY3q4OoDaqX79+qf8B//rrrxo4\ncKA6dOigtm3b2uUcl/sf/MmTJ/XNN9/Y5Vw1wYkTJ5SUlKQPPvhAoaGhrg6nWpYuXSpJOnbsmLXt\nySeftP6ckZGhG2+8sUw7UBXkG/syxuiJJ57QnDlzdPvtt+vIkSMaMmSIbrnlFv3xj390dXiXNXv2\nbEm/fwG+YOTIkdafMzMz1bp16zLtwOWQa+zr3Llzmjhxot566y21b99eW7Zs0VNPPaWPP/7Y1aFV\nCd9tSqOgcoKrrrpK119/vQ4cOKDvvvtOq1ev1pkzZ+Tv76/k5GStWrVK7777riwWi5o0aaLp06fr\nhhtu0K+//qqEhAQdPXpULVq0KPVHe9NNN+nzzz9XYGCg/v73v2vdunXy9vbW9ddfr+eff16TJ0/W\n2bNnNXjwYK1du1YHDhzQ7NmzdeLECZWUlCg2NlbDhg2T9PuVgw0bNqhJkya6/vrrK3wfq1ev1jvv\nvCNPT081bdpU8+bNK/X8f/7zHz377LM6ffq0jh49qrZt2+qVV16Rr6+vXnvtNX3yySeqV6+emjZt\nqrlz56p58+YVtl/qo48+UvPmzfX0009r27ZtFcbYvn17jRo1ShkZGTp9+rT++te/6s9//rPWrl1b\n5nN//fXXtXHjRnl5eSkkJETTp09XUFCQJOmTTz7RkiVLdPbsWQ0aNEjjx4+XJC1evFiffvqpioqK\ndObMGcXHx6tv376SpP379+svf/mLTp48qXbt2mnGjBny9/dXnz599Oqrr5aKMyEhQa1bt1b9+vX1\n7bff6oUXXpCXl5dSU1PVunVrjR07Vvv37y/3d1ZYWKjJkycrOztbnp6euvnmm/Xss8/K05MBZ5Bv\nrjTfnDt3To8//rhuv/12SdLVV1+tpk2b6siRI2UKqj59+ujuu+/WF198ofz8fD344IOKiYlRRkaG\nZs+erYYNG+r06dNavXq11q1bp+TkZHl6eqpZs2aaPn26QkJCJP1e5Hz88ccqKChQz549FR8fL29v\nb61evVorV65UcXGxTp48qYcfflgxMTGSpNzcXI0dO1ZHjx7Vtddeq1mzZikoKEixsbH6y1/+og4d\nOljjXLBggY4fP67bbrtNmzdvVnp6uurXr6+8vDwdP35ciYmJ+vXXX/Xss8/q8OHDKi4u1oABA/To\no4/q/PnzmjVrlr788kvVq1dPLVu21Ny5c+Xn51fNv0zUNuSaK8s1Pj4+SktLU7169WSM0X//+181\nbdq03Bj5buMGDOzqv//9r+nUqVOpti+//NJ0797dHDp0yKxZs8Z0797d5OfnG2OMycjIMDExMeb0\n6dPGGGO2b99u+vfvb4wx5rHHHjMvv/yyMcaYAwcOmE6dOpk1a9YYY4xp06aNOXbsmPn000/Nn//8\nZ3PixAljjDFz5swxixYtKhVHcXGxueeee8y3335rjDHm1KlTpn///uarr74yn3zyibnnnntMfn6+\nKS4uNo888oi5//77y7yv77//3oSGhppDhw4ZY4x55513zPTp083OnTvNgAEDjDHGPP/88+b99983\nxhhz7tw5M3DgQLNp0yZz6NAh06VLF1NUVGSMMeatt94yn3zySYXtlVmzZo155JFHKny+TZs25o03\n3rDG3LVrV3Ps2LEyn/vq1atNdHS0KSwsNMYY89prr5kxY8YYY4y5//77zbhx40xxcbHJz883/fr1\nM1u3bjU5OTkmNjbWnDlzxhhjzL/+9S8zcOBA6+vvvPNOc+zYMWOxWMykSZPMCy+8YIwxpnfv3mbv\n3r2lPqv4+Hjz5ptvWs/30UcflWqv7He2bt06a6znz583U6dONQcOHKj0c0PtRL5xbL4xxpj33nvP\n3HHHHdZ/9xfr3bu3mT59urFYLObw4cMmNDTU7Nu3z+zcudO0bdvW5OTkGGOM2bFjh7n77rvNsWPH\njDG/57H+/fsbi8Vi4uPjzZAhQ0xhYaEpKioy999/v1m+fLkpKCgwUVFRJi8vzxhjzFdffWX9jNes\nWWM6depk/Xf/0ksvmSeffNIY8798cvHv5LXXXjMzZ840xpTOPRe3x8bGmtTUVGOMMWfPnjWxsbFm\n48aNZvfu3aZfv37GYrEYY4x54YUXTGZm5mU/N9Qu5BrH5Zrc3FwTFhZmbr755gr78d2m5mOEygEu\nXD2RpJKSEjVt2lTz58/XNddcI+n3KzD+/v6SpK1btyo7O1sjRoywvv7kyZM6ceKEduzYofj4eEnS\n9ddfX+5Ut88//1z9+vVT48aNJUmTJ0+WVHq6x4EDB/TLL79oypQppWL87rvvtH//fvXt29caz9Ch\nQ5WcnFzuecLCwqzvYfTo0ZJ+H9K94KmnnlJ6erqWLl2qAwcO6OjRozp9+rSuuuoqtW3bVkOGDFF4\neLjCw8N12223yWKxlNt+pe6//35JUtu2bdWmTRvt3r1bUunPPS0tTZGRkWrYsKEk6YEHHtDixYt1\n7tw5SdKwYcPk7e0tf39/RUREaMeOHbrjjjs0b948bdiwQdnZ2dqzZ48KCwut5+3bt68CAwOtn+ML\nL7xg83uo7HfWq1cvvfzyy4qNjdXtt9+uUaNGVXr1DbUb+cZx+WbJkiX65z//qTfffFP169cvt09M\nTIw8PDx09dVXq1evXkpPT9fNN9+sa665Rtdee60kafv27brnnnus+SEyMlKzZ8+2fm6DBw+25qJ7\n771X27ZtU0xMjBYvXqxt27bpwIED2rdvn06fPm097+233279dz9s2DDrVXlbnD59Wrt379bJkyet\nV5xPnz6tffv2KSwsTF5eXho+fLjCwsIUERGhjh072nwuuC9yjWNyTbNmzbR9+3ZlZWVp9OjRuuGG\nG6yj1xfju03NRkHlAJfOM77UhT90SbJYLBo8eLCeeuop6+OjR4+qcePG8vDwkDHG2tfbu+yvy8vL\nSx4eHtbHp06dKnNDZ0lJiRo1alQqpt9++00BAQGaP39+qXN4eXmVG/Ol5zl79qwOHjxYqs9f//pX\nlZSUqH///rrzzjt1+PBhGWPk6empZcuW6ZtvvtHnn3+uOXPmKDQ0VNOmTSu3/bbbbtNrr70mSWre\nvLl1nm5VXfweLBaL9fHFn/vF7/lCv/Pnz5d7DGOMvL29lZWVpccee0yjR49Wz5491b17d82cObPS\n19iqst+Zr6+vPvnkE2VkZGjnzp168MEHNW3aNPXr18/m88F9kW/sn2/OnTunhIQE/fzzz3rvvffU\nsmXLCj/fiz8ni8VinZ5SWb650HYh51z6OXh7e+vIkSOKjo5WVFSUunbtqn79+mnLli3lfnZXmm8s\nFouMMXrvvffUoEEDSVJeXp58fX3l5+en9evX68svv9TOnTs1YcIEPfDAA9Yvnqg7yDX2zTVJSUna\nuXOndWrdzTffrLZt2+rHH38st6Diu03N5iYTE2uvnj17auPGjTp69Kik31ddGjVqlCSpV69eWrly\npSTp0KFDpa6YXHD77bfrk08+UUFBgaTf58r/4x//kLe3t0pKSmSMUUhIiHx9fa1/wIcPH9bAgQP1\n7bffqlevXtq0aZNOnToli8VSYbIMDQ3V559/bo3zvffe0/z580v1+eyzz/T444/rnnvukYeHh/bs\n2aOSkhLt27dPAwcO1A033KBx48Zp9OjR+uGHHypsv+uuu7R+/XqtX7++2sWUJOtKMllZWfrPf/6j\n7t27l+kTFhamtWvXWq/4Jicnq3v37vLx8bEewxijkydP6qOPPlJ4eLh2796tDh066MEHH1SPHj2U\nmpqqkpIS6zE3b96skydPqqSkRCtXrlR4eHiV4vXy8iqV8CRV+jtbsWKFJk+erLCwMD311FMKCwvT\nTz/9VO3PCXUP+aZq+eaJJ55QQUHBZYsp6X/55tChQ0pPTy/3331YWJg+/PBD5eXlSZLWrFlT6r6O\njRs36ty5cyoqKtLatWsVHh6ub7/9VoGBgXrsscfUq1cvazF1IedkZGTo0KFD1t/jleQbf39/derU\nSe+8846k37+8jhw5UqmpqdqyZYtGjx6tzp07Ky4uTvfdd5/27dtXpXOh7iLXXD7XeHp6asqUKcrM\nzJQk/fTTT/p//+//6ZZbbik3Vr7b1GyMULlYr1699PDDD2vMmDHy8PCQv7+/Fi5cKA8PD82YMUOT\nJ09W//79dfXVV5e7is4dd9yhn3/+2bpa04033qhZs2apQYMGat++vfr37693331XixYt0uzZs/Xm\nm2/q/PnzevLJJ9W1a1dJ0g8//KChQ4eqUaNGatu2rY4fP17mPDfddJOeeuopPfTQQ5KkoKAgzZkz\nRwcOHLD2mThxoh5//HE1btxYDRo0UPfu3fXLL79o+PDh6t+/v4YOHaqGDRuqfv36mjZtmtq2bVtu\n+5X68ssvlZKSIovFopdfftk6ZeBiw4YN0+HDhzV8+HBZLBZdf/31evHFF63PBwQEKDIyUmfPntX9\n99+v0NBQ3XDDDfr3v/+te+65R/Xq1dNtt92mkydPWhP+heR56tQpde3aVY888kiV4u3du7fmzZun\n4uJia5uPj0+Fv7N27dpp165duueee9SgQQO1aNHCpmVkUfeQby6fbzIzM7Vlyxb98Y9/LLUK3t/+\n9jf16tWrTP+cnBxrrpg2bZpatWql3NzcUn169uyp0aNHa9SoUbJYLNYb7i+MZrVs2VIjR47U6dOn\n1bdvXw0ZMkRnz57V6tWr1a9fPzVo0EAdO3ZUYGCgsrOzJUlt2rTRlClT9Ntvv6lVq1YVLm18qfDw\ncM2aNatM+4svvqhZs2Zp0KBBOnfunAYOHKh7771XJSUlSktL08CBA9WwYUM1bty43NcDFyPXXD7X\n+Pn56fXXX9ecOXN0/vx5+fj46MUXX9TVV19d7mfKd5uazcOUNxcBcFMXrxAEAI50YZWrP/3pT64O\nBUAtxnebmo8pfwAAAABgI0aoAAAAAMBGjFABAAAAgI0oqAAAAADARg4tqPbs2aPY2NhSbRs2bFB0\ndLT1cUpKiiIjIxUVFWVdFjYvL09jxoxRTEyMJkyYoDNnzlTYFwAAAABcxWHLpi9dulQffPCBdZNA\nSfruu++0evVq68Zjubm5Sk5O1po1a1RUVKSYmBj17NlTixYt0sCBAxUZGaklS5Zo5cqVGjBgQLl9\nL6ytX5Hz50t0/PjpSvvUFE2bNiRWByBWxwgKCnB1CDUO+cYxiNUx3ClW8k1p5BrHIFbHcKdYbc01\nDhuhCg4O1oIFC6yPjx8/rqSkJE2ZMsXatnfvXnXu3Fk+Pj4KCAhQcHCw9u3bp8zMTOt+H+Hh4dqx\nY0eFfS/H27v83bFrImJ1DGKFs7jT749YHYNY4Qzu9LsjVscg1prFYSNUERERysnJkfT7zu5Tp07V\n5MmT5evra+1TUFCggID/VYJ+fn4qKCgo1e7n56f8/PwK+1aFO13ZIlbHIFYAAAA4gsMKqotlZWUp\nOztbzzzzjIqKivTzzz9r9uzZuvXWW1VYWGjtV1hYqICAAPn7+6uwsFD169dXYWGhGjVqZG27tG9V\n5Obm2/09OUJQUACxOgCxOgaFHwAAgJNW+evYsaM2btyo5ORkJSUl6cYbb9TUqVPVsWNHZWZmqqio\nSPn5+dq/f7/atGmjLl26aNu2bZKktLQ0de3atcK+AAAAAOAqLl02PSgoSLGxsYqJidGoUaM0ceJE\n+fr6avz48dq4caNGjBihr776Svfff3+FfQFAYlVRAADgGg6d8teyZUulpKRU2hYVFaWoqKhSfZo1\na6a33nqrzPHK6wsANWVVUQAAUPewsS8At1dTVhUFAAB1j1MWpQAAR2JVUdsQq2MQKwDULRRUAGoV\nVhWtGndbUZJY7c/dYgWAmoqCCqgjxjy/2W7Hejuhj92OZW8XVhWVpJycHP31r3/V1KlTlZubq1de\neUVFRUU6d+5cmVVFIyMjS60qWl7fyxk0ab3d3kdN/owBuJ49cjp5BrAPCioAdcLFK4UaY0qtKhof\nH6+UlBQ1bdpUL730kho2bFhuXwAAgEtRUAGoFVhVFAAAuAKr/AEAAACAjSioAAAAAMBGFFQAAAAA\nYCMKKgAAAACwEQUVAAAAANiIggoAAAAAbERBBQAAAAA2oqACAAAAABtRUAEAAACAjSioAAAAqmHP\nnj2KjY0t1bZhwwZFR0dbH6ekpCgyMlJRUVHasmWLJCkvL09jxoxRTEyMJkyYoDNnzlTYF4D78HZ1\nAAAAAO5i6dKl+uCDD9SgQQNr23fffafVq1fLGCNJys3NVXJystasWaOioiLFxMSoZ8+eWrRokQYO\nHKjIyEgtWbJEK1eu1IABA8rt6+Pj46q3CKCaGKECAACoouDgYC1YsMD6+Pjx40pKStKUKVOsbXv3\n7lXnzp3l4+OjgIAABQcHa9++fcrMzFSvXr0kSeHh4dqxY0eFfQG4D0aoAAAAqigiIkI5OTmSpJKS\nEk2dOlWTJ0+Wr6+vtU9BQYECAgKsj/38/FRQUFCq3c/PT/n5+RX2dYagoIDLd3Kj89gDsTqGO8Vq\nCwoqAAAAG2RlZSk7O1vPPPOMioqK9PPPP2v27Nm69dZbVVhYaO1XWFiogIAA+fv7q7CwUPXr11dh\nYaEaNWpkbbu0rzPk5uY7/BxBQQFOOY89EKtjuFustmDKHwAAgA06duyojRs3Kjk5WUlJSbrxxhs1\ndepUdezYUZmZmSoqKlJ+fr7279+vNm3aqEuXLtq2bZskKS0tTV27dq2wLwD3wQgVAACAHQUFBSk2\nNlYxMTEyxmjixIny9fXV+PHjFR8fr5SUFDVt2lQvvfSSGjZsWG5fAO6DggoAAKAaWrZsqZSUlErb\noqKiFBUVVapPs2bN9NZbb5U5Xnl9AbgPh075u3ifhu+//14xMTGKjY3V2LFj9dtvv0linwYAAAAA\n7sthI1SX7tMwe/ZsTZ8+Xe3atdN7772npUuX6qGHHmKfBgAAAABuy2EjVJfu05CUlKR27dpJ+n2Z\nUV9fX/ZpAAAAAODWHDZCdfE+DZLUvHlzSdKXX36pZcuWafny5dq+fbtT9mlwp7XvidUxiNW+3CFG\nAAAAZ3DqohQffvih3njjDS1ZskSBgYEV7r1g730a3Gnte2K1P2K1v9zcfIoqAAAAOXEfqvXr12vZ\nsmVKTk7WddddJ0ns0wDAblgEBwAAuIJTRqhKSko0e/ZsXXPNNYqLi5Mkde/eXU888QT7NAC4YiyC\nAwAAXMWhBdXFezLs2rWr3D7s0wDgSl1YBOfpp5+W9PsiOBfu2yxvERwfH59Si+CMGzdO0u+L4CQl\nJem6664rt2/Hjh1d9h4BAEDNxMa+ANxeTVoEx16ccY+aO90HR6yO4U6xAkBNRUEFoFZy1SI49uLo\nxUncZQEUiVgdxd1iBYCaymmLUgCAs7AIDgAAcBZGqADUKiyCAwAAnImCCkCtwCI4AADAFZjyBwAA\nAAA2oqACAAAAABtRUAEAAACAjSioAAAAAMBGFFQAAAAAYCMKKgAAAACwEQUVAABANezZs0exsbGS\npO+//14xMTGKjY3V2LFj9dtvv0mSUlJSFBkZqaioKG3ZskWSlJeXpzFjxigmJkYTJkzQmTNnKuwL\nwH2wDxUAAEAVLV26VB988IEaNGggSZo9e7amT5+udu3a6b333tPSpUv10EMPKTk5WWvWrFFRUZFi\nYmLUs2dPLVq0SAMHDlRkZKSWLFmilStXasCAAeX29fHxcfE7BVBVjFABAABUUXBwsBYsWGB9nJSU\npHbt2kmSSkpK5Ovrq71796pz587y8fFRQECAgoODtW/fPmVmZqpXr16SpPDwcO3YsaPCvgDcByNU\nAACbjXl+s92O9XZCH7sdC3CUiIgI5eTkWB83b95ckvTll19q2bJlWr58ubZv366AgABrHz8/PxUU\nFKigoMDa7ufnp/z8/FJtF/d1hqCggMt3cqPz2AOxOoY7xWoLCioAAIAr8OGHH+qNN97QkiVLFBgY\nKH9/fxUWFlqfLywsVEBAgLW9fv36KiwsVKNGjSrs6wy5ufkOP0dQUIBTzmMPxOoY7harLZjyBwAA\nYKP169dr2bJlSk5O1nXXXSdJ6tixozIzM1VUVKT8/Hzt379fbdq0UZcuXbRt2zZJUlpamrp27Vph\nXwDugxEqAAAAG5SUlGj27Nm65pprFBcXJ0nq3r27nnjiCcXGxiomJkbGGE2cOFG+vr4aP3684uPj\nlZKSoqZNm+qll15Sw4YNy+0LwH1QUAEAAFRDy5YtlZKSIknatWtXuX2ioqIUFRVVqq1Zs2Z66623\nqtQXgPtgyh8AAAAA2IiCCgAAAABsREEFAAAAADaioAIAAAAAGzm0oNqzZ49iY2MlSdnZ2Ro5cqRi\nYmI0Y8YMWSwWSdLChQs1bNgwjRgxQnv37q12XwAAAABwFYcVVEuXLtW0adNUVFQkSZo7d64mTJig\nFStWyBij1NRUZWVladeuXVq1apWSkpI0c+bMavcFAAAAAFdxWEEVHBysBQsWWB9nZWWpR48ekqTw\n8HDt2LFDmZmZCgsLk4eHh1q0aKGSkhLl5eVVqy8AAAAAuIrD9qGKiIhQTk6O9bExRh4eHpIkPz8/\n5efnq6CgQE2aNLH2udBenb6BgYGXjSUoKMBeb8vhiNUxiNW+3CFGAAAAZ3Daxr6env8bDCssLFSj\nRo3k7++vwsLCUu0BAQHV6lsVubn5dngHjhcUFECsDkCs9pebm1/jiqo9e/boxRdfVHJysrKzs5WQ\nkCAPDw+1bt1aM2bMkKenpxYuXKitW7fK29tbU6ZMUceOHavVFwAA4FJOW+Wvffv2ysjIkCSlpaWp\nW7du6tKliz777DNZLBYdOnRIFotFgYGB1eoLANyzCQAAXMVpI1Tx8fGaPn26kpKS1KpVK0VERMjL\ny0vdunVTdHS0LBaLEhMTq90XAC7cs/n0009LKnvPZnp6ukJCQqp0z2ZlfbmIAwAALuXQgqply5ZK\nSUmRJIWEhGjZsmVl+sTFxSkuLq5UW3X6AkBNumfTXpwxpbKmTdusLJ6aFmtliBUA6hanjVABgLO4\n8p5Ne3H0vXQ18X69iuKpibFWhFgdg8IPQE3mtHuoAMBZuGcTAAA4CyNUddyY5zfb9XhvJ/Sx6/EA\nW3DPJgAAcBYKKgC1AvdsAgAAV2DKHwAAAADYiIIKAAAAAGxEQQUAAAAANqKgAgAAAAAbUVABAABU\nw549exQbGytJys7O1siRIxUTE6MZM2bIYrFIkhYuXKhhw4ZpxIgR2rt3b7X7AnAfFFQAAABVtHTp\nUk2bNk1FRUWSpLlz52rChAlasWKFjDFKTU1VVlaWdu3apVWrVikpKUkzZ86sdl8A7oOCCgAAoIqC\ng4O1YMEC6+OsrCz16NFDkhQeHq4dO3YoMzNTYWFh8vDwUIsWLVRSUqK8vLxq9QXgPtiHCgAAoIoi\nIiKUk5NjfWyMkYeHhyTJz89P+fn5KigoUJMmTax9LrRXp29gYKDD30tQUIDDz+HM89gDsTqGO8Vq\nCwoqAAAAG3l6/m+yT2FhoRo1aiR/f38VFhaWag8ICKhWX2fIzc13+DmCggKcch57IFbHcLdYbcGU\nPwAAABu1b99eGRkZkqS0tDR169ZNXbp00WeffSaLxaJDhw7JYrEoMDCwWn0BuA9GqAAAAGwUHx+v\n6dOnKykpSa1atVJERIS8vLzUrVs3RUdHy2KxKDExsdp9AbgPCioAAIBqaNmypVJSUiRJISEhWrZs\nWZk+cXFxiouLK9VWnb4A3EeVpvw9/PDD+uijj1RcXOzoeADUceQbAM5ArgFgL1UqqB555BFt375d\nERERmjlzJpvOAXAY8g0AZyDXALCXKk356969u7p3766zZ89q06ZNeuKJJ+Tv769hw4YpJiZGPj4+\njo4TQB1BvgHgDOQaAPZS5XuoMjIytH79eqWnpys8PFz33HOPduzYofHjx+utt95yZIwA6hjyDQBn\nINcAsIcqFVS9e/dWy5YtNXToUCUmJqp+/fqSpNDQUA0dOtShAQKoW8g3AJyBXAPAXqpUUP3f//2f\n/Pz89Ic//EFnz55Vdna2rr/+enl6emrdunWOjhFAHUK+AeAM5BoA9lKlRSm2bt2qhx56SJJ07Ngx\nPfroo1q5cqVDAwNQN5FvADgDuQaAvVSpoEpJSdHy5cslSddee63Wrl1b7j4Kl1NcXKxJkyZpxIgR\niomJ0f79+5Wdna2RI0cqJiZGM2bMkMVikSQtXLhQw4YN04gRI6wr71TUF0DtYa98AwCVIdcAsJcq\nTfkrLi4utdpNvXr1bDrZtm3bdP78eb333ntKT0/XK6+8ouLiYk2YMEGhoaFKTExUamqqWrRooV27\ndmnVqlU6fPiw4uLitGbNGs2dO7dM3759+9oUC4CayV75BgAqQ66xrzHPb77iY7yd0McOkQDOV6WC\n6u6779a/Z0+HAAAeX0lEQVSoUaPUv39/SdK///1v9elT/T/6kJAQlZSUyGKxqKCgQN7e3vr666/V\no0cPSVJ4eLjS09MVEhKisLAweXh4qEWLFiopKVFeXp6ysrLK9KWgAmoXe+UbAKgMuQaAvVSpoHrq\nqae0adMm7d69W97e3nrggQd09913V/tkDRs21MGDB9W/f38dP35cixcv1u7du+Xh4SFJ8vPzU35+\nvgoKCtSkSRPr6y60G2PK9K2KoKCAasfqKu4Ua3lqavw1Na7yuEOsjozRXvmmuLhYCQkJOnjwoDw9\nPTVr1ix5e3srISFBHh4eat26tWbMmCFPT08tXLhQW7dulbe3t6ZMmaKOHTsqOzu73L4Aagd75RoA\nqPI+VDfccIOaNWsmY4wkaffu3erevXu1TvaPf/xDYWFhmjRpkg4fPqxRo0apuLjY+nxhYaEaNWok\nf39/FRYWlmoPCAgo9WXmQt+qyM2tWuHlakFBAW4Ta0VqYvzu9Lm6S6y5ufkOLarskW+YYgzgcuyR\nawCgSgXVzJkztWXLFl133XXWNg8PD/3zn/+s1skaNWpknaPcuHFjnT9/Xu3bt1dGRoZCQ0OVlpam\nW2+9VcHBwZo/f77Gjh2rI0eOyGKxKDAwsNy+AGoXe+UbphgDqIy9cg0AVKmgSk9P16ZNm6yb3tlq\n9OjRmjJlimJiYlRcXKyJEyeqQ4cOmj59upKSktSqVStFRETIy8tL3bp1U3R0tCwWixITEyVJ8fHx\nZfoCqF3slW9cNcXYXpwx9bOmTS+tLJ6aFmtliNU92CvXAECVCqrrrrvOOhx+Jfz8/PTqq6+WaS9v\nmdK4uDjFxcWVagsJCWFJU6CWs1e+cdUUY3tx9NTPmji9tKJ4amKsFSFWx3BE4WevXAMAVSqoGjdu\nrAEDBqhz586llhidO3euwwIDUDfZK98wxRhAZfhuA8BeqlRQ9erVS7169XJ0LABgt3zDFGMAleG7\nDQB7qVJBNWTIEOXk5Ojnn39WWFiYDh8+XOomTgCwF3vlG6YYA6gM320A2EuVNlX58MMPNX78eM2e\nPVsnT57UiBEjtH79ekfHBqAOIt8AcAZyDQB7qVJBtXTpUr377rvy8/PTH/7wB61bt05LlixxdGwA\n6iDyDQBnsGeuKS4u1qRJkzRixAjFxMRo//79ys7O1siRIxUTE6MZM2bIYrFIkhYuXKhhw4ZpxIgR\n2rt3ryRV2BeAe6hSQeXp6Sl/f3/r4+bNm5daAQsA7IV8A8AZ7JlrLt5I/PHHH9crr7xi3Rx8xYoV\nMsYoNTVVWVlZ1o3Ek5KSNHPmTEkqty8A91Gle6hat26tZcuW6fz58/r++++1YsUKtW3b1tGxAaiD\nyDcAnMGeuYaNxIG6rUoFVWJiot544w35+vpqypQpuvXWWxUfH+/o2ADUQeQbAM5gz1zjrhuJ17SN\nnWtCPDUhhqoi1pqjSgVVw4YNNWnSJE2aNMnR8QCo48g3AJzBnrnGXTcSr2kbO7s6Hnfb7JpY7c/W\nwq9KBVXbtm2tV07+d8IgpaWl2XRSAKgI+QaAM9gz17CROFC3Vamg2rdvn/Xn4uJiffrpp/r6668d\nFhSAuot8A8AZ7Jlr2EgcqNuqVFBdrF69eurfv78WL17siHgAwIp8A8AZrjTXsJE4ULdVqaB6//33\nrT8bY/TTTz9Zh7YBwJ7INwCcgVwDwF6qVFBlZGSUety0aVO9/PLLDgkIQN1GvgHgDOQaAPZSpYJq\n7ty5jo4DACSRbwA4B7kGgL1UqaDq06dPmZVwJFn3TWBHbwD2Qr4B4AzkGgD2UqWCatCgQapXr56i\noqLk7e2tDRs26JtvvtHEiRMdHR+AOoZ8A8AZyDUA7KVKBdX27du1du1a6+NRo0YpMjJS1157rcMC\nA1A3kW8AOAO5BoC9eF6+y+927Nhh/XnLli3y8/NzSEAAQL4B4AzkGgD2UKURqmeffVbx8fH67bff\nJEmtWrXSvHnzHBoYgLqJfAPAGcg1AOylSgVVhw4dtHHjRuXl5cnX15crOAAchnwDwBnINQDspUoF\n1cGDBzVt2jQdPHhQy5cv1/jx4zVnzhy1bNnS0fEBqGPIN4415vnNdjvW2wl97HYswNnINQDspUr3\nUCUmJmrs2LFq2LChmjVrpoEDByo+Pt6mE/79739XdHS0IiMjtWrVKmVnZ2vkyJGKiYnRjBkzZLFY\nJEkLFy7UsGHDNGLECO3du1eSKuwLoPawZ74BgIqQawDYS5UKquPHjyssLEyS5OHhoaioKBUUFFT7\nZBkZGfrqq6/07rvvKjk5WUeOHNHcuXM1YcIErVixQsYYpaamKisrS7t27dKqVauUlJSkmTNnSlK5\nfQHULvbKNwBQGXINAHupUkFVv359HTlyxLoB3hdffCEfH59qn+yzzz5TmzZt9Pjjj+vRRx/VnXfe\nqaysLPXo0UOSFB4erh07digzM1NhYWHy8PBQixYtVFJSory8vHL7Aqhd7JVvAKAy5BoA9lKle6gm\nT56scePG6ZdfftHgwYN18uRJvfrqq9U+2fHjx3Xo0CEtXrxYOTk5Gj9+vHVHckny8/NTfn6+CgoK\n1KRJE+vrLrSX17cqgoICqh2rq7hTrOWpqfHX1LjK4w6xOjJGe+Ub6fcpxps3b1ZxcbFGjhypHj16\nKCEhQR4eHmrdurVmzJghT09PLVy4UFu3bpW3t7emTJmijh07Kjs7u9y+AGoHe+YaAHVblQqqY8eO\nafXq1Tpw4IBKSkrUqlUrm67iNGnSxPraVq1aydfXV0eOHLE+X1hYqEaNGsnf31+FhYWl2gMCAkp9\nmbnQtypyc6tWeLlaUFCA28RakZoYvzt9ru4Sa25uvsOKKnvlm4unGJ85c0Zvv/22ddpwaGioEhMT\nlZqaqhYtWlinGB8+fFhxcXFas2ZNuX379u3rgHcMwBXslWsAoEqXW+fPn6969eqpdevWatu2rc0J\np2vXrtq+fbuMMfr111915swZ3XbbbcrIyJAkpaWlqVu3burSpYs+++wzWSwWHTp0SBaLRYGBgWrf\nvn2ZvgBqF3vlG6YYA6iMvXINAFRphOq6667T5MmTdcstt6h+/frW9vvuu69aJ+vdu7d2796tYcOG\nyRijxMREtWzZUtOnT1dSUpJatWqliIgIeXl5qVu3boqOjpbFYlFiYqIkKT4+vkxfALWLvfKNq6YY\n20tdnPpZ2fHc4fO4gFjdg71yDQBUWlD9+uuvuuqqq9S0aVNJ0p49e0o9b0vSefrpp8u0LVu2rExb\nXFyc4uLiSrWFhISU2xeA+7N3vnHVFGN7cZepn844nrtMhZWI1VHsWfg54rsNgLqt0oLq0Ucf1bp1\n6zR37ly9/fbbGjNmjLPiAlDH2DvfdO3aVf/85z/14IMP6ujRo6WmGIeGhiotLU233nqrgoODNX/+\nfI0dO1ZHjhwpM8X44r4A3B/fbQDYW6X3UBljrD9v2LDB4cEAqLvsnW969+6tdu3aadiwYRo/frwS\nExMVHx+vBQsWKDo6WsXFxYqIiFCHDh2sU4zj4uJKTTG+tC8A98d3GwD2VukI1YX7B6TSCQgA7M0R\n+YYpxgAu5ajvNmzTANRdVVqUQiqdgADAkcg3AJzBXrmGbRrsY8zzm6/4GG8n9LFDJED1VFpQ/fTT\nT7rrrrsk/X4T54WfL6x+lZqa6vgIAdQJ5BsAzuCIXHPxNg0FBQV6+umnlZKSUmrrhfT0dIWEhFRp\nm4b09PQ6WVAB7qrSgurjjz92VhyoRexxhekCrjTVHeQbAM7giFzjrts01MZl86/0PbnTZ0KsNUel\nBdW1117rrDgA1HHkGwDO4Ihc467bNLjLsvnVcSXvyd22EiBW+7O18OOORwAAgCvQtWtXbd++XcYY\n/frrr6W2aZCktLQ0devWTV26dNFnn30mi8WiQ4cOldmm4eK+ANxHlRelAAAAQFm9e/fW7t27NWzY\nMBljlJiYqJYtW2r69OlKSkpSq1atFBERIS8vL+s2DRaLpdQ2DZf2BeA+KKgAAACuENs0AHUXU/4A\nAAAAwEYUVAAAAABgIwoqAAAAALARBRUAAAAA2IiCCgAAAABsREEFAAAAADZi2XQAQI0x5vnNdj3e\n2wl97Ho8AAAuxQgVAAAAANiIggoAAAAAbERBBQAAAAA2oqACAAAAABtRUAEAAACAjSioAAAAAMBG\nLlk2/dixY4qMjNTbb78tb29vJSQkyMPDQ61bt9aMGTPk6emphQsXauvWrfL29taUKVPUsWNHZWdn\nl9u3LrHnksIsJwwAAABcGadXI8XFxUpMTFT9+vUlSXPnztWECRO0YsUKGWOUmpqqrKws7dq1S6tW\nrVJSUpJmzpxZYV8AAAAAcBWnF1Tz5s3TiBEj1Lx5c0lSVlaWevToIUkKDw/Xjh07lJmZqbCwMHl4\neKhFixYqKSlRXl5euX0BoDLHjh3THXfcof379ys7O1sjR45UTEyMZsyYIYvFIklauHChhg0bphEj\nRmjv3r2SVGFfAACAizl1yt/atWsVGBioXr16acmSJZIkY4w8PDwkSX5+fsrPz1dBQYGaNGlifd2F\n9vL6VkVQUICd34njODNWR5zL3se01/H4G7Avd4hRqnhEPDQ0VImJiUpNTVWLFi2sI+KHDx9WXFyc\n1qxZU27fvn37uvgdAQCAmsapBdWaNWvk4eGhzz//XN9//73i4+OVl5dnfb6wsFCNGjWSv7+/CgsL\nS7UHBASUul/qQt+qyM2tWuHlakFBAU6N1RHnsvcx7XE8Z3+uV8JdYs3NzXeLourCiPiFCziXjnKn\np6crJCSkSiPi6enpFFQAah173JvNPdmo65xaUC1fvtz6c2xsrJ555hnNnz9fGRkZCg0NVVpamm69\n9VYFBwdr/vz5Gjt2rI4cOSKLxaLAwEC1b9++TF8AKI+rRsTtxR0K1po6Iu3oY9bEc9rKnWIFgJrK\nJav8XSw+Pl7Tp09XUlKSWrVqpYiICHl5ealbt26Kjo6WxWJRYmJihX0BoDyuGhG3F3cZqazJx3PU\nMSvjLqPMkvvFCgA1lcsKquTkZOvPy5YtK/N8XFyc4uLiSrWFhISU2xcALsWIOABnYksYoO7iXyyA\nOiM+Pl4LFixQdHS0iouLFRERoQ4dOlhHxOPi4kqNiF/aFwDKw5YwQN3m8il/AOBojIgDcCQWwAHq\nNgoqAAAAG7nzAji1cWuSK42lJr2XyyHWmoOCCgAAwEbuvACOvRYlqUmLm1xJLO62UAux2p+thR/3\nUAEAANho+fLlWrZsmZKTk9WuXTvNmzdP4eHhysjIkCSlpaWpW7du6tKliz777DNZLBYdOnSozAI4\nF/cF4F4YoQIAALAjtoQB6hYKKgAAADtgARygbmLKHwAAAADYiIIKAAAAAGxEQQUAAAAANqKgAgAA\nAAAbUVABAAAAgI0oqAAAAADARhRUAAAAAGAjCioAAAAAsBEFFQAAAADYiIIKAAAAAGxEQQUAAAAA\nNqKgAgAAAAAbebs6AOByxjy/2a7Hezuhj12PBwAAgLqLESoAAAAAsBEFFQAAAADYiCl/AAAAqDXs\ncasAtwegOpxaUBUXF2vKlCk6ePCgzp07p/Hjx+vGG29UQkKCPDw81Lp1a82YMUOenp5auHChtm7d\nKm9vb02ZMkUdO3ZUdnZ2uX0BAAAAwBWcWlB98MEHatKkiebPn68TJ07ovvvuU9u2bTVhwgSFhoYq\nMTFRqampatGihXbt2qVVq1bp8OHDiouL05o1azR37twyffv27evMtwDATXABBwAAOINTvx3069dP\nTz75pCTJGCMvLy9lZWWpR48ekqTw8HDt2LFDmZmZCgsLk4eHh1q0aKGSkhLl5eWV2xcAynPhAs6K\nFSv05ptvatasWdaLMitWrJAxRqmpqcrKyrJewElKStLMmTMlqdy+AAAAl3LqCJWfn58kqaCgQE88\n8YQmTJigefPmycPDw/p8fn6+CgoK1KRJk1Kvy8/PlzGmTN+qCAoKsPM7cRxnxuqIc9n7mO4Qo73V\n9Pgk94ixX79+ioiIkFTxBZz09HSFhIRU6QJOeno6I+IAymA0HIDTF6U4fPiwHn/8ccXExGjQoEGa\nP3++9bnCwkI1atRI/v7+KiwsLNUeEBBQKsFc6FsVublVK7xcLSgowKmxOuJc9j6mO8RoT87+G7BV\nbm5+jS+qXHUBx15q+ucrcQGlJp3TVu4Ua03F7QwAnFpQ/fbbbxozZowSExN12223SZLat2+vjIwM\nhYaGKi0tTbfeequCg4M1f/58jR07VkeOHJHFYlFgYGC5fQGgIq64gGMv7lJY1+TjOeqYlXGXiyKS\n+8VaUzEaDsCpBdXixYt16tQpLVq0SIsWLZIkTZ06Vc8995ySkpLUqlUrRUREyMvLS926dVN0dLQs\nFosSExMlSfHx8Zo+fXqpvgBQHi7gAHAGdx4Nt1ehWpMK3tr4niriDjFe4E6x2sKpBdW0adM0bdq0\nMu3Lli0r0xYXF6e4uLhSbSEhIeX2BYBLcQEHgLO462i4vUYoa9JIZ218T+VxtxFmd4rVFmzsC6BW\n4gIOAGdgNBwABRUAAICNGA0HQEEFAABgI0bDAbDRAQAAAADYiBEqAKihxjy/2a7Hezuhj12PBwAA\nKKhQR9nziypfUgEAAOoupvwBAAAAgI0oqAAAAADARkz5AwDUakzxBQA4EiNUAAAAAGAjCioAAAAA\nsBFT/gAAAIBL2GO6MNOE6wZGqAAAAADARoxQAXbgiA1YuZEeAACg5mOECgAAAABsREEFAAAAADai\noAIAAAAAG1FQAQAAAICNWJTCgRyxUAEAAACAmoMRKgAAAACwEQUVAAAAANiIKX8AAACAg9jjFhBu\n+6jZKKguwkaqAAAAAKrD7Qoqi8WiZ555Rj/88IN8fHz03HPP6frrr3d1WABqIfINAGcg1wDuze0K\nqk8//VTnzp3TypUr9fXXX+v555/XG2+84eqwANRC5BsAzkCuQVXYa+ogUxDtz+0KqszMTPXq1UuS\n1KlTJ3377bcujghAbUW+QXmYHg57I9fAHVGY/Y+HMca4OojqmDp1qv785z/rjjvukCTdeeed+vTT\nT+Xt7Xa1IYAajnwDwBnINYB7c7tl0/39/VVYWGh9bLFYSDgAHIJ8A8AZyDWAe3O7gqpLly5KS0uT\nJH399ddq06aNiyMCUFuRbwA4A7kGcG9uN+Xvwko4P/74o4wxmjNnjm644QZXhwWgFiLfAHAGcg3g\n3tyuoAIAAACAmsLtpvwBAAAAQE1BQQUAAAAANqoVBZXFYlFiYqKio6MVGxur7OzsUs+npKQoMjJS\nUVFR2rJli4ui/N3lYv3HP/6h4cOHa/jw4Vq4cKGLovzd5WK90Oehhx7Su+++64IIS8dRWazbtm1T\nVFSUhg8frmeeeUaunOl6uVjffvttRUZGaujQofrkk09cFGVpe/bsUWxsbJn2zZs3a+jQoYqOjlZK\nSooLInM+8o1jkG8cg3zj3sg3jkG+cYw6nW9MLfDxxx+b+Ph4Y4wxX331lXn00Uetzx09etQMHDjQ\nFBUVmVOnTll/dpXKYv3ll1/MkCFDzPnz543FYjHR0dHm+++/d1WolcZ6wUsvvWSGDx9uVqxY4ezw\nSqks1vz8fDNgwABz7NgxY4wxS5Yssf7sCpXFevLkSXPHHXeYoqIic+LECXPnnXe6KkyrJUuWmIED\nB5rhw4eXaj937py5++67zYkTJ0xRUZGJjIw0ubm5LorSecg3jkG+cQzyjXsj3zgG+cYx6nK+qRUj\nVJXtML5371517txZPj4+CggIUHBwsPbt2+eqUCuN9eqrr9abb74pLy8veXh46Pz58/L19XVVqJfd\nuX3Tpk3y8PCw9nGlymL96quv1KZNG82bN08xMTFq1qyZAgMDXRVqpbE2aNBALVq00JkzZ3TmzBl5\neHi4Kkyr4OBgLViwoEz7/v37FRwcrMaNG8vHx0ddu3bV7t27XRChc5FvHIN84xjkG/dGvnEM8o1j\n1OV8Uyt2jSsoKJC/v7/1sZeXl86fPy9vb28VFBQoICDA+pyfn58KCgpcEaakymOtV6+eAgMDZYzR\nCy+8oPbt2yskJKRGxvrjjz/qX//6l1577TW9/vrrLovxgspiPX78uDIyMvT++++rYcOG+stf/qJO\nnTq57LOtLFZJuuaaazRgwACVlJRo3LhxLonxYhEREcrJySnTXtP+bTkL+cb5sZJvHBOrRL6p6cg3\nzo+VfOOYWKXanW9qRUFV2Q7jlz5XWFhY6kNytsvthl5UVKQpU6bIz89PM2bMcEWIVpXF+v777+vX\nX3/VqFGjdPDgQdWrV0/XXnutwsPDa1ysTZo00Z/+9CcFBQVJkrp166bvv//eZQmnsljT0tJ09OhR\npaamSpLGjh2rLl26qGPHji6JtTI17d+Ws5BvHIN84/xYyTc1H/nGMcg3zo+1tuebWjHlr7Idxjt2\n7KjMzEwVFRUpPz9f+/fvd+kO5JXFaozRY489pptuuknPPvusvLy8XBWmpMpjffrpp7Vq1SolJydr\nyJAhGj16tMuSjVR5rDfffLN+/PFH5eXl6fz589qzZ49uvPFGV4VaaayNGzdW/fr15ePjI19fXwUE\nBOjUqVOuCrVSN9xwg7Kzs3XixAmdO3dOX3zxhTp37uzqsByOfOMY5BvHIN+4N/KNY5BvHKMu55ta\nMULVt29fpaena8SIEdYdxt955x0FBwfrrrvuUmxsrGJiYmSM0cSJE106b7eyWC0Wi3bt2qVz585p\n+/btkqS//vWvLvufxuU+15rkcrFOmjRJDz30kCSpX79+Lv2fzuVi3bFjh6KiouTp6akuXbqoZ8+e\nLou1PBs2bNDp06cVHR2thIQEjR07VsYYDR06VFdddZWrw3M48o3zYyXfOC5W8k3NRr5xfqzkG8fF\nWpvzjYcxLlxfEQAAAADcWK2Y8gcAAAAArkBBBQAAAAA2oqACAAAAABtRUAEAAACAjSioAAAAAMBG\nFFSokpycHHXo0EGDBw/WfffdpwEDBujBBx/UkSNHqnyMm266qVrnTEhI0Nq1a8u0p6am6tVXX5Uk\n9enTRzk5OaXaXnvtNX3xxRfVOheAmoN8A8BZyDewh1qxDxWco3nz5lq/fr318UsvvaRZs2bp9ddf\nd2ocd911V5l9Ii5u2717t0JDQ50aEwD7It8AcBbyDa4UI1SwWbdu3XTgwAH16dNHEyZMUEREhI4d\nO6Y1a9Zo4MCBGjRokBISElRYWGh9zfTp0zV48GCNHj1ahw4dkiTt2rVLI0eO1JAhQ9SnTx999NFH\n1v5bt25VZGSkBg0apA8//FCStHbtWiUkJJSK5ULb+++/r2+//VbTpk3TDz/8oDvvvFMWi8V6ngub\n3wFwL+QbAM5CvkF1UVDBJsXFxfroo4/UpUsXSVJ4eLg+/vhj/fbbb1q8eLGSk5O1YcMGNWjQQAsX\nLrS+rnv37lq/fr369u2r2bNnS5KWLVum5557TuvWrdPs2bO1aNEia/8zZ84oJSVFb775pubMmaPc\n3NxK47rvvvvUoUMHPffcc7rpppvUsmVLZWRkSJLWrVunyMhIe38UAByMfAPAWcg3sAUFFars6NGj\nGjx4sAYPHqx7771XxhhNmjRJknTLLbdI+n04unfv3mratKkkKTo6Wjt37pQk1a9fX/fee68kafDg\nwdq1a5ckaf78+frpp5/0+uuv65133il1xWfIkCHy9vbWVVddpU6dOmnPnj3Vinno0KH64IMPdObM\nGe3cuVN33333lX0IAJyCfAPAWcg3uFLcQ4Uqu3SO8cV8fX0lyTr8fIExRufPn5ckeXp6lmr39v79\nzy8mJkahoaEKDQ3Vbbfdpr/97W/Wfl5eXqVeU69evWrF3K9fP7388sv6+OOPFR4eLh8fn2q9HoBr\nkG8AOAv5BleKESrYVY8ePbR582adOHFCkpSSkmK9gfL06dNKTU2VJK1Zs0a33367Tpw4oQMHDujJ\nJ5/UHXfcofT0dJWUlFiPt3HjRhljdPDgQX3zzTf605/+dNkYvLy8rMdo0KCBwsPDlZSUxHA4UMuQ\nbwA4C/kGlWGECnbVtm1bjRs3TrGxsSouLtbNN9+smTNnSpIaNWqkTz/9VK+++qquuuoqzZ07V02a\nNNHw4cM1YMAA+fv7q1OnTjp79qxOnz4tSWrYsKEiIyN1/vx5PfvsswoMDLxsDL169dKMGTM0b948\ndenSRQMGDNCXX35pHbYHUDuQbwA4C/kGlfEwxhhXBwE4SklJiV5++WX94Q9/0IMPPujqcADUYuQb\nAM5CvqlZGKFCrTZ06FA1bdpUb7zxhqtDAVDLkW8AOAv5pmZhhAoAAAAAbMSiFAAAAABgIwoqAAAA\nALARBRUAAAAA2IiCCgAAAABsREEFAAAAADaioAIAAAAAG/1/df9UwNkgGdkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11417da90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# histogram of predicted probabilities\n",
    "plt.figure(figsize=(12, 4))\n",
    "nclasses = 3\n",
    "for i in range(nclasses):\n",
    "    \n",
    "    plt.subplot(1, 3, i+1)\n",
    "    plt.hist(train_proba[:, i], bins=10, histtype='bar', rwidth=0.95)\n",
    "    plt.xlim(0,1)\n",
    "    plt.title('Predicted class-{} probabilities'.format(i+1))\n",
    "    plt.xlabel('Probability')\n",
    "    plt.ylabel('Frequency')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### C. Over-Predict a Label than Under-Predict:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 585,
   "metadata": {},
   "outputs": [],
   "source": [
    "def re_predict(data, threshods):\n",
    "\n",
    "    argmax = np.argmax(data)\n",
    "\n",
    "    ## If the argmax is 2 (class-3) then ovbiously return this highest label\n",
    "    if argmax == 2: \n",
    "        return (argmax +1)\n",
    "\n",
    "    # If argmax is 1 (class-2) there is a chnace that, label is class-2 if\n",
    "    # the probability of the class is greater than the threshold otherwise obviously\n",
    "    # return this highest label (class-3)\n",
    "    elif argmax == 1:\n",
    "        if data[argmax] >= threshods[argmax] : \n",
    "            return (argmax +1)\n",
    "        else:\n",
    "            return (argmax +2)\n",
    "\n",
    "    # If the argmax is 0 (class-1) then there are chances that label is class-1 if\n",
    "    # the probability of the class is greater than the threshold otherwise label can be\n",
    "    # either next two highest labels (class-2 or class-3). To determine the exact class\n",
    "    # class, we have to consider four cases.\n",
    "    # case A : if class_2_prob >= threshold and class_3_prob < threshold then pick class-2\n",
    "    # case B : if class_3_prob >= threshold and class_2_prob < threshold then pick class-3\n",
    "    # case C : if class_2_prob < threshold and class_3_prob < threshold then pick class-1\n",
    "    # case D : if class_2_prob > threshold and class_3_prob > threshold then pick class-3\n",
    "\n",
    "    elif argmax == 0:\n",
    "\n",
    "        if data[argmax] >= threshods[argmax] : \n",
    "            return (argmax +1)\n",
    "        else:\n",
    "            # case A : if class_2_prob >= threshold and class_3_prob < threshold then pick class-2\n",
    "            if data[argmax + 1] >= threshods[argmax + 1] and data[argmax + 2] < threshods[argmax + 2]:\n",
    "                return (argmax + 2)\n",
    "\n",
    "            # case B : if class_3_prob >= threshold and class_2_prob < threshold then pick class-3\n",
    "            if data[argmax + 2] >= threshods[argmax + 2] and data[argmax + 1] < threshods[argmax + 1]:\n",
    "                return (argmax + 3)\n",
    "\n",
    "            # case C : if class_2_prob < threshold and class_3_prob < threshold then pick class-1\n",
    "            if data[argmax + 1] < threshods[argmax + 1] and data[argmax + 2] < threshods[argmax + 2]:\n",
    "                return (argmax + 1)\n",
    "\n",
    "            # case D : if class_2_prob > threshold and class_3_prob > threshold then pick class-3\n",
    "            if data[argmax + 1] > threshods[argmax + 1] and data[argmax + 2] > threshods[argmax + 2]:\n",
    "                return (argmax + 3)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding threshold probability of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = label_binarize(ytrain, classes=[1, 2, 3])\n",
    "_, _, th1 = roc_curve(y[:, 0], train_proba[:, 0])\n",
    "_, _, th2 = roc_curve(y[:, 1], train_proba[:, 1])\n",
    "_, _, th3 = roc_curve(y[:, 2], train_proba[:, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 587,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.41782919929\n",
      "0.300981056675\n",
      "0.202450504927\n"
     ]
    }
   ],
   "source": [
    "print(np.median(th1))\n",
    "print(np.median(th2))\n",
    "print(np.median(th3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = [0.47, 0.30, 0.15]\n",
    "new_pred = []\n",
    "for i in range(train_pred.shape[0]):\n",
    "    new_pred.append(re_predict(train_proba[i, :], threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. The F-1 score of the model 0.3187033756136204\n",
      "\n",
      "2. The recall score of the model 0.5635569577437387\n",
      "\n",
      "3. Classification report \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.99      0.57      0.73     37064\n",
      "          2       0.12      0.50      0.19      1426\n",
      "          3       0.02      0.62      0.04       339\n",
      "\n",
      "avg / total       0.95      0.57      0.70     38829\n",
      " \n",
      "\n",
      "4. Confusion matrix \n",
      " [[21249  5094 10721]\n",
      " [  114   710   602]\n",
      " [   29   100   210]] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('1. The F-1 score of the model {}\\n'.format(f1_score(ytrain, new_pred, average='macro')))\n",
    "print('2. The recall score of the model {}\\n'.format(recall_score(ytrain, new_pred, average='macro')))\n",
    "print('3. Classification report \\n {} \\n'.format(classification_report(ytrain, new_pred)))\n",
    "print('4. Confusion matrix \\n {} \\n'.format(confusion_matrix(ytrain, new_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16641, 3)"
      ]
     },
     "execution_count": 591,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_proba.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 600,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_tpred_prob3 = []\n",
    "for i in range(test_proba.shape[0]):\n",
    "    final_tpred_prob3.append(re_predict(test_proba[i, :], threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "metadata": {},
   "outputs": [],
   "source": [
    "tpred_prob3 = pd.DataFrame(final_tpred_prob3)\n",
    "tpred_prob3.to_csv('final.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}