Challenges/Haptik/Haptik ML Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Haptic ML Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import LambdaCallback, ModelCheckpoint\n",
    "from keras.datasets import imdb\n",
    "from keras.layers import Dense, AlphaDropout, BatchNormalization, GRU, Embedding\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('training.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>959</td>\n",
       "      <td>0</td>\n",
       "      <td>5573 1189 4017 1207 4768 8542 17 1189 5085 5773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>994</td>\n",
       "      <td>0</td>\n",
       "      <td>6315 7507 6700 4742 1944 2692 3647 4413 6700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>995</td>\n",
       "      <td>0</td>\n",
       "      <td>5015 8067 5335 1615 7957 5773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>996</td>\n",
       "      <td>0</td>\n",
       "      <td>2925 7199 1994 4647 7455 5773 4518 2734 2807 8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>997</td>\n",
       "      <td>0</td>\n",
       "      <td>7136 1207 6781 237 4971 3669 6193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id  category                                               text\n",
       "0  959         0    5573 1189 4017 1207 4768 8542 17 1189 5085 5773\n",
       "1  994         0       6315 7507 6700 4742 1944 2692 3647 4413 6700\n",
       "2  995         0                      5015 8067 5335 1615 7957 5773\n",
       "3  996         0  2925 7199 1994 4647 7455 5773 4518 2734 2807 8...\n",
       "4  997         0                  7136 1207 6781 237 4971 3669 6193"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(columns='id', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the length of each tokenized sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = [el.split(' ') for el in data['text']]\n",
    "num_tokens = [len(tokens) for tokens in x_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = pd.DataFrame(data=num_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0\n",
       "0  10\n",
       "1   9\n",
       "2   6\n",
       "3  24\n",
       "4   7"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's look at some statistical information of the length of the tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3464.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>18.840069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>107.460556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6290.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "count  3464.000000\n",
       "mean     18.840069\n",
       "std     107.460556\n",
       "min       1.000000\n",
       "25%       8.000000\n",
       "50%      13.000000\n",
       "75%      23.000000\n",
       "max    6290.000000"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Looks like 23 tokens per sentence can cover 100% information for 75% of the data set\n",
    "## We have some outliers that is causing the standard deviation to be very high\n",
    "### Let us plot with seq_lim bins and see how the plots are"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_lim = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x000000E493CF9940>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAF0JJREFUeJzt3X+QVeV9x/H3J+CvukYwmh0KTMEJ6cQfDeoOYu1kFk0RSaeYmTiDdSIxZkgTnEkmTismk5ponDGdmHS0xmRTiZgSN9TEwhBSS9E7jn+oQIICEmSjRFcINAUxq9YU++0f51l7XXfZu3d/nD0+n9fMnXvO9zznnO8zXPez99yzV0UEZmaWn3eV3YCZmZXDAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZkMk6RRJD0h6RdKvJf1V2T2ZNWNi2Q2YVdCdwO+BVmA28FNJT0bEjnLbMhsa+S+BzRon6UTgEHBWRDyTaj8AXoyI5aU2ZzZEvgRkNjTvB97o/eGfPAmcWVI/Zk1zAJgNTQtwuE/tMHBSCb2YDYsDwGxoeoB396m9G/hdCb2YDYsDwGxongEmSppVV/sg4A+ArXL8IbDZEEnqBAL4FMVdQOuBP/VdQFY1fgdgNnSfBU4ADgD3AZ/xD3+rIr8DMDPLlN8BmJllygFgZpYpB4CZWaYcAGZmmRrXXwZ36qmnxowZM5re/5VXXuHEE08cuYbGmPsvX9XnUPX+ofpzKKP/LVu2/DYiThts3LgOgBkzZrB58+am96/VarS3t49cQ2PM/Zev6nOoev9Q/TmU0b+kXzcyzpeAzMwy5QAwM8vUoAEg6XhJT0h6UtIOSV9N9XskPSdpa3rMTnVJul1Sl6SnJJ1bd6wlknanx5LRm5aZmQ2mkc8AXgcuiogeSccAj0r6Wdr2NxFxf5/xlwKz0uN84C7gfEmnADcCbRTfo7JF0tqIODQSEzEzs6EZ9B1AFHrS6jHpcbTvj1gE3Jv2ewyYJGkKcAmwISIOph/6G4AFw2vfzMya1dB3AUmaAGwB3gfcGRHXS7oHuIDiHcJGYHlEvC5pHXBrRDya9t0IXA+0A8dHxNdS/cvAaxHxjT7nWgosBWhtbT2vs7Oz6cn19PTQ0tLS9P5lc//lq/ocqt4/VH8OZfQ/b968LRHRNti4hm4DjYg3gNmSJgEPSDoLuAH4DXAs0EHxQ/4mQP0d4ij1vufqSMejra0thnP7lG8fK1fV+4fqz6Hq/UP15zCe+x/SXUAR8RJQAxZExL50med14PvAnDSsG5het9s0YO9R6mZmVoJG7gI6Lf3mj6QTgA8Dv0zX9ZEk4DJge9plLXBVuhtoLnA4IvYBDwLzJU2WNBmYn2pmZlaCRi4BTQFWps8B3gWsjoh1kh6SdBrFpZ2twF+n8euBhUAX8CpwNUBEHJR0M7ApjbspIg6O3FTebtuLh/nE8p8CsOfWj4zmqczMKmfQAIiIp4Bz+qlfNMD4AJYNsG0FsGKIPZqZ2SjwXwKbmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWVq0ACQdLykJyQ9KWmHpK+m+kxJj0vaLelHko5N9ePSelfaPqPuWDek+i5Jl4zWpMzMbHCNvAN4HbgoIj4IzAYWSJoLfB34VkTMAg4B16Tx1wCHIuJ9wLfSOCSdASwGzgQWAN+WNGEkJ2NmZo0bNACi0JNWj0mPAC4C7k/1lcBlaXlRWidtv1iSUr0zIl6PiOeALmDOiMzCzMyGbGIjg9Jv6luA9wF3Ar8CXoqII2lINzA1LU8FXgCIiCOSDgPvSfXH6g5bv0/9uZYCSwFaW1up1WpDm1Gd1hPgurOLFodznLL09PRUsu9eVe8fqj+HqvcP1Z/DeO6/oQCIiDeA2ZImAQ8AH+hvWHrWANsGqvc9VwfQAdDW1hbt7e2NtNivO1at4bZtxRT3XNn8ccpSq9UYzvzLVvX+ofpzqHr/UP05jOf+h3QXUES8BNSAucAkSb0BMg3Ym5a7gekAafvJwMH6ej/7mJnZGGvkLqDT0m/+SDoB+DCwE3gY+FgatgRYk5bXpnXS9ociIlJ9cbpLaCYwC3hipCZiZmZD08gloCnAyvQ5wLuA1RGxTtLTQKekrwG/AO5O4+8GfiCpi+I3/8UAEbFD0mrgaeAIsCxdWjIzsxIMGgAR8RRwTj/1Z+nnLp6I+G/g8gGOdQtwy9DbNDOzkea/BDYzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy9SgASBpuqSHJe2UtEPS51L9K5JelLQ1PRbW7XODpC5JuyRdUldfkGpdkpaPzpTMzKwRExsYcwS4LiJ+LukkYIukDWnbtyLiG/WDJZ0BLAbOBP4Q+A9J70+b7wT+HOgGNklaGxFPj8REzMxsaAYNgIjYB+xLy7+TtBOYepRdFgGdEfE68JykLmBO2tYVEc8CSOpMYx0AZmYlUEQ0PliaATwCnAV8AfgE8DKwmeJdwiFJ/wg8FhH/nPa5G/hZOsSCiPhUqn8cOD8iru1zjqXAUoDW1tbzOjs7m50bBw4eZv9rxfLZU09u+jhl6enpoaWlpew2mlb1/qH6c6h6/1D9OZTR/7x587ZERNtg4xq5BASApBbgx8DnI+JlSXcBNwORnm8DPgmon92D/j9veFv6REQH0AHQ1tYW7e3tjbb4NnesWsNt24op7rmy+eOUpVarMZz5l63q/UP151D1/qH6cxjP/TcUAJKOofjhvyoifgIQEfvrtn8PWJdWu4HpdbtPA/am5YHqZmY2xhq5C0jA3cDOiPhmXX1K3bCPAtvT8lpgsaTjJM0EZgFPAJuAWZJmSjqW4oPitSMzDTMzG6pG3gFcCHwc2CZpa6p9EbhC0myKyzh7gE8DRMQOSaspPtw9AiyLiDcAJF0LPAhMAFZExI4RnIuZmQ1BI3cBPUr/1/XXH2WfW4Bb+qmvP9p+ZmY2dvyXwGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmRo0ACRNl/SwpJ2Sdkj6XKqfImmDpN3peXKqS9LtkrokPSXp3LpjLUnjd0taMnrTMjOzwTTyDuAIcF1EfACYCyyTdAawHNgYEbOAjWkd4FJgVnosBe6CIjCAG4HzgTnAjb2hYWZmY2/QAIiIfRHx87T8O2AnMBVYBKxMw1YCl6XlRcC9UXgMmCRpCnAJsCEiDkbEIWADsGBEZ2NmZg1TRDQ+WJoBPAKcBTwfEZPqth2KiMmS1gG3RsSjqb4RuB5oB46PiK+l+peB1yLiG33OsZTinQOtra3ndXZ2Nj25AwcPs/+1YvnsqSc3fZyy9PT00NLSUnYbTat6/1D9OVS9f6j+HMrof968eVsiom2wcRMbPaCkFuDHwOcj4mVJAw7tpxZHqb+1ENEBdAC0tbVFe3t7oy2+zR2r1nDbtmKKe65s/jhlqdVqDGf+Zat6/1D9OVS9f6j+HMZz/w3dBSTpGIof/qsi4iepvD9d2iE9H0j1bmB63e7TgL1HqZuZWQkauQtIwN3Azoj4Zt2mtUDvnTxLgDV19avS3UBzgcMRsQ94EJgvaXL68Hd+qpmZWQkauQR0IfBxYJukran2ReBWYLWka4DngcvTtvXAQqALeBW4GiAiDkq6GdiUxt0UEQdHZBZmZjZkgwZA+jB3oAv+F/czPoBlAxxrBbBiKA2amdno8F8Cm5llygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllatAAkLRC0gFJ2+tqX5H0oqSt6bGwbtsNkrok7ZJ0SV19Qap1SVo+8lMxM7OhaOQdwD3Agn7q34qI2emxHkDSGcBi4My0z7clTZA0AbgTuBQ4A7gijTUzs5JMHGxARDwiaUaDx1sEdEbE68BzkrqAOWlbV0Q8CyCpM419esgdm5nZiBg0AI7iWklXAZuB6yLiEDAVeKxuTHeqAbzQp35+fweVtBRYCtDa2kqtVmu6wdYT4LqzjwAM6zhl6enpqWTfvareP1R/DlXvH6o/h/Hcf7MBcBdwMxDp+Tbgk4D6GRv0f6kp+jtwRHQAHQBtbW3R3t7eZItwx6o13LatmOKeK5s/TllqtRrDmX/Zqt4/VH8OVe8fqj+H8dx/UwEQEft7lyV9D1iXVruB6XVDpwF70/JAdTMzK0FTt4FKmlK3+lGg9w6htcBiScdJmgnMAp4ANgGzJM2UdCzFB8Vrm2/bzMyGa9B3AJLuA9qBUyV1AzcC7ZJmU1zG2QN8GiAidkhaTfHh7hFgWUS8kY5zLfAgMAFYERE7Rnw2ZmbWsEbuArqin/LdRxl/C3BLP/X1wPohdWdmZqPGfwlsZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpapQQNA0gpJByRtr6udImmDpN3peXKqS9LtkrokPSXp3Lp9lqTxuyUtGZ3pmJlZoxp5B3APsKBPbTmwMSJmARvTOsClwKz0WArcBUVgADcC5wNzgBt7Q8PMzMoxaABExCPAwT7lRcDKtLwSuKyufm8UHgMmSZoCXAJsiIiDEXEI2MDbQ8XMzMZQs58BtEbEPoD0/N5Unwq8UDeuO9UGqpuZWUkmjvDx1E8tjlJ/+wGkpRSXj2htbaVWqzXdTOsJcN3ZRwCGdZyy9PT0VLLvXlXvH6o/h6r3D9Wfw3juv9kA2C9pSkTsS5d4DqR6NzC9btw0YG+qt/ep1/o7cER0AB0AbW1t0d7e3t+whtyxag23bSumuOfK5o9TllqtxnDmX7aq9w/Vn0PV+4fqz2E899/sJaC1QO+dPEuANXX1q9LdQHOBw+kS0YPAfEmT04e/81PNzMxKMug7AEn3Ufz2fqqkboq7eW4FVku6BngeuDwNXw8sBLqAV4GrASLioKSbgU1p3E0R0feDZTMzG0ODBkBEXDHApov7GRvAsgGOswJYMaTuzMxs1Pgvgc3MMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPL1LACQNIeSdskbZW0OdVOkbRB0u70PDnVJel2SV2SnpJ07khMwMzMmjMS7wDmRcTsiGhL68uBjRExC9iY1gEuBWalx1LgrhE4t5mZNWk0LgEtAlam5ZXAZXX1e6PwGDBJ0pRROL+ZmTVAEdH8ztJzwCEggO9GRIeklyJiUt2YQxExWdI64NaIeDTVNwLXR8TmPsdcSvEOgdbW1vM6Ozub7u/AwcPsf61YPnvqyU0fpyw9PT20tLSU3UbTqt4/VH8OVe8fqj+HMvqfN2/elrqrMgOaOMzzXBgReyW9F9gg6ZdHGat+am9Ln4joADoA2traor29venm7li1htu2FVPcc2XzxylLrVZjOPMvW9X7h+rPoer9Q/XnMJ77H9YloIjYm54PAA8Ac4D9vZd20vOBNLwbmF63+zRg73DOb2ZmzWs6ACSdKOmk3mVgPrAdWAssScOWAGvS8lrgqnQ30FzgcETsa7pzMzMbluFcAmoFHpDUe5wfRsS/SdoErJZ0DfA8cHkavx5YCHQBrwJXD+PcZmY2TE0HQEQ8C3ywn/p/ARf3Uw9gWbPnMzOzkeW/BDYzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy9SYB4CkBZJ2SeqStHysz29mZoWJY3kySROAO4E/B7qBTZLWRsTTo33uGct/+pb1Pbd+ZLRPaWY2ro31O4A5QFdEPBsRvwc6gUVj3IOZmTHG7wCAqcALdevdwPn1AyQtBZam1R5Ju4ZxvlOB3/a3QV8fxlHHzoD9V0TV+4fqz6Hq/UP151BG/3/UyKCxDgD1U4u3rER0AB0jcjJpc0S0jcSxyuD+y1f1OVS9f6j+HMZz/2N9CagbmF63Pg3YO8Y9mJkZYx8Am4BZkmZKOhZYDKwd4x7MzIwxvgQUEUckXQs8CEwAVkTEjlE85YhcSiqR+y9f1edQ9f6h+nMYt/0rIgYfZWZm7zj+S2Azs0w5AMzMMvWODIDx/HUTklZIOiBpe13tFEkbJO1Oz5NTXZJuT/N4StK5dfssSeN3S1oyhv1Pl/SwpJ2Sdkj6XJXmIOl4SU9IejL1/9VUnynp8dTLj9JNCkg6Lq13pe0z6o51Q6rvknTJWPRfd+4Jkn4haV1F+98jaZukrZI2p1olXkPpvJMk3S/pl+m/hQuq1P+bIuId9aD4cPlXwOnAscCTwBll91XX34eAc4HtdbW/B5an5eXA19PyQuBnFH8/MRd4PNVPAZ5Nz5PT8uQx6n8KcG5aPgl4BjijKnNIfbSk5WOAx1Nfq4HFqf4d4DNp+bPAd9LyYuBHafmM9No6DpiZXnMTxvB19AXgh8C6tF61/vcAp/apVeI1lM69EvhUWj4WmFSl/t+cx1iebIz+YS4AHqxbvwG4oey++vQ4g7cGwC5gSlqeAuxKy98Frug7DrgC+G5d/S3jxnguayi+26lycwD+APg5xV+j/xaY2Pc1RHHH2gVpeWIap76vq/pxY9D3NGAjcBGwLvVTmf7T+fbw9gCoxGsIeDfwHOkmmqr1X/94J14C6u/rJqaW1EujWiNiH0B6fm+qDzSXcTHHdDnhHIrfoiszh3T5ZCtwANhA8dvvSxFxpJ9e3uwzbT8MvIdy/w3+Afhb4H/T+nuoVv9QfAPAv0vaouLrX6A6r6HTgf8Evp8uw/2TpBOpTv9veicGwKBfN1EhA82l9DlKagF+DHw+Il4+2tB+aqXOISLeiIjZFL9JzwE+cJRexlX/kv4COBARW+rLR+llXPVf58KIOBe4FFgm6UNHGTve5jCR4jLuXRFxDvAKxSWfgYy3/t/0TgyAKn7dxH5JUwDS84FUH2gupc5R0jEUP/xXRcRPUrlScwCIiJeAGsV12UmSev8wsr6XN/tM208GDlJe/xcCfylpD8W36V5E8Y6gKv0DEBF70/MB4AGKIK7Ka6gb6I6Ix9P6/RSBUJX+3/RODIAqft3EWqD3DoAlFNfVe+tXpbsI5gKH01vLB4H5kianOw3mp9qokyTgbmBnRHyzanOQdJqkSWn5BODDwE7gYeBjA/TfO6+PAQ9FccF2LbA43WUzE5gFPDHa/UfEDRExLSJmULy2H4qIK6vSP4CkEyWd1LtM8W+/nYq8hiLiN8ALkv44lS4Gnq5K/28xlh84jNWD4lP3Zyiu7X6p7H769HYfsA/4H4rfAK6huCa7Edidnk9JY0XxP9D5FbANaKs7zieBrvS4egz7/zOKt6lPAVvTY2FV5gD8CfCL1P924O9S/XSKH4BdwL8Ax6X68Wm9K20/ve5YX0rz2gVcWsJrqZ3/vwuoMv2nXp9Mjx29/41W5TWUzjsb2JxeR/9KcRdPZfrvffirIMzMMvVOvARkZmYNcACYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlqn/A3ZGyXEDFSauAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe493cf9438>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lengths.hist(bins=seq_lim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Well the plot cements our earlier inference, most of the sequences are indeed in <= 23 tokens per sentence range\n",
    "## Let's limit the tokens to ~4 times the 75% of data, the other sequences\n",
    "## i.e., 24 to 80 should have enough information for the larger sentences/tokens to give us enough information to decide the class of the sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Limiting tokens to seq_lim, i.e., we are compressing the data, it is a lossy compression, similar to how a FLAC is compresses to MP3, the sounds are the same you may miss out on the quality a bit but it is not noticable to most users/ users with lower quality audio equipment.\n",
    "## We run the inference one again with this crude data transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = [el[0:seq_lim] for el in x_train]\n",
    "num_tokens = [len(tokens) for tokens in x_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = pd.DataFrame(data=num_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0\n",
       "0  10\n",
       "1   9\n",
       "2   6\n",
       "3  24\n",
       "4   7"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3464.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>16.866917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>12.527160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "count  3464.000000\n",
       "mean     16.866917\n",
       "std      12.527160\n",
       "min       1.000000\n",
       "25%       8.000000\n",
       "50%      13.000000\n",
       "75%      23.000000\n",
       "max      80.000000"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x000000E46568EC18>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFBlJREFUeJzt3X+s3XV9x/Hne+AY9iI/ht5VILuYVCfSUe0NoizLvbppgcVqogZGtFVMzYabbE1mccnUGJIuEZXFjawKA3+MK0MYTUEd67ghLkFsEWyxop102Ja1/iiFonEW3vvjfDsOl9ue3/d876fPR3Jzzvnezznn1XNOX/dzP+d7vjcyE0lSuX5t2AEkSYNl0UtS4Sx6SSqcRS9JhbPoJalwFr0kFc6il6TCWfTSYUTEKRFxW0Q8FRH/HRF/POxMUjeOHXYAqcb+HvhfYBRYAtwREQ9m5kPDjSV1JvxkrPR8EbEA2AecnZnfr7Z9AdiVmWuGGk7qkEs30uxeDjx9qOQrDwKvGlIeqWsWvTS7EWD/jG37gROGkEXqiUUvze4A8KIZ214EPDmELFJPLHppdt8Hjo2IRU3bzgF8I1bzjm/GSocREVNAAu+jsdfNncDr3etG840zeunw/hQ4HtgL3AT8iSWv+cgZvSQVzhm9JBXOopekwln0klQ4i16SCleLg5qdeuqpOTY21vb4p556igULFgwuUJfqmgvqm62uuaC+2eqaC+qbra65oLdsmzdv/klmvrjlwMwc+tfSpUuzE3fffXdH4+dKXXNl1jdbXXNl1jdbXXNl1jdbXXNl9pYN2JRtdKxLN5JUOItekgpn0UtS4Sx6SSqcRS9JhbPoJalwFr0kFc6il6TCWfSSVLhaHAJhvhhbc8dzLu9Ye9GQkkhS+5zRS1LhnNEfwcwZvCTNR87oJalwFr0kFc6il6TCtSz6iDgjIu6OiG0R8VBEfLDa/tGI2BURD1RfFzZd58qI2B4RD0fEmwf5D5AkHVk7b8YeBFZn5v0RcQKwOSLuqr73qcz8RPPgiDgLuBh4FfBS4N8j4uWZ+XQ/g0uS2tNyRp+Zj2Xm/dX5J4FtwGlHuMpyYCozf5mZjwDbgXP7EVaS1Llo/DWqNgdHjAH3AGcDfwmsBJ4ANtGY9e+LiM8A92bmF6vrXAd8NTNvmXFbq4BVAKOjo0unpqbaznHgwAFGRkbaHt+tLbv2H/H7i0878TmX5ypXN+qara65oL7Z6poL6putrrmgt2yTk5ObM3O81bi296OPiBHgK8AVmflERFwLfBzI6vRq4L1AzHL15/00ycx1wDqA8fHxnJiYaDcK09PTdDK+Wytb7Ee/49LnZpirXN2oa7a65oL6ZqtrLqhvtrrmgrnJ1tZeNxHxAhol/6XMvBUgM/dk5tOZ+QzwWZ5dntkJnNF09dOB3f2LLEnqRDt73QRwHbAtMz/ZtH1h07C3AVur8+uBiyPiuIg4E1gE3Ne/yJKkTrSzdHM+8C5gS0Q8UG37MHBJRCyhsSyzA3g/QGY+FBE3A9+lscfO5e5xI0nD07LoM/MbzL7ufucRrnMVcFUPuSRJfeInYyWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAWvSQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSpc238zVs83NuNvyt6wbMGQkkjS4Tmjl6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAtj0cfEWcAnwd+C3gGWJeZ10TEKcCXgTFgB/DOzNwXEQFcA1wI/BxYmZn3DyZ+f808vrwklaCdGf1BYHVmvhI4D7g8Is4C1gAbM3MRsLG6DHABsKj6WgVc2/fUkqS2tSz6zHzs0Iw8M58EtgGnAcuBG6thNwJvrc4vBz6fDfcCJ0XEwr4nlyS1JTKz/cERY8A9wNnAo5l5UtP39mXmyRGxAVibmd+otm8EPpSZm2bc1ioaM35GR0eXTk1NtZ3jwIEDjIyMtD2+XVt27e/p+meeeMxAcvXDoB6zXtU1F9Q3W11zQX2z1TUX9JZtcnJyc2aOtxrX9t+MjYgR4CvAFZn5RGMpfvahs2x73k+TzFwHrAMYHx/PiYmJdqMwPT1NJ+PbtbLHNfobli0YSK5+GNRj1qu65oL6ZqtrLqhvtrrmgrnJ1tZeNxHxAhol/6XMvLXavOfQkkx1urfavhM4o+nqpwO7+xNXktSplkVf7UVzHbAtMz/Z9K31wIrq/Arg9qbt746G84D9mflYHzNLkjrQztLN+cC7gC0R8UC17cPAWuDmiLgMeBR4R/W9O2nsWrmdxu6V7+lrYklSR1oWffWm6uEW5N84y/gELu8xlySpT/xkrCQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSqcRS9JhbPo+2jLrv2MrbnD49pLqhWLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcG3/hSl1ZuaeNzvWXjSkJJKOds7oJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcH4ydo74SVlJw+KMXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBWuZdFHxPURsTcitjZt+2hE7IqIB6qvC5u+d2VEbI+IhyPizYMKLklqTzsz+huAZbNs/1RmLqm+7gSIiLOAi4FXVdf5h4g4pl9hJUmda1n0mXkP8LM2b285MJWZv8zMR4DtwLk95JMk9Sgys/WgiDFgQ2aeXV3+KLASeALYBKzOzH0R8Rng3sz8YjXuOuCrmXnLLLe5ClgFMDo6unRqaqrt0AcOHGBkZKTt8e3asmt/T9cfPR72/KK9sYtPO7Gn++rUoB6zXtU1F9Q3W11zQX2z1TUX9JZtcnJyc2aOtxrX7SEQrgU+DmR1ejXwXiBmGTvrT5LMXAesAxgfH8+JiYm273x6eppOxrdr5YzDFHRq9eKDXL2lvYd0x6UTPd1Xpwb1mPWqrrmgvtnqmgvqm62uuWBusnW1101m7snMpzPzGeCzPLs8sxM4o2no6cDu3iJKknrRVdFHxMKmi28DDu2Rsx64OCKOi4gzgUXAfb1FlCT1ouU6Q0TcBEwAp0bETuAjwERELKGxLLMDeD9AZj4UETcD3wUOApdn5tODiS5JakfLos/MS2bZfN0Rxl8FXNVLKElS/xz1x6OfeZx4SSqNh0CQpMJZ9JJUOItekgpn0UtS4Sx6SSqcRS9JhbPoJalwFr0kFc6il6TCWfSSVDiLXpIKd9Qf62ZYZh5jZ8fai4aURFLpnNFLUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwln0klQ496OviSP97Vr3sZfUC2f0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAWvSQVzqKXpMJZ9JJUOItekgrXsugj4vqI2BsRW5u2nRIRd0XED6rTk6vtERF/FxHbI+I7EfGaQYaXJLXWzoz+BmDZjG1rgI2ZuQjYWF0GuABYVH2tAq7tT0xJUrdaFn1m3gP8bMbm5cCN1fkbgbc2bf98NtwLnBQRC/sVVpLUucjM1oMixoANmXl2dfnxzDyp6fv7MvPkiNgArM3Mb1TbNwIfysxNs9zmKhqzfkZHR5dOTU21HfrAgQOMjIy0Pf5Ituza35fbARg9Hvb8om839/8Wn3Ziz7fRz8esn+qaC+qbra65oL7Z6poLess2OTm5OTPHW43r92GKY5Zts/4kycx1wDqA8fHxnJiYaPtOpqen6WT8kaw8wuGBO7V68UGu3tL/Iz/vuHSi59vo52PWT3XNBfXNVtdcUN9sdc0Fc5Ot271u9hxakqlO91bbdwJnNI07HdjdfTxJUq+6Lfr1wIrq/Arg9qbt7672vjkP2J+Zj/WYUZLUg5brDBFxEzABnBoRO4GPAGuBmyPiMuBR4B3V8DuBC4HtwM+B9wwgsySpAy2LPjMvOcy33jjL2AQu7zWUJKl//GSsJBXOopekwvV/X0AN3NiMXUJ3rL1oSEkkzQfO6CWpcBa9JBXOopekwrlGPw/MXJOXpE44o5ekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAWvSQVzg9MFcCDnEk6Emf0klQ4i/4oMLbmDrbs2s/Ymjs8nIJ0FLLoJalwFr0kFc6il6TCuddNgVyHl9TMGb0kFc6il6TCWfSSVDiLXpIK55uxRzkPnyCVz6I/CrlXjnR0celGkgrnjF7PMZdLOWNr7mD14oOsrO7TZSNpMHoq+ojYATwJPA0czMzxiDgF+DIwBuwA3pmZ+3qLKUnqVj+WbiYzc0lmjleX1wAbM3MRsLG6LEkakkEs3SwHJqrzNwLTwIcGcD9d8Y3IwWm17NPJY99qrMs8UvsiM7u/csQjwD4ggX/MzHUR8XhmntQ0Zl9mnjzLdVcBqwBGR0eXTk1NtX2/Bw4cYGRkpKvMW3bt7+p67Rg9Hvb8YmA335Nusy0+7cS2x858bGded7bHvjlX8/hWz1MnubrVy+tskOqaC+qbra65oLdsk5OTm5tWUw6r16J/aWbujoiXAHcBfwasb6fom42Pj+emTZvavt/p6WkmJia6yjzIGf3qxQe5eks939/uNlurWXnz97uZ0TfnOtJttco1CL28zgaprrmgvtnqmgt6yxYRbRV9T62Umbur070RcRtwLrAnIhZm5mMRsRDY28t9qN5cCpPqr+s3YyNiQUSccOg88CZgK7AeWFENWwHc3mtISVL3epnRjwK3RcSh2/nnzPxaRHwLuDkiLgMeBd7Re0xJUre6LvrM/CFwzizbfwq8sZdQkqT+8RAIklS4eu4iotro5c3WTq/rG7vSYDijl6TCOaPXvORx9KX2OaOXpMJZ9JJUOItekgrnGr2OOq7v62jjjF6SCueMXkVwli4dnkWv4vlBLB3tXLqRpMIVP6N3Nnd08nmXnuWMXpIKV/yMXpLqZOZvmzcsWzDw+7ToddRzjx2Vrriid21Wkp7LNXpJKlxxM3qp35p/S3RZR/ORM3pJKpwzemmGQzP41YsPstL3fFQAZ/SSVDiLXpIKZ9FLUuHm/Rq9+82rTvzwlerIGb0kFW7ez+ilYfI3Ss0HFr00QC7lqA5cupGkwjmjlzrgUo3mI4teGpK5XNZxCenoNrCij4hlwDXAMcDnMnPtoO5Lmi+O9BtBqzKeeWiGXsp6WMXvD5zhGEjRR8QxwN8DfwjsBL4VEesz87uDuD9J81Onxd/PI4l2sgw3l7kGYVAz+nOB7Zn5Q4CImAKWAxa91KZWRXSk7/ezbDopxNWLDzLR5XU7Hd9Lrk718t5MHd7Xiczs/41GvB1Ylpnvqy6/C3htZn6gacwqYFV18RXAwx3cxanAT/oUt5/qmgvqm62uuaC+2eqaC+qbra65oLdsv52ZL241aFAz+phl23N+omTmOmBdVzcesSkzx7u57iDVNRfUN1tdc0F9s9U1F9Q3W11zwdxkG9R+9DuBM5ounw7sHtB9SZKOYFBF/y1gUUScGRG/DlwMrB/QfUmSjmAgSzeZeTAiPgB8ncbulddn5kN9vIuulnzmQF1zQX2z1TUX1DdbXXNBfbPVNRfMQbaBvBkrSaoPj3UjSYWz6CWpcPOq6CNiWUQ8HBHbI2LNkLNcHxF7I2Jr07ZTIuKuiPhBdXryEHKdERF3R8S2iHgoIj5Yo2y/ERH3RcSDVbaPVdvPjIhvVtm+XL2BP+ci4piI+HZEbKhZrh0RsSUiHoiITdW2OjyfJ0XELRHxver19rqa5HpF9Vgd+noiIq6oSba/qF77WyPipur/xMBfZ/Om6JsOq3ABcBZwSUScNcRINwDLZmxbA2zMzEXAxuryXDsIrM7MVwLnAZdXj1Mdsv0SeENmngMsAZZFxHnA3wKfqrLtAy4bQjaADwLbmi7XJRfAZGYuadrfug7P5zXA1zLzd4BzaDx2Q8+VmQ9Xj9USYCnwc+C2YWeLiNOAPwfGM/NsGjuqXMxcvM4yc158Aa8Dvt50+UrgyiFnGgO2Nl1+GFhYnV8IPFyDx+12GsccqlU24IXA/cBraXwq8NjZnuc5zHM6jf/8bwA20PjQ39BzVfe9Azh1xrahPp/Ai4BHqHboqEuuWXK+CfjPOmQDTgN+BJxCY4/HDcCb5+J1Nm9m9Dz7IB2ys9pWJ6OZ+RhAdfqSYYaJiDHg1cA3qUm2annkAWAvcBfwX8DjmXmwGjKs5/XTwF8Bz1SXf7MmuaDxqfJ/i4jN1aFDYPjP58uAHwP/VC13fS4iFtQg10wXAzdV54eaLTN3AZ8AHgUeA/YDm5mD19l8KvqWh1XQsyJiBPgKcEVmPjHsPIdk5tPZ+JX6dBoHv3vlbMPmMlNE/BGwNzM3N2+eZeiwXm/nZ+ZraCxbXh4Rvz+kHM2OBV4DXJuZrwaeYjjLR4dVrXW/BfiXYWcBqN4TWA6cCbwUWEDjOZ2p76+z+VT08+GwCnsiYiFAdbp3GCEi4gU0Sv5LmXlrnbIdkpmPA9M03kc4KSIOfXhvGM/r+cBbImIHMEVj+ebTNcgFQGburk730lhrPpfhP587gZ2Z+c3q8i00in/YuZpdANyfmXuqy8PO9gfAI5n548z8FXAr8Hrm4HU2n4p+PhxWYT2wojq/gsb6+JyKiACuA7Zl5idrlu3FEXFSdf54Gi/8bcDdwNuHlS0zr8zM0zNzjMbr6j8y89Jh5wKIiAURccKh8zTWnLcy5OczM/8H+FFEvKLa9EYahyEf+uusySU8u2wDw8/2KHBeRLyw+n966DEb/OtsmG+UdPFmxoXA92ms6/71kLPcRGOd7Vc0ZjeX0VjX3Qj8oDo9ZQi5fo/Gr37fAR6ovi6sSbbfBb5dZdsK/E21/WXAfcB2Gr9mHzfE53UC2FCXXFWGB6uvhw697mvyfC4BNlXP578CJ9chV5XthcBPgRObtg09G/Ax4HvV6/8LwHFz8TrzEAiSVLj5tHQjSeqCRS9JhbPoJalwFr0kFc6il6TCWfSSVDiLXpIK939EAoslsJQmEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe46566cf60>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lengths.hist(bins=seq_lim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We see significant improvement in the standard deviation of the lengths\n",
    "## Only about 30 sequences are >= 80 tokens, seems like a good enough compression with less loss. The are probably better ways to compress but this is the best I could think of for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['text'] = x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>116.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       category\n",
       "count     116.0\n",
       "mean        1.0\n",
       "std         0.0\n",
       "min         1.0\n",
       "25%         1.0\n",
       "50%         1.0\n",
       "75%         1.0\n",
       "max         1.0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ones = data.loc[data['category'] == 1]\n",
    "zeroes =  data.loc[data['category'] == 0]\n",
    "ones.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The data set is imbalanced, We balance the data set in a crude manner by only taking the same number of both class data to train the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_zeroes = zeroes.sample(n=116, replace=False, random_state=1)\n",
    "frames = [ones, t_zeroes]\n",
    "data_balanced = pd.concat(frames)\n",
    "train, test = train_test_split(data_balanced, test_size=0.2)\n",
    "data_x_train = data_balanced['text']\n",
    "data_y_train = data_balanced['category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train , y_train = train['text'], train['category']\n",
    "x_test , y_test = test['text'], test['category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train-set size:  185\n",
      "Test-set size:   47\n"
     ]
    }
   ],
   "source": [
    "print(\"Train-set size: \", len(x_train))\n",
    "print(\"Test-set size:  \", len(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_tokens = seq_lim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre pad the sequence to make them equal length to feed the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = 'pre'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_pad = pad_sequences(x_train, maxlen=max_tokens,\n",
    "                            padding=pad, truncating=pad)\n",
    "x_test_pad = pad_sequences(x_test, maxlen=max_tokens,\n",
    "                           padding=pad, truncating=pad)\n",
    "data_x_train_pad = pad_sequences(data_x_train, maxlen=max_tokens,\n",
    "                           padding=pad, truncating=pad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = seq_lim\n",
    "max_features = seq_lim\n",
    "epochs = 50\n",
    "modelh5 = 'HaptikGRUClassifier'\n",
    "loadmodelh5 = 'HaptikGRUClassifier-best'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The model is pretty simple, embedding layers, followed by GRU layers, binary crossentropy is the loss function, we use adam optimizer and we observe the accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building the model for the first time\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    model = load_model(loadmodelh5 + '.h5')\n",
    "    print('Model loaded successfully')\n",
    "except IOError:\n",
    "    print('Building the model for the first time')\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(input_dim=max_features,\n",
    "                        output_dim=128,\n",
    "                        input_length=seq_lim,\n",
    "                        name='layer_embedding'))\n",
    "    model.add(GRU(units=128, return_sequences=True))\n",
    "    model.add(GRU(units=8))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "savebestmodel = ModelCheckpoint(modelh5 + '-best.h5', monitor='loss', verbose=0, save_best_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "layer_embedding (Embedding)  (None, 80, 128)           10240     \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 80, 128)           98688     \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, 8)                 3288      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 9         \n",
      "=================================================================\n",
      "Total params: 112,225\n",
      "Trainable params: 112,225\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 175 samples, validate on 10 samples\n",
      "Epoch 1/50\n",
      "175/175 [==============================] - 9s 50ms/step - loss: 0.6933 - acc: 0.2000 - val_loss: 0.6930 - val_acc: 0.5000\n",
      "Epoch 2/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.6925 - acc: 0.5257 - val_loss: 0.6927 - val_acc: 0.5000\n",
      "Epoch 3/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.6915 - acc: 0.5257 - val_loss: 0.6922 - val_acc: 0.5000\n",
      "Epoch 4/50\n",
      "175/175 [==============================] - 1s 7ms/step - loss: 0.6905 - acc: 0.5257 - val_loss: 0.6918 - val_acc: 0.5000\n",
      "Epoch 5/50\n",
      "175/175 [==============================] - 1s 8ms/step - loss: 0.6892 - acc: 0.5543 - val_loss: 0.6911 - val_acc: 0.5000\n",
      "Epoch 6/50\n",
      "175/175 [==============================] - 1s 7ms/step - loss: 0.6878 - acc: 0.5543 - val_loss: 0.6902 - val_acc: 0.5000\n",
      "Epoch 7/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.6860 - acc: 0.5543 - val_loss: 0.6889 - val_acc: 0.5000\n",
      "Epoch 8/50\n",
      "175/175 [==============================] - 1s 4ms/step - loss: 0.6835 - acc: 0.5771 - val_loss: 0.6867 - val_acc: 0.6000\n",
      "Epoch 9/50\n",
      "175/175 [==============================] - 1s 4ms/step - loss: 0.6801 - acc: 0.5886 - val_loss: 0.6835 - val_acc: 0.6000\n",
      "Epoch 10/50\n",
      "175/175 [==============================] - 1s 3ms/step - loss: 0.6754 - acc: 0.6343 - val_loss: 0.6787 - val_acc: 0.6000\n",
      "Epoch 11/50\n",
      "175/175 [==============================] - 1s 4ms/step - loss: 0.6687 - acc: 0.6857 - val_loss: 0.6717 - val_acc: 0.7000\n",
      "Epoch 12/50\n",
      "175/175 [==============================] - 1s 3ms/step - loss: 0.6591 - acc: 0.7029 - val_loss: 0.6616 - val_acc: 0.7000\n",
      "Epoch 13/50\n",
      "175/175 [==============================] - 1s 3ms/step - loss: 0.6454 - acc: 0.7543 - val_loss: 0.6470 - val_acc: 0.7000\n",
      "Epoch 14/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.6254 - acc: 0.7886 - val_loss: 0.6272 - val_acc: 0.7000\n",
      "Epoch 15/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5975 - acc: 0.8057 - val_loss: 0.6042 - val_acc: 0.7000\n",
      "Epoch 16/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5628 - acc: 0.8229 - val_loss: 0.5954 - val_acc: 0.7000\n",
      "Epoch 17/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5456 - acc: 0.7943 - val_loss: 0.6145 - val_acc: 0.4000\n",
      "Epoch 18/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5680 - acc: 0.6400 - val_loss: 0.5913 - val_acc: 0.7000\n",
      "Epoch 19/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5334 - acc: 0.7943 - val_loss: 0.5804 - val_acc: 0.7000\n",
      "Epoch 20/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5133 - acc: 0.8171 - val_loss: 0.5810 - val_acc: 0.7000\n",
      "Epoch 21/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5104 - acc: 0.8229 - val_loss: 0.5837 - val_acc: 0.7000\n",
      "Epoch 22/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5105 - acc: 0.8171 - val_loss: 0.5851 - val_acc: 0.7000\n",
      "Epoch 23/50\n",
      "175/175 [==============================] - 0s 2ms/step - loss: 0.5074 - acc: 0.8057 - val_loss: 0.5848 - val_acc: 0.7000\n",
      "Epoch 24/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4994 - acc: 0.8057 - val_loss: 0.5836 - val_acc: 0.7000\n",
      "Epoch 25/50\n",
      "175/175 [==============================] - 1s 8ms/step - loss: 0.4870 - acc: 0.8057 - val_loss: 0.5833 - val_acc: 0.7000\n",
      "Epoch 26/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4727 - acc: 0.8229 - val_loss: 0.5857 - val_acc: 0.7000\n",
      "Epoch 27/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4609 - acc: 0.8229 - val_loss: 0.5911 - val_acc: 0.7000\n",
      "Epoch 28/50\n",
      "175/175 [==============================] - 1s 4ms/step - loss: 0.4578 - acc: 0.8286 - val_loss: 0.5982 - val_acc: 0.7000\n",
      "Epoch 29/50\n",
      "175/175 [==============================] - 1s 4ms/step - loss: 0.4616 - acc: 0.8114 - val_loss: 0.6050 - val_acc: 0.7000\n",
      "Epoch 30/50\n",
      "175/175 [==============================] - 1s 3ms/step - loss: 0.4574 - acc: 0.8171 - val_loss: 0.6138 - val_acc: 0.7000\n",
      "Epoch 31/50\n",
      "175/175 [==============================] - 1s 3ms/step - loss: 0.4480 - acc: 0.8286 - val_loss: 0.6246 - val_acc: 0.7000\n",
      "Epoch 32/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4447 - acc: 0.8286 - val_loss: 0.6348 - val_acc: 0.7000\n",
      "Epoch 33/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4470 - acc: 0.8229 - val_loss: 0.6436 - val_acc: 0.7000\n",
      "Epoch 34/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4498 - acc: 0.8171 - val_loss: 0.6514 - val_acc: 0.7000\n",
      "Epoch 35/50\n",
      "175/175 [==============================] - 1s 6ms/step - loss: 0.4496 - acc: 0.8171 - val_loss: 0.6579 - val_acc: 0.7000\n",
      "Epoch 36/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4466 - acc: 0.8229 - val_loss: 0.6625 - val_acc: 0.7000\n",
      "Epoch 37/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4433 - acc: 0.8286 - val_loss: 0.6643 - val_acc: 0.7000\n",
      "Epoch 38/50\n",
      "175/175 [==============================] - 1s 6ms/step - loss: 0.4437 - acc: 0.8286 - val_loss: 0.6647 - val_acc: 0.7000\n",
      "Epoch 39/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4470 - acc: 0.8286 - val_loss: 0.6673 - val_acc: 0.7000\n",
      "Epoch 40/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4472 - acc: 0.8286 - val_loss: 0.6725 - val_acc: 0.7000\n",
      "Epoch 41/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4442 - acc: 0.8286 - val_loss: 0.6766 - val_acc: 0.7000\n",
      "Epoch 42/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4430 - acc: 0.8286 - val_loss: 0.6780 - val_acc: 0.7000\n",
      "Epoch 43/50\n",
      "175/175 [==============================] - 1s 6ms/step - loss: 0.4442 - acc: 0.8286 - val_loss: 0.6775 - val_acc: 0.7000\n",
      "Epoch 44/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4451 - acc: 0.8286 - val_loss: 0.6759 - val_acc: 0.7000\n",
      "Epoch 45/50\n",
      "175/175 [==============================] - 1s 6ms/step - loss: 0.4442 - acc: 0.8286 - val_loss: 0.6731 - val_acc: 0.7000\n",
      "Epoch 46/50\n",
      "175/175 [==============================] - 1s 6ms/step - loss: 0.4422 - acc: 0.8286 - val_loss: 0.6691 - val_acc: 0.7000\n",
      "Epoch 47/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4410 - acc: 0.8286 - val_loss: 0.6643 - val_acc: 0.7000\n",
      "Epoch 48/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4413 - acc: 0.8286 - val_loss: 0.6601 - val_acc: 0.7000\n",
      "Epoch 49/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4415 - acc: 0.8286 - val_loss: 0.6573 - val_acc: 0.7000\n",
      "Epoch 50/50\n",
      "175/175 [==============================] - 1s 5ms/step - loss: 0.4402 - acc: 0.8286 - val_loss: 0.6548 - val_acc: 0.7000\n",
      "Wall time: 56 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xe479823ba8>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(x_train_pad, y_train,\n",
    "          validation_split=0.05, epochs=epochs, batch_size=256, callbacks=[savebestmodel])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47/47 [==============================] - 0s 4ms/step\n",
      "Wall time: 202 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = model.evaluate(x_test_pad, y_test, batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.74      0.80        90\n",
      "          1       0.79      0.89      0.84        95\n",
      "\n",
      "avg / total       0.83      0.82      0.82       185\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true, y_prob = y_train, model.predict(x_train_pad, batch_size=512)\n",
    "y_pred = y_prob >= 0.5\n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.77      0.85        26\n",
      "          1       0.77      0.95      0.85        21\n",
      "\n",
      "avg / total       0.87      0.85      0.85        47\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true, y_prob = y_test, model.predict(x_test_pad, batch_size=512)\n",
    "y_pred = y_prob >= 0.5\n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# F1-Score of 0.81 on training set and 0.85 on test set is a pretty good score, given that we have limited knowledge of the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 208 samples, validate on 24 samples\n",
      "Epoch 1/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3583 - acc: 0.8702 - val_loss: 0.7272 - val_acc: 0.7083\n",
      "Epoch 2/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3569 - acc: 0.8702 - val_loss: 0.7185 - val_acc: 0.7083\n",
      "Epoch 3/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3558 - acc: 0.8702 - val_loss: 0.7102 - val_acc: 0.7083\n",
      "Epoch 4/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3548 - acc: 0.8606 - val_loss: 0.7025 - val_acc: 0.7083\n",
      "Epoch 5/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3539 - acc: 0.8606 - val_loss: 0.6960 - val_acc: 0.7083\n",
      "Epoch 6/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3531 - acc: 0.8654 - val_loss: 0.6913 - val_acc: 0.7083\n",
      "Epoch 7/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3523 - acc: 0.8654 - val_loss: 0.6890 - val_acc: 0.7083\n",
      "Epoch 8/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3514 - acc: 0.8654 - val_loss: 0.6890 - val_acc: 0.7083\n",
      "Epoch 9/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3504 - acc: 0.8654 - val_loss: 0.6913 - val_acc: 0.7083\n",
      "Epoch 10/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3493 - acc: 0.8654 - val_loss: 0.6958 - val_acc: 0.7083\n",
      "Epoch 11/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3481 - acc: 0.8654 - val_loss: 0.7018 - val_acc: 0.7083\n",
      "Epoch 12/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3469 - acc: 0.8654 - val_loss: 0.7091 - val_acc: 0.7083\n",
      "Epoch 13/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3458 - acc: 0.8654 - val_loss: 0.7158 - val_acc: 0.7083\n",
      "Epoch 14/15\n",
      "208/208 [==============================] - 1s 4ms/step - loss: 0.3451 - acc: 0.8654 - val_loss: 0.7229 - val_acc: 0.7083\n",
      "Epoch 15/15\n",
      "208/208 [==============================] - 1s 5ms/step - loss: 0.3450 - acc: 0.8654 - val_loss: 0.7318 - val_acc: 0.7083\n",
      "Wall time: 15.8 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xe4794a3940>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(data_x_train_pad, data_y_train,\n",
    "          validation_split=0.1, epochs=15, batch_size=256, callbacks=[savebestmodel])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.81      0.85       116\n",
      "          1       0.83      0.90      0.86       116\n",
      "\n",
      "avg / total       0.86      0.85      0.85       232\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true, y_prob = data_y_train, model.predict(data_x_train_pad, batch_size=512)\n",
    "y_pred = y_prob >= 0.5\n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on all the data for a few more epoachs, our F1-Score is pretty stable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFbtJREFUeJzt3X9s3Pd93/Hne5btZKYjSnZCaJIwOqvWNYsXVyZcbRkCMmoTWx0qD4gBd8aseRo0tG6QoCtmZQW6FhhQZ4Cb1V7gQp29yYEaxnVrSLDdH4JiIvAfdiIlsmRHc0R7isNIE5FIZsLYbebuvT/uQ+dEUeLx+OPuPns+gMN9v5/vh997v33Ui19+7o6OzESSVK+/1ekCJEnLy6CXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPopTlExNqIeDIifhQR346If9HpmqR2rep0AVKX+jzwY2AAuAl4OiJezMyXO1uWtHDhJ2OlC0XENcB54IOZ+a0y9gXgu5m5u6PFSW1w6Ua62N8H/mYm5IsXgX/YoXqkRTHopYv1AVOzxqaAaztQi7RoBr10sWngPbPG3gP8sAO1SItm0EsX+xawKiI2NY19CPCFWPUkX4yV5hARo0AC/4bGu26eAf6J77pRL/KKXprbrwLvBiaBLwK/YsirV3lFL0mV84pekipn0EtS5Qx6SaqcQS9JleuKP2p2/fXX5+DgYFtf+6Mf/YhrrrlmaQtaYfbQeb1eP9hDt1jJHo4cOfK9zHzvfPO6IugHBwc5fPhwW187NjbG8PDw0ha0wuyh83q9frCHbrGSPUTEt1uZ59KNJFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVrueD/vh3pxjc/TSDu5/udCmS1JV6PuglSZdn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmq3LxBHxE/HRFHm24/iIhPR8TaiDgYESfL/ZoyPyLiwYgYj4hjEbF5+duQJF3KvEGfma9k5k2ZeRNwM/Am8CSwGziUmZuAQ2Uf4DZgU7ntAh5ejsIlSa1Z6NLNVuDVzPw2sB3YW8b3AreX7e3AY9nwPNAfEeuWpFpJ0oJFZrY+OeJR4OuZ+V8j4o3M7G86dj4z10TEU8D9mflcGT8E3JeZh2edaxeNK34GBgZuHh0dbauByXNTnH2rsX3j+tVtnaPTpqen6evr63QZi9LrPfR6/WAP3WIlexgZGTmSmUPzzVvV6gkj4irgl4DPzDd1jrGLfppk5h5gD8DQ0FAODw+3WsoFHtq3nweON9o4dVd75+i0sbEx2u2/W/R6D71eP9hDt+jGHhaydHMbjav5s2X/7MySTLmfLOMTwMamr9sAnF5soZKk9iwk6H8Z+GLT/gFgR9neAexvGr+7vPtmCzCVmWcWXakkqS0tLd1ExN8GfgH4t03D9wOPR8RO4HXgjjL+DLANGKfxDp17lqxaSdKCtRT0mfkmcN2sse/TeBfO7LkJ3Lsk1UmSFs1PxkpS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqlxLQR8R/RHxRET8z4g4ERH/OCLWRsTBiDhZ7teUuRERD0bEeEQci4jNy9uCJOlyWr2i/33gzzPzHwAfAk4Au4FDmbkJOFT2AW4DNpXbLuDhJa1YkrQg8wZ9RLwH+AjwCEBm/jgz3wC2A3vLtL3A7WV7O/BYNjwP9EfEuiWvXJLUksjMy0+IuAnYA3yTxtX8EeBTwHczs79p3vnMXBMRTwH3Z+ZzZfwQcF9mHp513l00rvgZGBi4eXR0tK0GJs9NcfatxvaN61e3dY5Om56epq+vr9NlLEqv99Dr9YM9dIuV7GFkZORIZg7NN29VC+daBWwGPpmZL0TE7/OTZZq5xBxjF/00ycw9NH6AMDQ0lMPDwy2UcrGH9u3ngeONNk7d1d45Om1sbIx2++8Wvd5Dr9cP9tAturGHVtboJ4CJzHyh7D9BI/jPzizJlPvJpvkbm75+A3B6acq9vMHdTzO4++mVeChJ6hnzBn1m/m/gOxHx02VoK41lnAPAjjK2A9hftg8Ad5d332wBpjLzzNKWLUlqVStLNwCfBPZFxFXAa8A9NH5IPB4RO4HXgTvK3GeAbcA48GaZK0nqkJaCPjOPAnMt+G+dY24C9y6yLknSEvGTsZJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKtdS0EfEqYg4HhFHI+JwGVsbEQcj4mS5X1PGIyIejIjxiDgWEZuXswFJ0uUt5Ip+JDNvysyZ/3fsbuBQZm4CDpV9gNuATeW2C3h4qYqVJC3cYpZutgN7y/Ze4Pam8cey4XmgPyLWLeJxJEmL0GrQJ/CXEXEkInaVsYHMPANQ7t9XxtcD32n62okyJknqgMjM+SdF/J3MPB0R7wMOAp8EDmRmf9Oc85m5JiKeBn43M58r44eAf5+ZR2adcxeNpR0GBgZuHh0dbauByXNTnH3rwrEb169u61ydMj09TV9fX6fLWJRe76HX6wd76BYr2cPIyMiRpuX0S1rVysky83S5n4yIJ4FbgLMRsS4zz5SlmckyfQLY2PTlG4DTc5xzD7AHYGhoKIeHh1sp5SIP7dvPA8cvbOPUXe2dq1PGxsZot/9u0es99Hr9YA/doht7mHfpJiKuiYhrZ7aBjwEvAQeAHWXaDmB/2T4A3F3efbMFmJpZ4pEkrbxWrugHgCcjYmb+H2Xmn0fE14DHI2In8DpwR5n/DLANGAfeBO5Z8qolSS2bN+gz8zXgQ3OMfx/YOsd4AvcuSXWSpEXzk7GSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5VoO+oi4IiK+ERFPlf0bIuKFiDgZEV+KiKvK+NVlf7wcH1ye0iVJrVjIFf2ngBNN+58FPpeZm4DzwM4yvhM4n5k/BXyuzJMkdUhLQR8RG4BfBP5b2Q/go8ATZcpe4Payvb3sU45vLfMlSR0QmTn/pIgngN8FrgV+A/hXwPPlqp2I2Aj8WWZ+MCJeAm7NzIly7FXg5zLze7POuQvYBTAwMHDz6OhoWw1Mnpvi7FsXjt24fnVb5+qU6elp+vr6Ol3GovR6D71eP9hDt1jJHkZGRo5k5tB881bNNyEi/hkwmZlHImJ4ZniOqdnCsZ8MZO4B9gAMDQ3l8PDw7CkteWjffh44fmEbp+5q71ydMjY2Rrv9d4te76HX6wd76Bbd2MO8QQ98GPiliNgGvAt4D/BfgP6IWJWZbwMbgNNl/gSwEZiIiFXAauDcklcuSWrJvGv0mfmZzNyQmYPAncCXM/Mu4FngE2XaDmB/2T5Q9inHv5ytrA9JkpbFYt5Hfx/w6xExDlwHPFLGHwGuK+O/DuxeXImSpMVoZenmHZk5BoyV7deAW+aY81fAHUtQmyRpCfjJWEmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlZs36CPiXRHx1Yh4MSJejojfKeM3RMQLEXEyIr4UEVeV8avL/ng5Pri8LUiSLqeVK/q/Bj6amR8CbgJujYgtwGeBz2XmJuA8sLPM3wmcz8yfAj5X5kmSOmTeoM+G6bJ7Zbkl8FHgiTK+F7i9bG8v+5TjWyMilqxiSdKCtLRGHxFXRMRRYBI4CLwKvJGZb5cpE8D6sr0e+A5AOT4FXLeURUuSWheZ2frkiH7gSeC3gP9elmeIiI3AM5l5Y0S8DHw8MyfKsVeBWzLz+7POtQvYBTAwMHDz6OhoWw1Mnpvi7FsXjt24fnVb5+qU6elp+vr6Ol3GovR6D71eP9hDt1jJHkZGRo5k5tB881Yt5KSZ+UZEjAFbgP6IWFWu2jcAp8u0CWAjMBERq4DVwLk5zrUH2AMwNDSUw8PDCynlHQ/t288Dxy9s49Rd7Z2rU8bGxmi3/27R6z30ev1gD92iG3to5V037y1X8kTEu4GfB04AzwKfKNN2APvL9oGyTzn+5VzIrw2SpCXVyhX9OmBvRFxB4wfD45n5VER8ExiNiP8EfAN4pMx/BPhCRIzTuJK/cxnqliS1aN6gz8xjwM/OMf4acMsc438F3LEk1UmSFs1PxkpS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZUz6CWpcga9JFXOoJekyhn0klQ5g16SKmfQS1LlDHpJqpxBL0mVmzfoI2JjRDwbESci4uWI+FQZXxsRByPiZLlfU8YjIh6MiPGIOBYRm5e7CUnSpbVyRf828O8y82eALcC9EfEBYDdwKDM3AYfKPsBtwKZy2wU8vORVS5JaNm/QZ+aZzPx62f4hcAJYD2wH9pZpe4Hby/Z24LFseB7oj4h1S165JKklkZmtT44YBL4CfBB4PTP7m46dz8w1EfEUcH9mPlfGDwH3ZebhWefaReOKn4GBgZtHR0fbamDy3BRn37pw7Mb1q9s6V6dMT0/T19fX6TIWpdd76PX6wR66xUr2MDIyciQzh+abt6rVE0ZEH/AnwKcz8wcRccmpc4xd9NMkM/cAewCGhoZyeHi41VIu8NC+/Txw/MI2Tt3V3rk6ZWxsjHb77xa93kOv1w/20C26sYeW3nUTEVfSCPl9mfmnZfjszJJMuZ8s4xPAxqYv3wCcXppyJUkL1cq7bgJ4BDiRmb/XdOgAsKNs7wD2N43fXd59swWYyswzS1izJGkBWlm6+TDwL4HjEXG0jP0H4H7g8YjYCbwO3FGOPQNsA8aBN4F7lrRiSdKCzBv05UXVSy3Ib51jfgL3LrIuSdIS8ZOxklQ5g16SKmfQS1LlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIq18r/HPzRiJiMiJeaxtZGxMGIOFnu15TxiIgHI2I8Io5FxOblLF6SNL9Wruj/B3DrrLHdwKHM3AQcKvsAtwGbym0X8PDSlClJate8QZ+ZXwHOzRreDuwt23uB25vGH8uG54H+iFi3VMVKkhYuMnP+SRGDwFOZ+cGy/0Zm9jcdP5+ZayLiKeD+zHyujB8C7svMw3OccxeNq34GBgZuHh0dbauByXNTnH3rwrEb169u61ydMj09TV9fX6fLWJRe76HX6wd76BYr2cPIyMiRzByab96qJX7cmGNszp8kmbkH2AMwNDSUw8PDbT3gQ/v288DxC9s4dVd75+qUsbEx2u2/W/R6D71eP9hDt+jGHtp9183ZmSWZcj9ZxieAjU3zNgCn2y9PkrRY7Qb9AWBH2d4B7G8av7u8+2YLMJWZZxZZoyRpEeZduomILwLDwPURMQH8R+B+4PGI2Am8DtxRpj8DbAPGgTeBe5ahZknSAswb9Jn5y5c4tHWOuQncu9iiJElLx0/GSlLlDHpJqpxBL0mVM+glqXIGvSRVzqCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9JlTPoJalyBr0kVc6gl6TKGfSSVDmDXpIqZ9BLUuUMekmqnEEvSZVblqCPiFsj4pWIGI+I3cvxGJKk1sz7PwdfqIi4Avg88AvABPC1iDiQmd9c6se6lMHdT7+zfer+X1yph5WkrrQcV/S3AOOZ+Vpm/hgYBbYvw+O0ZHD30xcEvyT9/2bJr+iB9cB3mvYngJ+bPSkidgG7yu50RLzS5uNdD3xvvknx2TbPvjJa6qHL9XoPvV4/2EO3WMke/m4rk5Yj6GOOsbxoIHMPsGfRDxZxODOHFnueTrKHzuv1+sEeukU39rAcSzcTwMam/Q3A6WV4HElSC5Yj6L8GbIqIGyLiKuBO4MAyPI4kqQVLvnSTmW9HxK8BfwFcATyamS8v9eM0WfTyTxewh87r9frBHrpF1/UQmRctn0uSKuInYyWpcga9JFWup4O+V/7UQkSciojjEXE0Ig6XsbURcTAiTpb7NWU8IuLB0tOxiNjcoZofjYjJiHipaWzBNUfEjjL/ZETs6IIefjsivluei6MRsa3p2GdKD69ExMebxjvyfRYRGyPi2Yg4EREvR8SnynjPPA+X6aGXnod3RcRXI+LF0sPvlPEbIuKF8t/0S+XNJ0TE1WV/vBwfnK+3ZZeZPXmj8ULvq8D7gauAF4EPdLquS9R6Crh+1th/BnaX7d3AZ8v2NuDPaHweYQvwQodq/giwGXip3ZqBtcBr5X5N2V7T4R5+G/iNOeZ+oHwPXQ3cUL63rujk9xmwDthctq8FvlXq7Jnn4TI99NLzEEBf2b4SeKH8930cuLOM/wHwK2X7V4E/KNt3Al+6XG8r0UMvX9F31Z9aaMN2YG/Z3gvc3jT+WDY8D/RHxLqVLi4zvwKcmzW80Jo/DhzMzHOZeR44CNy6/NU3XKKHS9kOjGbmX2fm/wLGaXyPdez7LDPPZObXy/YPgRM0PnneM8/DZXq4lG58HjIzp8vuleWWwEeBJ8r47Odh5vl5AtgaEcGle1t2vRz0c/2phct9A3VSAn8ZEUei8acfAAYy8ww0/jEA7yvj3dzXQmvu1l5+rSxtPDqz7EGX91B+/f9ZGleTPfk8zOoBeuh5iIgrIuIoMEnjB+WrwBuZ+fYc9bxTazk+BVxHB3vo5aBv6U8tdIkPZ+Zm4Dbg3oj4yGXm9lJfMy5Vczf28jDw94CbgDPAA2W8a3uIiD7gT4BPZ+YPLjd1jrFu7aGnnofM/JvMvInGJ/1vAX7mMvV0XQ+9HPQ986cWMvN0uZ8EnqTxjXJ2Zkmm3E+W6d3c10Jr7rpeMvNs+Uf7f4E/5Ce/OndlDxFxJY2A3JeZf1qGe+p5mKuHXnseZmTmG8AYjTX6/oiY+dBpcz3v1FqOr6axhNixHno56HviTy1ExDURce3MNvAx4CUatc68+2EHsL9sHwDuLu+g2AJMzfya3gUWWvNfAB+LiDXlV/OPlbGOmfV6xz+n8VxAo4c7yzsmbgA2AV+lg99nZV33EeBEZv5e06GeeR4u1UOPPQ/vjYj+sv1u4OdpvNbwLPCJMm328zDz/HwC+HI2Xo29VG/LbyVe8V2uG413GXyLxnrZb3a6nkvU+H4ar7S/CLw8UyeNNbtDwMlyvzZ/8gr/50tPx4GhDtX9RRq/Uv8fGlciO9upGfjXNF50Ggfu6YIevlBqPEbjH966pvm/WXp4Bbit099nwD+l8av9MeBouW3rpefhMj300vPwj4BvlFpfAn6rjL+fRlCPA38MXF3G31X2x8vx98/X23Lf/BMIklS5Xl66kSS1wKCXpMoZ9JJUOYNekipn0EtS5Qx6SaqcQS9Jlft/+U78HQFaT8oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe46a002e10>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEwxJREFUeJzt3X+M5HV9x/Hnu6CIrHBc0e15EBfj1V9cQW+DqI3ZBdOiEOEPMFhCj/bMJa1a2tCUoyY1TW2KaavSxNpeRD0NZaUnlAuntuRkS/qH1DtEDj0RxBPvQE7r3ZWljXj67h/z3TIeszfz3d3Z+c5nn49ks/P9zmdm3+/5zr72u5/5fmciM5EkDb9fGnQBkqTFYaBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoWtYiYmVE3B4RT0fE9yLitwZdkzRfxw+6AGnAPgY8A4wC5wDbI+LrmfmNwZYl1ReeKarlKiJOAg4CZ2Xmt6t1nwX2Z+amgRYnzYNTLlrOfhX42WyYV74OvHZA9UgLYqBrORsBDh+17jDwogHUIi2Yga7lbAY4+ah1JwNPDaAWacEMdC1n3waOj4g1bevOBnxBVEPJF0W1rEXEFJDAu2kd5fIF4E0e5aJh5B66lrvfB04EDgC3AL9nmGtYuYcuSYVwD12SCmGgS1IhDHRJKoSBLkmFWNI35zrttNNybGys1m2efvppTjrppP4UtETsoRnsoRnsob5du3b9KDNf3G3ckgb62NgYO3furHWb6elpJiYm+lPQErGHZrCHZrCH+iLie72Mc8pFkgphoEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIKYaAXamzTdsY2bR90GZKWkIEuSYUw0CWpED0FekSsiIitEfGtiNgTEW+MiJURcVdEPFx9P7XfxfaDUxOSStHrHvqNwJcy81XA2cAeYBOwIzPXADuqZUnSgHQN9Ig4GXgLcBNAZj6TmYeAS4At1bAtwKX9KlKS1F1k5rEHRJwDbAa+SWvvfBdwDbA/M1e0jTuYmc+ZdomIjcBGgNHR0XVTU1O1CpyZmWFkZKTWberYvf8wAGtXn9K3n9HvHjpZ7L4G0cNis4dmsIf6Jicnd2XmeNeBmXnML2AcOAK8oVq+EfgL4NBR4w52u69169ZlXXfffXft29TxsuvuzJddd2dff0a/e+hksfsaRA+LzR6awR7qA3Zml3zNzJ7m0PcB+zLz3mp5K/B64MmIWAVQfT9Q5y+OJGlxdQ30zPwB8P2IeGW16gJa0y/bgPXVuvXAHX2pUJLUk14/U/R9wM0R8XzgUeB3aP0xuDUiNgCPAZf3p0RJUi96CvTMvJ/WXPrRLljcciRJ8+WZopJUCANdkgphoEtSIQx0SSrEsgh034BL0nKwLAJdkpYDA12SCmGgS1Ihej1TtAjt8+h7b7hogJUsXEm9SFoc7qFLUiEMdEkqRLFTLh6mKGm5cQ9dkgphoEtSIYqdctEv8qgYqXzuoUtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCLNvDFj2TVFJp3EOXpEIY6JJUCANdkgrR0xx6ROwFngJ+BhzJzPGIWAl8DhgD9gLvzMyD/SlTktRNnT30ycw8JzPHq+VNwI7MXAPsqJYlSQOykCmXS4At1eUtwKULL0eSNF+Rmd0HRXwXOAgk8I+ZuTkiDmXmirYxBzPz1A633QhsBBgdHV03NTVVq8CZmRlGRkZq3QZg9/7DtcavXX1K7Z/Rq/n2cCzt/XWqffb62eu6je+mHz0sNXtoBnuob3Jyclfb7Miceg30l2bm4xHxEuAu4H3Atl4Cvd34+Hju3Lmze/VtpqenmZiYqHUbqH+ceT/fUna+PRxLt7fDnb1+9rqFvn1uP3pYavbQDPZQX0T0FOg9Tblk5uPV9wPA7cC5wJMRsar6YauAA/MvV5K0UF0DPSJOiogXzV4GfgN4ENgGrK+GrQfu6FeRkqTuejlscRS4PSJmx/9TZn4pIr4K3BoRG4DHgMv7V6YkqZuugZ6ZjwJnd1j/X8AF/ShKklSfZ4pKUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoklSIZfuZoiXyc1Kl5c09dEkqhIEuSYUw0JehsU3bnZ6RCmSgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6JJUCANdkgphoEtSIQx0SSqE77ZYOE/xl5YP99AlqRAGuiQVoucpl4g4DtgJ7M/MiyPiTGAKWAncB1yVmc/0p8zBaJ+u2HvDRQOsRJK6q7OHfg2wp235Q8BHMnMNcBDYsJiFSZLq6SnQI+J04CLgE9VyAOcDW6shW4BL+1GgJKk3kZndB0VsBf4KeBHwx8DVwFcy8xXV9WcAX8zMszrcdiOwEWB0dHTd1NRUrQJnZmYYGRmpdRuA3fsP1xq/dvUpx7yPTtf3ar49HEun2haj57n0o4elZg/NYA/1TU5O7srM8W7jus6hR8TFwIHM3BURE7OrOwzt+JchMzcDmwHGx8dzYmKi07A5TU9PU/c2AFfXPFxv75XP/Rnt99Hp+l7Nt4dj6VTbYvQ8l370sNTsoRnsoX96eVH0zcA7IuLtwAuAk4GPAisi4vjMPAKcDjzevzIlSd10nUPPzOsz8/TMHAOuAL6cmVcCdwOXVcPWA3f0rUpJUlcLOVP0OmAqIj4IfA24aXFKUl2eDSoJagZ6Zk4D09XlR4FzF78kSdJ8eKaoJBXCN+fqoMlTGE2uTdJguYcuSYUw0CWpEAb6Aoxt2u4UiKTGMNAlqRAGuiQVwkCXpEIY6JJUCANdkgphoEtSITxTtNLt8MPZ6/1sUUlN5R66JBXCQJekQhjoklQIA12SCmGgS1IhDHRJKkRxhy367oe9a3+sPBxTGn7uoUtSIQx0SSpEcVMug9Dvs0iXYhrJM2Gl4eceuiQVwkCXpEJ0nXKJiBcA9wAnVOO3ZuYHIuJMYApYCdwHXJWZz/Sz2CbwKBpJTdXLHvpPgPMz82zgHODCiDgP+BDwkcxcAxwENvSvTElSN10DPVtmqsXnVV8JnA9srdZvAS7tS4WSpJ70NIceEcdFxP3AAeAu4DvAocw8Ug3ZB6zuT4mSpF5EZvY+OGIFcDvwZ8CnMvMV1fozgC9k5toOt9kIbAQYHR1dNzU1VavAmZkZRkZGeh6/e//hWve/mNauPqXj+ro9HG0pe+pXD01gD81gD/VNTk7uyszxbuNqHYeemYciYho4D1gREcdXe+mnA4/PcZvNwGaA8fHxnJiYqPMjmZ6eps5trh7gi5Z7r5zouL5uD0dbyp761UMT2EMz2EP/dJ1yiYgXV3vmRMSJwFuBPcDdwGXVsPXAHf0qUpLUXS976KuALRFxHK0/ALdm5p0R8U1gKiI+CHwNuKmPdUqSuuga6Jn5APC6DusfBc7tR1GSpPo8U1SSCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6JJUCD9TtE8W4zM6/TANSXW4hy5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIK4WGLS6jTYYgLOaxRktq5hy5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIKYaBLUiEMdEkqhIEuSYXoeqZoRJwBfAb4FeDnwObMvDEiVgKfA8aAvcA7M/Ng/0ot09Fnjw76zNH2egZdi6R6etlDPwJcm5mvBs4D3hMRrwE2ATsycw2wo1qWJA1I10DPzCcy877q8lPAHmA1cAmwpRq2Bbi0X0VKkrqLzOx9cMQYcA9wFvBYZq5ou+5gZp7a4TYbgY0Ao6Oj66ampmoVODMzw8jISM/jd+8/XOv+F9Pa1ac8p461q0/5/x56qa3TfQxKey11t0MT2UMz2EN9k5OTuzJzvNu4ngM9IkaAfwf+MjNvi4hDvQR6u/Hx8dy5c2dPP2/W9PQ0ExMTPY8f5Acrt885t39I9GwPvdTW6T4Gpb2WutuhieyhGeyhvojoKdB7OsolIp4HfB64OTNvq1Y/GRGrqutXAQfmW6wkaeF6OcolgJuAPZn54bartgHrgRuq73f0pcIhMug9aknLWy8fcPFm4Cpgd0TcX637U1pBfmtEbAAeAy7vT4mSpF50DfTM/A8g5rj6gsUtR5I0X54pKkmFMNAlqRAGuiQVwkCXpEIY6JJUCANdkgphoEtSIXo5sajxPENTktxDl6RiGOiSVAgDXZIKYaBrTmObtvv6hDREDHRJKoSBLkmFKOKwxSYb27Sda9ce4eoepy6GbYqjvd72j6yTtPTcQ5ekQhjoklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEJ0DfSI+GREHIiIB9vWrYyIuyLi4er7qf0tU5LUTS976J8GLjxq3SZgR2auAXZUy5KkAeoa6Jl5D/Djo1ZfAmypLm8BLl3kuiRJNUVmdh8UMQbcmZlnVcuHMnNF2/UHM7PjtEtEbAQ2AoyOjq6bmpqqVeDMzAwjIyPHHLN7/+Fa97nURk+EJ/930FUszNE9rF19CvCLj/3suqbq5bnUdPbQDEvdw+Tk5K7MHO82ru/vtpiZm4HNAOPj4zkxMVHr9tPT03S7Ta/vZDgo1649wt/uHu43tjy6h71XTgC/+NjPrmuqXp5LTWcPzdDUHuZ7lMuTEbEKoPp+YPFKkiTNx3wDfRuwvrq8HrhjccqRJM1X13mAiLgFmABOi4h9wAeAG4BbI2ID8BhweT+LVPMM2wdxSMtB10DPzHfNcdUFi1yLJGkBPFNUkgphoEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCGOiSVIihe8coz1CUpM7cQ5ekQhjoklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6Fo0Y5u2P+dM3k7rJPWHgS5JhTDQJakQQ/fmXGq+TlMss+v23nDRUpcjDcxSP+/dQ5ekQhjoklSIoZly8UiJcrVv207/mh5r2x/rX9lu99vL+Lr3sRSaMn3VlDr0rAXtoUfEhRHxUEQ8EhGbFqsoSVJ98w70iDgO+BjwNuA1wLsi4jWLVZgkqZ6F7KGfCzySmY9m5jPAFHDJ4pQlSaorMnN+N4y4DLgwM99dLV8FvCEz33vUuI3AxmrxlcBDNX/UacCP5lVkc9hDM9hDM9hDfS/LzBd3G7SQF0Wjw7rn/HXIzM3A5nn/kIidmTk+39s3gT00gz00gz30z0KmXPYBZ7Qtnw48vrByJEnztZBA/yqwJiLOjIjnA1cA2xanLElSXfOecsnMIxHxXuBfgeOAT2bmNxatsmfNe7qmQeyhGeyhGeyhT+b9oqgkqVk89V+SCmGgS1IhGhvow/i2AhFxRkTcHRF7IuIbEXFNtX5lRNwVEQ9X308ddK3dRMRxEfG1iLizWj4zIu6tevhc9UJ4Y0XEiojYGhHfqrbHG4dtO0TEH1XPowcj4paIeMEwbIeI+GREHIiIB9vWdXzso+Xvqt/zByLi9YOr/Flz9PDX1fPpgYi4PSJWtF13fdXDQxHxm4OpuqGBPsRvK3AEuDYzXw2cB7ynqnsTsCMz1wA7quWmuwbY07b8IeAjVQ8HgQ0Dqap3NwJfysxXAWfT6mVotkNErAb+ABjPzLNoHXhwBcOxHT4NXHjUurke+7cBa6qvjcDHl6jGbj7Nc3u4CzgrM38N+DZwPUD1O34F8NrqNn9fZdiSa2SgM6RvK5CZT2TmfdXlp2iFyGpatW+phm0BLh1Mhb2JiNOBi4BPVMsBnA9srYY0uoeIOBl4C3ATQGY+k5mHGLLtQOsotBMj4njghcATDMF2yMx7gB8ftXqux/4S4DPZ8hVgRUSsWppK59aph8z8t8w8Ui1+hda5N9DqYSozf5KZ3wUeoZVhS66pgb4a+H7b8r5q3dCIiDHgdcC9wGhmPgGt0AdeMrjKevJR4E+An1fLvwwcansyN317vBz4IfCpatroExFxEkO0HTJzP/A3wGO0gvwwsIvh2g7t5nrsh/V3/XeBL1aXG9NDUwO9p7cVaKqIGAE+D/xhZv73oOupIyIuBg5k5q721R2GNnl7HA+8Hvh4Zr4OeJoGT690Us0xXwKcCbwUOInW9MTRmrwdejFszy0i4v20pldvnl3VYdhAemhqoA/t2wpExPNohfnNmXlbtfrJ2X8jq+8HBlVfD94MvCMi9tKa6jqf1h77iupff2j+9tgH7MvMe6vlrbQCfpi2w1uB72bmDzPzp8BtwJsYru3Qbq7Hfqh+1yNiPXAxcGU+exJPY3poaqAP5dsKVHPNNwF7MvPDbVdtA9ZXl9cDdyx1bb3KzOsz8/TMHKP1uH85M68E7gYuq4Y1vYcfAN+PiFdWqy4AvskQbQdaUy3nRcQLq+fVbA9Dsx2OMtdjvw347epol/OAw7NTM00TERcC1wHvyMz/abtqG3BFRJwQEWfSeoH3PwdRI5nZyC/g7bReSf4O8P5B19Njzb9O61+tB4D7q6+305qD3gE8XH1fOehae+xnArizuvxyWk/SR4B/Bk4YdH1daj8H2Flti38BTh227QD8OfAt4EHgs8AJw7AdgFtozfv/lNbe64a5Hnta0xUfq37Pd9M6qqepPTxCa6589nf7H9rGv7/q4SHgbYOq21P/JakQTZ1ykSTVZKBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQvwft7E3eQzxT7UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe493ec7cc0>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = pd.read_csv('test.csv')\n",
    "\n",
    "data.head()\n",
    "\n",
    "ids = data['id']\n",
    "\n",
    "data.drop(columns='id', inplace=True)\n",
    "\n",
    "x_train = [el.split(' ') for el in data['text']]\n",
    "\n",
    "num_tokens = [len(tokens) for tokens in x_train]\n",
    "\n",
    "lengths = pd.DataFrame(data=num_tokens)\n",
    "\n",
    "lengths.head()\n",
    "\n",
    "lengths.describe()\n",
    "\n",
    "lengths.hist(bins=125)\n",
    "\n",
    "x_train = [el[0:125] for el in x_train]\n",
    "\n",
    "num_tokens = [len(tokens) for tokens in x_train]\n",
    "\n",
    "lengths = pd.DataFrame(data=num_tokens)\n",
    "\n",
    "lengths.head()\n",
    "\n",
    "lengths.describe()\n",
    "\n",
    "lengths.hist(bins=125)\n",
    "\n",
    "data['text'] = x_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The test set distribution is different from the training set, this would cause the test set results to be of lower accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[2705, 4888, 5050, 5815, 2472, 5157, 652, 2117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[389, 4978, 315, 5178, 513, 5249, 5853, 3267, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[4478, 4231, 4858, 2638, 4231, 867, 371, 686, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[3015, 1911, 112, 3905, 825, 337, 315, 1693, 4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[5136, 3918, 5153, 2023, 3091, 4159, 315, 3711...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0  [2705, 4888, 5050, 5815, 2472, 5157, 652, 2117...\n",
       "1  [389, 4978, 315, 5178, 513, 5249, 5853, 3267, ...\n",
       "2  [4478, 4231, 4858, 2638, 4231, 867, 371, 686, ...\n",
       "3  [3015, 1911, 112, 3905, 825, 337, 315, 1693, 4...\n",
       "4  [5136, 3918, 5153, 2023, 3091, 4159, 315, 3711..."
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = 'pre'\n",
    "x_test_pad = pad_sequences(data['text'], maxlen=max_tokens,\n",
    "                           padding=pad, truncating=pad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I've made an error above, the \"truncating=pad\" would lead to [45:125] being considered as the sequence for the longer sequences but in training [0:80] was being considered.\n",
    "# Given the longer sequences are very few, I don't think it would cause and major change is results even if this is fixed.\n",
    "# For the smaller sequence there wold be no effects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob = model.predict(x_test_pad, batch_size=512)\n",
    "y_pred = y_prob >= 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = y_pred.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.DataFrame(data=ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id\n",
       "0  3729\n",
       "1  3732\n",
       "2  3761\n",
       "3     5\n",
       "4     7"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "res['category'] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1]\n",
      " [1]\n",
      " [1]\n",
      " ...\n",
      " [1]\n",
      " [1]\n",
      " [1]]\n"
     ]
    }
   ],
   "source": [
    "print(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,    0,    0, ..., 1280, 2402, 2408],\n",
       "       [   0,    0,    0, ...,  819, 5773,  332],\n",
       "       [   0,    0,    0, ..., 1940, 3708, 3872],\n",
       "       ...,\n",
       "       [   0,    0,    0, ...,    0,    0, 7513],\n",
       "       [   0,    0,    0, ..., 1543, 2734, 3078],\n",
       "       [   0,    0,    0, ..., 4677, 5620, 3078]])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_x_train_pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(232, 80)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_x_train_pad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1360, 80)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_pad.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We got results of ~ 0.78 / 0.79 via skillenza's report for the above type of models\n",
    "# Given that the distributions of the test set is different and we have nearly 80% predictability for our model, we include our predicted results as true results and run the model again for 15 epoachs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_data = np.concatenate((data_x_train_pad, x_test_pad), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1592, 80)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(232,)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1360, 1)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_y = np.concatenate((np.array(data_y_train).reshape(232,1), y_pred), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1432 samples, validate on 160 samples\n",
      "Epoch 1/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1496 - acc: 0.9532 - val_loss: 0.0680 - val_acc: 1.0000\n",
      "Epoch 2/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1428 - acc: 0.9609 - val_loss: 0.0699 - val_acc: 0.9750\n",
      "Epoch 3/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1319 - acc: 0.9588 - val_loss: 0.0821 - val_acc: 0.9812\n",
      "Epoch 4/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1389 - acc: 0.9539 - val_loss: 0.0716 - val_acc: 1.0000\n",
      "Epoch 5/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1248 - acc: 0.9707 - val_loss: 0.0783 - val_acc: 0.9937\n",
      "Epoch 6/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1245 - acc: 0.9749 - val_loss: 0.0735 - val_acc: 0.9937\n",
      "Epoch 7/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1225 - acc: 0.9735 - val_loss: 0.0606 - val_acc: 1.0000\n",
      "Epoch 8/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1214 - acc: 0.9714 - val_loss: 0.0639 - val_acc: 0.9937\n",
      "Epoch 9/15\n",
      "1432/1432 [==============================] - 5s 4ms/step - loss: 0.1197 - acc: 0.9763 - val_loss: 0.0624 - val_acc: 0.9937\n",
      "Epoch 10/15\n",
      "1432/1432 [==============================] - 6s 4ms/step - loss: 0.1189 - acc: 0.9756 - val_loss: 0.0608 - val_acc: 0.9937\n",
      "Epoch 11/15\n",
      "1432/1432 [==============================] - 3s 2ms/step - loss: 0.1178 - acc: 0.9763 - val_loss: 0.0529 - val_acc: 0.9937\n",
      "Epoch 12/15\n",
      "1432/1432 [==============================] - 4s 2ms/step - loss: 0.1180 - acc: 0.9735 - val_loss: 0.0536 - val_acc: 0.9812\n",
      "Epoch 13/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1173 - acc: 0.9721 - val_loss: 0.0537 - val_acc: 1.0000\n",
      "Epoch 14/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1180 - acc: 0.9763 - val_loss: 0.0533 - val_acc: 0.9812\n",
      "Epoch 15/15\n",
      "1432/1432 [==============================] - 5s 3ms/step - loss: 0.1204 - acc: 0.9679 - val_loss: 0.0479 - val_acc: 1.0000\n",
      "Wall time: 1min 9s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xe48c9fbf28>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(new_train_data, new_train_y,\n",
    "          validation_split=0.1, epochs=15, batch_size=256, callbacks=[savebestmodel])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob = model.predict(x_test_pad, batch_size=512)\n",
    "y_pred = y_prob >= 0.5\n",
    "y_pred = y_pred.astype(int)\n",
    "res = pd.DataFrame(data=ids)\n",
    "res['category'] = y_pred\n",
    "res.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This gives us the extra 1 to 2% boost in result accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "## Things that could have been tried\n",
    "<ol>\n",
    "    <li>Different way to compress data</li>\n",
    "    <li>Other RNN networks like LSTM's</li>\n",
    "    <li>Have a different model for greater than 80 sequences examples</li>\n",
    "    <li>Different architectures</li>\n",
    "    <li>Ensembles models for the various models we build</li>\n",
    "</ol>\n",
    "\n",
    "### It was a nice challenge, You can check my other works at https://git.io/fjf20\n",
    "### Thank you"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}