{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.preprocessing.image import ImageDataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.load('data\\X_train.npy')\n",
    "y = np.load('data\\y.npy')\n",
    "X_test = np.load('data\\X_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 28, 28, 1)\n",
      "(42000, 10)\n",
      "(28000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct data generator which applies random transformations to the images to make our data more robust\n",
    "\n",
    "datagen = ImageDataGenerator(\n",
    "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        zoom_range = 0.1, # Randomly zoom image \n",
    "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "        height_shift_range=0.1,)  # randomly shift images vertically (fraction of total height)\n",
    "\n",
    "\n",
    "datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_LeNet_model(batch_size = 86, epochs=10):\n",
    "    model = tf.keras.models.Sequential([\n",
    "        # Convolutional layers\n",
    "        tf.keras.layers.Conv2D(6, (5,5), activation='relu', input_shape=(28,28,1), padding='same', strides=(1,1)),\n",
    "        tf.keras.layers.MaxPooling2D(2,2),\n",
    "        tf.keras.layers.Conv2D(16, (5,5), activation='relu', padding='valid', strides=(1,1)),\n",
    "        tf.keras.layers.MaxPooling2D(2,2),\n",
    "        tf.keras.layers.Conv2D(120, (5,5), activation='relu', padding='valid', strides=(1,1)),\n",
    "        \n",
    "        # Flatten out\n",
    "        tf.keras.layers.Flatten(),\n",
    "        \n",
    "        # Dense layers\n",
    "        tf.keras.layers.Dense(84, activation='relu'),\n",
    "        \n",
    "        # Output layer\n",
    "        tf.keras.layers.Dense(10, activation='softmax')\n",
    "        \n",
    "    ])\n",
    "    \n",
    "    # Compile\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer='SGD', metrics=['accuracy'])\n",
    "    \n",
    "    # Fit the model\n",
    "    history = model.fit_generator(datagen.flow(X_train,y, batch_size=batch_size),\n",
    "                                  epochs = epochs, \n",
    "                                  verbose = 2, \n",
    "                                  steps_per_epoch=X_train.shape[0] // batch_size)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isfile('models/LeNet.h5'):\n",
    "    LeNet_model = tf.keras.models.load_model('models/LeNet.h5')\n",
    "else:\n",
    "    LeNet_model = get_LeNet_model()\n",
    "    tf.keras.models.save_model('models/LeNet.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_Dropout_model(batch_size = 86, epochs=30):\n",
    "    model = tf.keras.models.Sequential([\n",
    "        # Convolutional layers\n",
    "        tf.keras.layers.Conv2D(32, (5,5), activation='relu', input_shape=(28,28,1), padding='same', strides=(1,1)),\n",
    "        tf.keras.layers.Conv2D(32, (5,5), activation='relu', padding='same', strides=(1,1)),\n",
    "        tf.keras.layers.MaxPooling2D(2,2),\n",
    "        tf.keras.layers.Dropout(0.25),\n",
    "        \n",
    "        tf.keras.layers.Conv2D(64, (5,5), activation='relu', padding='same', strides=(1,1)),\n",
    "        tf.keras.layers.Conv2D(64, (5,5), activation='relu', padding='same', strides=(1,1)),\n",
    "        tf.keras.layers.MaxPooling2D(2,2),\n",
    "        tf.keras.layers.Dropout(0.25),\n",
    "        \n",
    "        # Flatten out\n",
    "        tf.keras.layers.Flatten(),\n",
    "        \n",
    "        # Dense layers\n",
    "        tf.keras.layers.Dense(256, activation='relu'),\n",
    "        tf.keras.layers.Dropout(0.5),\n",
    "        \n",
    "        # Output layer\n",
    "        tf.keras.layers.Dense(10, activation='softmax')\n",
    "        \n",
    "    ])\n",
    "    \n",
    "    # Compile\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer='SGD', metrics=['accuracy'])\n",
    "    \n",
    "    # Fit the model\n",
    "    history = model.fit_generator(datagen.flow(X_train,y, batch_size=batch_size),\n",
    "                                  epochs = epochs, \n",
    "                                  verbose = 2, \n",
    "                                  steps_per_epoch=X_train.shape[0] // batch_size)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isfile('models/Dropout.h5'):\n",
    "    Dropout_model = tf.keras.models.load_model('models/Dropout.h5')\n",
    "else:\n",
    "    Dropout_model = get_Dropout_model()\n",
    "    tf.keras.models.save_model(Dropout_model, 'models/Dropout.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "submissions = []\n",
    "\n",
    "for model in [LeNet_model, Dropout_model]:\n",
    "    # predict results\n",
    "    results = model.predict(X_test)\n",
    "\n",
    "    # select the indix with the maximum probability\n",
    "    results = np.argmax(results,axis = 1)\n",
    "\n",
    "    results = pd.Series(results,name=\"Label\")\n",
    "    \n",
    "    submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n",
    "    \n",
    "    submissions.append(submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "submissions[0].to_csv(\"submissions/LeNet.csv\",index=False)\n",
    "submissions[1].to_csv(\"submissions/Dropout.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0.98685 test accuracy with LeNet style model\n",
    "# 0.99400 test accuracy with dropout model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}