{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "import urllib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import joblib\n",
    "from pandas.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.model_selection import StratifiedShuffleSplit as SSS\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml2/master/\"\n",
    "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n",
    "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):\n",
    "    os.makedirs(housing_path, exist_ok = True) # Create directory if not already there\n",
    "    tgz_path = os.path.join(housing_path, 'housing.tgz') # Make path for our tgz file\n",
    "    urllib.request.urlretrieve(housing_url, tgz_path) # Download the file\n",
    "    housing_tgz = tarfile.open(tgz_path) # Open the file\n",
    "    housing_tgz.extractall(path=housing_path) # Extract from tarfile\n",
    "    housing_tgz.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_housing_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_housing_data(housing_path=HOUSING_PATH):\n",
    "    csv_path = os.path.join(housing_path, 'housing.csv')\n",
    "    return pd.read_csv(csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>median_house_value</th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-122.23</td>\n",
       "      <td>37.88</td>\n",
       "      <td>41.0</td>\n",
       "      <td>880.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>8.3252</td>\n",
       "      <td>452600.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-122.22</td>\n",
       "      <td>37.86</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7099.0</td>\n",
       "      <td>1106.0</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>1138.0</td>\n",
       "      <td>8.3014</td>\n",
       "      <td>358500.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-122.24</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1467.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>496.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>7.2574</td>\n",
       "      <td>352100.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1274.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>558.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>5.6431</td>\n",
       "      <td>341300.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1627.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>565.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>3.8462</td>\n",
       "      <td>342200.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "0    -122.23     37.88                41.0        880.0           129.0   \n",
       "1    -122.22     37.86                21.0       7099.0          1106.0   \n",
       "2    -122.24     37.85                52.0       1467.0           190.0   \n",
       "3    -122.25     37.85                52.0       1274.0           235.0   \n",
       "4    -122.25     37.85                52.0       1627.0           280.0   \n",
       "\n",
       "   population  households  median_income  median_house_value ocean_proximity  \n",
       "0       322.0       126.0         8.3252            452600.0        NEAR BAY  \n",
       "1      2401.0      1138.0         8.3014            358500.0        NEAR BAY  \n",
       "2       496.0       177.0         7.2574            352100.0        NEAR BAY  \n",
       "3       558.0       219.0         5.6431            341300.0        NEAR BAY  \n",
       "4       565.0       259.0         3.8462            342200.0        NEAR BAY  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing = load_housing_data()\n",
    "housing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20640 entries, 0 to 20639\n",
      "Data columns (total 10 columns):\n",
      "longitude             20640 non-null float64\n",
      "latitude              20640 non-null float64\n",
      "housing_median_age    20640 non-null float64\n",
      "total_rooms           20640 non-null float64\n",
      "total_bedrooms        20433 non-null float64\n",
      "population            20640 non-null float64\n",
      "households            20640 non-null float64\n",
      "median_income         20640 non-null float64\n",
      "median_house_value    20640 non-null float64\n",
      "ocean_proximity       20640 non-null object\n",
      "dtypes: float64(9), object(1)\n",
      "memory usage: 1.6+ MB\n"
     ]
    }
   ],
   "source": [
    "housing.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1H OCEAN     9136\n",
       "INLAND        6551\n",
       "NEAR OCEAN    2658\n",
       "NEAR BAY      2290\n",
       "ISLAND           5\n",
       "Name: ocean_proximity, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can see that ocean_proximity is the only non numerical feature\n",
    "# We use value_counts to look at what the categories consist of\n",
    "\n",
    "housing['ocean_proximity'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>median_house_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20433.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "      <td>20640.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>-119.569704</td>\n",
       "      <td>35.631861</td>\n",
       "      <td>28.639486</td>\n",
       "      <td>2635.763081</td>\n",
       "      <td>537.870553</td>\n",
       "      <td>1425.476744</td>\n",
       "      <td>499.539680</td>\n",
       "      <td>3.870671</td>\n",
       "      <td>206855.816909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>2.003532</td>\n",
       "      <td>2.135952</td>\n",
       "      <td>12.585558</td>\n",
       "      <td>2181.615252</td>\n",
       "      <td>421.385070</td>\n",
       "      <td>1132.462122</td>\n",
       "      <td>382.329753</td>\n",
       "      <td>1.899822</td>\n",
       "      <td>115395.615874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>-124.350000</td>\n",
       "      <td>32.540000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.499900</td>\n",
       "      <td>14999.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>-121.800000</td>\n",
       "      <td>33.930000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1447.750000</td>\n",
       "      <td>296.000000</td>\n",
       "      <td>787.000000</td>\n",
       "      <td>280.000000</td>\n",
       "      <td>2.563400</td>\n",
       "      <td>119600.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>-118.490000</td>\n",
       "      <td>34.260000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>2127.000000</td>\n",
       "      <td>435.000000</td>\n",
       "      <td>1166.000000</td>\n",
       "      <td>409.000000</td>\n",
       "      <td>3.534800</td>\n",
       "      <td>179700.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>-118.010000</td>\n",
       "      <td>37.710000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>3148.000000</td>\n",
       "      <td>647.000000</td>\n",
       "      <td>1725.000000</td>\n",
       "      <td>605.000000</td>\n",
       "      <td>4.743250</td>\n",
       "      <td>264725.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>-114.310000</td>\n",
       "      <td>41.950000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>39320.000000</td>\n",
       "      <td>6445.000000</td>\n",
       "      <td>35682.000000</td>\n",
       "      <td>6082.000000</td>\n",
       "      <td>15.000100</td>\n",
       "      <td>500001.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          longitude      latitude  housing_median_age   total_rooms  \\\n",
       "count  20640.000000  20640.000000        20640.000000  20640.000000   \n",
       "mean    -119.569704     35.631861           28.639486   2635.763081   \n",
       "std        2.003532      2.135952           12.585558   2181.615252   \n",
       "min     -124.350000     32.540000            1.000000      2.000000   \n",
       "25%     -121.800000     33.930000           18.000000   1447.750000   \n",
       "50%     -118.490000     34.260000           29.000000   2127.000000   \n",
       "75%     -118.010000     37.710000           37.000000   3148.000000   \n",
       "max     -114.310000     41.950000           52.000000  39320.000000   \n",
       "\n",
       "       total_bedrooms    population    households  median_income  \\\n",
       "count    20433.000000  20640.000000  20640.000000   20640.000000   \n",
       "mean       537.870553   1425.476744    499.539680       3.870671   \n",
       "std        421.385070   1132.462122    382.329753       1.899822   \n",
       "min          1.000000      3.000000      1.000000       0.499900   \n",
       "25%        296.000000    787.000000    280.000000       2.563400   \n",
       "50%        435.000000   1166.000000    409.000000       3.534800   \n",
       "75%        647.000000   1725.000000    605.000000       4.743250   \n",
       "max       6445.000000  35682.000000   6082.000000      15.000100   \n",
       "\n",
       "       median_house_value  \n",
       "count        20640.000000  \n",
       "mean        206855.816909  \n",
       "std         115395.615874  \n",
       "min          14999.000000  \n",
       "25%         119600.000000  \n",
       "50%         179700.000000  \n",
       "75%         264725.000000  \n",
       "max         500001.000000  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58696 characters]\n",
      "text/plain": [
       "<Figure size 1440x1080 with 9 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot histograms for all the features\n",
    "\n",
    "housing.hist(bins=50, figsize=(20,15))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that most features are left scewed (tail-heavy) or multi-modal. Not the most ideal distributions of data. Also worth noting is that the median_income data is not in USD and seems to be scaled from 0 to 15. If we look back at the max and min of the data we find that it is scaled to be from 0.4999 to 15.0001. Similarly the median age and median house values are capped (as seen by the sharp jump at the end). It might be nice to transform the data to be more uniform."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8488 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "housing['income_cat'] = pd.cut(housing['median_income'], # Chopping up median income because it's important\n",
    "                              bins=[0., 1.5, 3.0, 4.5, 6., np.inf], # Chop into categories of 0 to 1.5, 1.5 to 3, etc\n",
    "                              labels=[1,2,3,4,5]) # Generic labels\n",
    "train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42) # Generic train/test split\n",
    "housing['income_cat'].hist()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we want to do some stratified sampling based on median_income which seems to be fairly important\n",
    "\n",
    "split = SSS(n_splits=1, test_size=0.2, random_state=42) # Initialize our split with proper test size/random state to match book\n",
    "for train_index, test_index in split.split(housing, housing['income_cat']):\n",
    "    strat_train_set = housing.loc[train_index]\n",
    "    strat_test_set = housing.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def income_cat_proportions(data):\n",
    "    return data[\"income_cat\"].value_counts() / len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a table showing the how a stratified sampling ensures our test set matches our original distribution for median_income\n",
    "\n",
    "compare_props = pd.DataFrame({\n",
    "    \"Overall\": income_cat_proportions(housing),\n",
    "    \"Stratified\": income_cat_proportions(strat_test_set),\n",
    "    \"Random\": income_cat_proportions(test_set),\n",
    "}).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Overall</th>\n",
       "      <th>Stratified</th>\n",
       "      <th>Random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.039826</td>\n",
       "      <td>0.039729</td>\n",
       "      <td>0.040213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.318847</td>\n",
       "      <td>0.318798</td>\n",
       "      <td>0.324370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.350581</td>\n",
       "      <td>0.350533</td>\n",
       "      <td>0.358527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.176308</td>\n",
       "      <td>0.176357</td>\n",
       "      <td>0.167393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.114438</td>\n",
       "      <td>0.114583</td>\n",
       "      <td>0.109496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Overall  Stratified    Random\n",
       "1  0.039826    0.039729  0.040213\n",
       "2  0.318847    0.318798  0.324370\n",
       "3  0.350581    0.350533  0.358527\n",
       "4  0.176308    0.176357  0.167393\n",
       "5  0.114438    0.114583  0.109496"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_props.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now that we finished with our income_cat feature we can drop it\n",
    "\n",
    "for set_ in (strat_train_set, strat_test_set):\n",
    "    set_.drop('income_cat', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With our test set put aside we can explore the data\n",
    "\n",
    "housing = strat_train_set.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31912 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Simple scatter plot\n",
    "\n",
    "housing.plot(kind='scatter', x='longitude', y='latitude')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 57320 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Scatter plot emphasizing denser areas\n",
    "\n",
    "housing.plot(kind='scatter', x='longitude', y='latitude', alpha=0.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 125944 characters]\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "housing.plot(kind='scatter', x='longitude', y='latitude', alpha=0.4, \n",
    "            s = housing['population'] / 100,  # Adjusit size of points to size of population\n",
    "            label='population', \n",
    "            figsize=(10,7), \n",
    "            c = 'median_house_value', # Base color map on median house value\n",
    "            cmap=plt.get_cmap('jet'), \n",
    "            colorbar=True)\n",
    "plt.legend()\n",
    "plt.xlabel=('longitude')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "median_house_value    1.000000\n",
       "median_income         0.687160\n",
       "total_rooms           0.135097\n",
       "housing_median_age    0.114110\n",
       "households            0.064506\n",
       "total_bedrooms        0.047689\n",
       "population           -0.026920\n",
       "longitude            -0.047432\n",
       "latitude             -0.142724\n",
       "Name: median_house_value, dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check out correlations\n",
    "\n",
    "corr_matrix = housing.corr()\n",
    "corr_matrix['median_house_value'].sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see a strong correlation between house value and median income which is to be expected. However, we see almost no correlation geographically even though our heat map earlier suggests a strong corrolation between location and price. This just means the features have a non linear relation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 302312 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Grab scatter plots between most promising features to inspect\n",
    "\n",
    "attributes= ['median_house_value', 'median_income', 'total_rooms', 'housing_median_age']\n",
    "scatter_matrix(housing[attributes], figsize=(12,8))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 82940 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Strongest looking correlation is between median income and median house value so we look more closely at that\n",
    "\n",
    "housing.plot(kind='scatter', x='median_income', y='median_house_value', alpha=0.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see a strong correlation as well as a few oddities. There are some abnormal horizontal lines at 500k, 350k and possible 250k that may have to be dealt with later. Now we can move on to feature creation! Some useful features could be rooms per household (more useful than just the number of rooms), bedrooms per room (proportion of rooms which are actually bedrooms), and people per household."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing['rooms_per_household'] = housing['total_rooms'] / housing['households']\n",
    "housing['bedrooms_per_room'] = housing['total_bedrooms'] / housing['total_rooms']\n",
    "housing['population_per_household'] = housing['population'] / housing['households']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "median_house_value          1.000000\n",
       "median_income               0.687160\n",
       "rooms_per_household         0.146285\n",
       "total_rooms                 0.135097\n",
       "housing_median_age          0.114110\n",
       "households                  0.064506\n",
       "total_bedrooms              0.047689\n",
       "population_per_household   -0.021985\n",
       "population                 -0.026920\n",
       "longitude                  -0.047432\n",
       "latitude                   -0.142724\n",
       "bedrooms_per_room          -0.259984\n",
       "Name: median_house_value, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_matrix = housing.corr()\n",
    "corr_matrix['median_house_value'].sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now see our artificial features and how correlated they are to the target feature with the most promising being rooms_per_household and bedrooms_per_room with a strong negative correlation. This makes intuitive sense when you consider a 4 bedroom 2.5 bath compared to a 3 bedroom 1 bath. Anothing factor could be extra non bedrooms such as extra living rooms and kitchens driving the price up considerably. With some decent correlations on our features we can move on to preping our data for ML algorithms."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing = strat_train_set.drop('median_house_value', axis=1)\n",
    "housing_labels = strat_train_set['median_house_value'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16512 entries, 17606 to 15775\n",
      "Data columns (total 9 columns):\n",
      "longitude             16512 non-null float64\n",
      "latitude              16512 non-null float64\n",
      "housing_median_age    16512 non-null float64\n",
      "total_rooms           16512 non-null float64\n",
      "total_bedrooms        16354 non-null float64\n",
      "population            16512 non-null float64\n",
      "households            16512 non-null float64\n",
      "median_income         16512 non-null float64\n",
      "ocean_proximity       16512 non-null object\n",
      "dtypes: float64(8), object(1)\n",
      "memory usage: 1.3+ MB\n"
     ]
    }
   ],
   "source": [
    "housing.info() # Bring back up to see where we have null values (in total_bedrooms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Median imputation using SkLearn\n",
    "\n",
    "imputer = SimpleImputer(strategy='median')\n",
    "housing_num = housing.drop('ocean_proximity', axis=1) # Drop non numerical feature for imputation\n",
    "imputer.fit(housing_num) # Computes and stores median values for all features in housing_num which we may need later\n",
    "X = imputer.transform(housing_num) # Apply median imputation\n",
    "housing_tr = pd.DataFrame(X, columns=housing_num.columns, index=housing_num.index) # Remake our df from our numpy array X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>17606</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18632</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14650</td>\n",
       "      <td>NEAR OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3230</td>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3555</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19480</td>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8879</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13685</td>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4937</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4861</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ocean_proximity\n",
       "17606       <1H OCEAN\n",
       "18632       <1H OCEAN\n",
       "14650      NEAR OCEAN\n",
       "3230           INLAND\n",
       "3555        <1H OCEAN\n",
       "19480          INLAND\n",
       "8879        <1H OCEAN\n",
       "13685          INLAND\n",
       "4937        <1H OCEAN\n",
       "4861        <1H OCEAN"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check out our categorical feature\n",
    "housing_cat = housing[['ocean_proximity']] # doubles brackets to get a dataframe instead of a series\n",
    "housing_cat.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17606     <1H OCEAN\n",
       "18632     <1H OCEAN\n",
       "14650    NEAR OCEAN\n",
       "3230         INLAND\n",
       "3555      <1H OCEAN\n",
       "            ...    \n",
       "6563         INLAND\n",
       "12053        INLAND\n",
       "13908        INLAND\n",
       "11159     <1H OCEAN\n",
       "15775      NEAR BAY\n",
       "Name: ocean_proximity, Length: 16512, dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing['ocean_proximity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<16512x5 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 16512 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_encoder = OneHotEncoder()\n",
    "housing_cat_1hot = cat_encoder.fit_transform(housing_cat)\n",
    "housing_cat_1hot # Note this is stored as a sparse matrix since most of the matrix would be empy anyways"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 1.],\n",
       "       ...,\n",
       "       [0., 1., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_cat_1hot.toarray() # What our matrix looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array(['<1H OCEAN', 'INLAND', 'ISLAND', 'NEAR BAY', 'NEAR OCEAN'],\n",
       "       dtype=object)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_encoder.categories_ # List all our categories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SkLearn is very useful, but sometimes we need to create our own functions to modify the data. In doing so it is important to stay compatible with SkLearn by creating our own classes with the methods of fit(), transform(), and fit_transform(). Given example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "# column index\n",
    "rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6\n",
    "\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, add_bedrooms_per_room = True): # no *args or **kargs\n",
    "        self.add_bedrooms_per_room = add_bedrooms_per_room\n",
    "    def fit(self, X, y=None):\n",
    "        return self  # nothing else to do\n",
    "    def transform(self, X):\n",
    "        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]\n",
    "        population_per_household = X[:, population_ix] / X[:, households_ix]\n",
    "        if self.add_bedrooms_per_room:\n",
    "            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]\n",
    "            return np.c_[X, rooms_per_household, population_per_household,\n",
    "                         bedrooms_per_room]\n",
    "        else:\n",
    "            return np.c_[X, rooms_per_household, population_per_household]\n",
    "\n",
    "attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)\n",
    "housing_extra_attribs = attr_adder.transform(housing.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup a pipeline to do all we've done already for numerical features\n",
    "# Note that all estimators but the last must be transformers(specifically they must have a\n",
    "# fit_transform() method)\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "    ('imputer', SimpleImputer(strategy='median')),\n",
    "    ('attribs_adder', CombinedAttributesAdder()), \n",
    "    ('std_scaler', StandardScaler())\n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_num_tr = num_pipeline.fit_transform(housing_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our previous pipeline applies to all columns in a dataframe. To deal with both our categorical and\n",
    "# numerical features at once we use ColumnTransformer to perform our pipeline only on numerical features\n",
    "# and OneHotEncoder only on categorical features\n",
    "\n",
    "num_attribs = list(housing_num)\n",
    "cat_attribs = ['ocean_proximity']\n",
    "\n",
    "full_pipeline = ColumnTransformer([\n",
    "    ('num', num_pipeline, num_attribs), \n",
    "    ('cat', OneHotEncoder(), cat_attribs)\n",
    "])\n",
    "\n",
    "housing_prepared = full_pipeline.fit_transform(housing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try out simple model\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(housing_prepared, housing_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68628.19819848923"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check out our error on training set\n",
    "\n",
    "lin_reg_pred = lin_reg.predict(housing_prepared)\n",
    "lin_mse = mean_squared_error(housing_labels, lin_reg_pred)\n",
    "lin_rmse = np.sqrt(lin_mse)\n",
    "lin_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Better evaluation with Cross Validation (k-fold)\n",
    "\n",
    "scores = cross_val_score(lin_reg, housing_prepared, housing_labels,\n",
    "                        scoring='neg_mean_squared_error', cv=10)\n",
    "lin_reg_rmse_scores = np.sqrt(-scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_scores(scores):\n",
    "    print('Scores: ', scores)\n",
    "    print('Mean: ', scores.mean())\n",
    "    print('Standard Deviation: ', scores.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores:  [66782.73843989 66960.118071   70347.95244419 74739.57052552\n",
      " 68031.13388938 71193.84183426 64969.63056405 68281.61137997\n",
      " 71552.91566558 67665.10082067]\n",
      "Mean:  69052.46136345083\n",
      "Standard Deviation:  2731.6740017983425\n"
     ]
    }
   ],
   "source": [
    "display_scores(lin_reg_rmse_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\tsb\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "22020.531016739864"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Random Forest\n",
    "\n",
    "forest_reg = RandomForestRegressor()\n",
    "forest_reg.fit(housing_prepared, housing_labels)\n",
    "forest_reg_pred = forest_reg.predict(housing_prepared)\n",
    "forest_reg_mse = mean_squared_error(housing_labels, forest_reg_pred)\n",
    "forest_reg_rmse = np.sqrt(forest_reg_mse)\n",
    "forest_reg_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get k-fold scores\n",
    "\n",
    "forest_reg_scores = cross_val_score(forest_reg, housing_prepared, housing_labels,\n",
    "                        scoring='neg_mean_squared_error', cv=10)\n",
    "forest_reg_rmse_scores = np.sqrt(-forest_reg_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores:  [51471.26384635 50121.66522618 53509.84575046 54704.7006039\n",
      " 52832.4840061  57155.84968564 52357.59400134 51823.56197907\n",
      " 56258.53544821 52717.53079744]\n",
      "Mean:  53295.303134468486\n",
      "Standard Deviation:  2066.62611382041\n"
     ]
    }
   ],
   "source": [
    "# Display\n",
    "\n",
    "display_scores(forest_reg_rmse_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/forest_reg.pkl']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save our models\n",
    "\n",
    "joblib.dump(lin_reg, \"models/lin_reg.pkl\")\n",
    "joblib.dump(forest_reg, \"models/forest_reg.pkl\")\n",
    "\n",
    "# Note: can load back in with joblib.load('path/model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=RandomForestRegressor(bootstrap=True, criterion='mse',\n",
       "                                             max_depth=None,\n",
       "                                             max_features='auto',\n",
       "                                             max_leaf_nodes=None,\n",
       "                                             min_impurity_decrease=0.0,\n",
       "                                             min_impurity_split=None,\n",
       "                                             min_samples_leaf=1,\n",
       "                                             min_samples_split=2,\n",
       "                                             min_weight_fraction_leaf=0.0,\n",
       "                                             n_estimators='warn', n_jobs=None,\n",
       "                                             oob_score=False, random_state=None,\n",
       "                                             verbose=0, warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid=[{'max_features': [2, 4, 6, 8],\n",
       "                          'n_estimators': [3, 10, 30]},\n",
       "                         {'bootstrap': [False], 'max_features': [2, 3, 4],\n",
       "                          'n_estimators': [3, 10]}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='neg_mean_squared_error', verbose=0)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fine tuning our models (Grid Search)\n",
    "\n",
    "param_grid = [\n",
    "    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},\n",
    "    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]}\n",
    "]\n",
    "\n",
    "forest_reg = RandomForestRegressor()\n",
    "\n",
    "grid_search = GridSearchCV(forest_reg, param_grid, cv=5,\n",
    "                          scoring='neg_mean_squared_error',\n",
    "                          return_train_score=True)\n",
    "\n",
    "grid_search.fit(housing_prepared, housing_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_features': 6, 'n_estimators': 30}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "                      max_features=6, max_leaf_nodes=None,\n",
       "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                      min_samples_leaf=1, min_samples_split=2,\n",
       "                      min_weight_fraction_leaf=0.0, n_estimators=30,\n",
       "                      n_jobs=None, oob_score=False, random_state=None,\n",
       "                      verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63766.249031932166 {'max_features': 2, 'n_estimators': 3}\n",
      "55241.42801213305 {'max_features': 2, 'n_estimators': 10}\n",
      "52535.380892189394 {'max_features': 2, 'n_estimators': 30}\n",
      "59700.32170336908 {'max_features': 4, 'n_estimators': 3}\n",
      "52924.53101060204 {'max_features': 4, 'n_estimators': 10}\n",
      "50393.77338577394 {'max_features': 4, 'n_estimators': 30}\n",
      "59301.43966695725 {'max_features': 6, 'n_estimators': 3}\n",
      "52035.17103216119 {'max_features': 6, 'n_estimators': 10}\n",
      "49813.18816875522 {'max_features': 6, 'n_estimators': 30}\n",
      "58081.320376052456 {'max_features': 8, 'n_estimators': 3}\n",
      "51737.611817250836 {'max_features': 8, 'n_estimators': 10}\n",
      "50338.53670533141 {'max_features': 8, 'n_estimators': 30}\n",
      "62649.34948878523 {'bootstrap': False, 'max_features': 2, 'n_estimators': 3}\n",
      "54152.03055272061 {'bootstrap': False, 'max_features': 2, 'n_estimators': 10}\n",
      "59796.1703788161 {'bootstrap': False, 'max_features': 3, 'n_estimators': 3}\n",
      "52826.151917810435 {'bootstrap': False, 'max_features': 3, 'n_estimators': 10}\n",
      "58196.408759742335 {'bootstrap': False, 'max_features': 4, 'n_estimators': 3}\n",
      "51906.57937919814 {'bootstrap': False, 'max_features': 4, 'n_estimators': 10}\n"
     ]
    }
   ],
   "source": [
    "# Display rmse for each of the models tested in our Grid Search\n",
    "\n",
    "cvres = grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres['mean_test_score'], cvres['params']):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.29557797449661544, 'median_income'),\n",
       " (0.15501755658961477, 'INLAND'),\n",
       " (0.10808150971584977, 'pop_per_hhold'),\n",
       " (0.09567946033989604, 'bedrooms_per_room'),\n",
       " (0.08025900590023524, 'longitude'),\n",
       " (0.07448394445267537, 'latitude'),\n",
       " (0.06378150432622012, 'rooms_per_hhold'),\n",
       " (0.04012776886007714, 'housing_median_age'),\n",
       " (0.017890873234094058, 'total_rooms'),\n",
       " (0.016940497934044608, 'population'),\n",
       " (0.01665603928514349, 'households'),\n",
       " (0.01572966535262306, 'total_bedrooms'),\n",
       " (0.010971161123643354, '<1H OCEAN'),\n",
       " (0.004910668398105027, 'NEAR OCEAN'),\n",
       " (0.003847444862073042, 'NEAR BAY'),\n",
       " (4.4925129089441225e-05, 'ISLAND')]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look into how important each feature is in deciding our output and consider dropping less important\n",
    "\n",
    "feature_importances = grid_search.best_estimator_.feature_importances_\n",
    "\n",
    "extra_attribs = ['rooms_per_hhold', 'pop_per_hhold', 'bedrooms_per_room']\n",
    "cat_encoder = full_pipeline.named_transformers_['cat']\n",
    "cat_one_hot_attribs = list(cat_encoder.categories_[0])\n",
    "attributes = num_attribs + extra_attribs + cat_one_hot_attribs\n",
    "sorted(zip(feature_importances, attributes), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final post tuning evaluation on our test set\n",
    "\n",
    "final_model = grid_search.best_estimator_\n",
    "\n",
    "X_test = strat_test_set.drop('median_house_value', axis=1)\n",
    "y_test = strat_test_set['median_house_value']\n",
    "\n",
    "X_test_prepared = full_pipeline.transform(X_test)\n",
    "\n",
    "final_pred = final_model.predict(X_test_prepared)\n",
    "\n",
    "final_mse = mean_squared_error(y_test, final_pred)\n",
    "final_rmse = np.sqrt(final_mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47702.116203907615\n"
     ]
    }
   ],
   "source": [
    "# Definitive RMSE score for our tuned model\n",
    "\n",
    "print(final_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([45750.65472887, 49576.82293716])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 95% confidence interval for our RMSE\n",
    "\n",
    "confidence = 0.95\n",
    "squared_errors = (final_pred - y_test) ** 2\n",
    "np.sqrt(stats.t.interval(confidence, len(squared_errors) - 1,\n",
    "                        loc=squared_errors.mean(),\n",
    "                        scale=stats.sem(squared_errors)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 95% sure our RMSE on actual data would be between 45750 and 49576"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/final_model.pkl']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save our model\n",
    "\n",
    "joblib.dump(final_model, 'models/final_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
