{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import datasets\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn import datasets\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "X = iris[\"data\"][:, (2, 3)]  # petal length, petal width\n",
    "y = iris[\"target\"]\n",
    "\n",
    "setosa_or_versicolor = (y == 0) | (y == 1)\n",
    "X = X[setosa_or_versicolor]\n",
    "y = y[setosa_or_versicolor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the data\n",
    "\n",
    "def plot_setosa_versicolor(X = X, y = y):\n",
    "    plt.plot(X[:,0][y==0], X[:,1][y==0], 'bo', label='iris setosa')\n",
    "    plt.plot(X[:,0][y==1], X[:,1][y==1], 'r^', label='iris verticolor')\n",
    "\n",
    "    plt.xlabel('petal length', fontsize=15)\n",
    "    plt.ylabel('petal width', fontsize=15)\n",
    "    plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13976 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_setosa_versicolor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**EXERCISE 8**\n",
    "\n",
    "Train LinearSVC on linearly seperable data, then train SVC and SGCClassifier and compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_decision_boundary(pipeline, X=X, y=y):\n",
    "    # Get params for our decision boundary\n",
    "    transfX = pipeline['scaler'].fit_transform(X)\n",
    "\n",
    "    Xmin = transfX.min()\n",
    "    Xmax = transfX.max()\n",
    "\n",
    "    b = pipeline['clf'].intercept_\n",
    "    w0 = pipeline['clf'].coef_[0][0]\n",
    "    w1 = pipeline['clf'].coef_[0][1]\n",
    "\n",
    "    # get our input values to build line\n",
    "    x0 = np.linspace(Xmin, Xmax, 200)\n",
    "\n",
    "    # Setup boundary\n",
    "    # b + w0x + w1y = 0  ==> y = -w0/w1 * x - b/w1\n",
    "    boundary = -(w0/w1) * x0 - b/w1\n",
    "    margin = 1/w1\n",
    "    top_gutter = boundary + margin\n",
    "    bot_gutter = boundary - margin\n",
    "\n",
    "    # Plot our boundary and gutters\n",
    "    plt.plot(x0, boundary, 'k-')\n",
    "    plt.plot(x0, top_gutter, 'r--')\n",
    "    plt.plot(x0, bot_gutter, 'b--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preprocess the data\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "Linear_SVM_clf = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('clf', LinearSVC()) \n",
    "])\n",
    "\n",
    "Linear_SVM_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24904 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_setosa_versicolor(X = transfX)\n",
    "plot_decision_boundary(Linear_SVM_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('scaler',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('clf',\n",
       "                 SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None,\n",
       "                     coef0=0.0, decision_function_shape='ovr', degree=3,\n",
       "                     gamma='scale', kernel='linear', max_iter=-1,\n",
       "                     probability=False, random_state=None, shrinking=True,\n",
       "                     tol=0.001, verbose=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "SVC_clf = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('clf', SVC(kernel='linear')) \n",
    "])\n",
    "\n",
    "SVC_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24824 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_setosa_versicolor(X = transfX)\n",
    "plot_decision_boundary(SVC_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('scaler',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('clf',\n",
       "                 SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "                               early_stopping=False, epsilon=0.1, eta0=0.001,\n",
       "                               fit_intercept=True, l1_ratio=0.15,\n",
       "                               learning_rate='constant', loss='hinge',\n",
       "                               max_iter=1000, n_iter_no_change=5, n_jobs=None,\n",
       "                               penalty='l2', power_t=0.5, random_state=42,\n",
       "                               shuffle=True, tol=0.001, validation_fraction=0.1,\n",
       "                               verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('clf', SGDClassifier(loss=\"hinge\", learning_rate=\"constant\", eta0=0.001,\n",
    "                        max_iter=1000, tol=1e-3, random_state=42)) \n",
    "])\n",
    "\n",
    "sgd_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23244 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_setosa_versicolor(X = transfX)\n",
    "plot_decision_boundary(sgd_clf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**EXERCISE 10**\n",
    "\n",
    "Train SVR on California Housing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X = housing[\"data\"]\n",
    "y = housing[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20640, 8)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('scaler',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('model',\n",
       "                 SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1,\n",
       "                     gamma='scale', kernel='rbf', max_iter=-1, shrinking=True,\n",
       "                     tol=0.001, verbose=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVR\n",
    "\n",
    "SVR_model = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('model', SVR())\n",
    "])\n",
    "\n",
    "SVR_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7275639524733043"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVR_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3570026426754465\n",
      "0.5974969813107398\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "y_pred = SVR_model.predict(X_test)\n",
    "mse = mean_squared_error(y_pred, y_test)\n",
    "rmse = np.sqrt(mse)\n",
    "print(mse)\n",
    "print(rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
      "[CV] model__C=8.732501769442347, model__gamma=0.014138684138012492 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__C=8.732501769442347, model__gamma=0.014138684138012492, total=  10.9s\n",
      "[CV] model__C=8.732501769442347, model__gamma=0.014138684138012492 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   10.8s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__C=8.732501769442347, model__gamma=0.014138684138012492, total=  10.7s\n",
      "[CV] model__C=8.732501769442347, model__gamma=0.014138684138012492 ...\n",
      "[CV]  model__C=8.732501769442347, model__gamma=0.014138684138012492, total=  10.9s\n",
      "[CV] model__C=10.6073996426796, model__gamma=0.0010782720368618492 ...\n",
      "[CV]  model__C=10.6073996426796, model__gamma=0.0010782720368618492, total=  10.1s\n",
      "[CV] model__C=10.6073996426796, model__gamma=0.0010782720368618492 ...\n",
      "[CV]  model__C=10.6073996426796, model__gamma=0.0010782720368618492, total=  10.1s\n",
      "[CV] model__C=10.6073996426796, model__gamma=0.0010782720368618492 ...\n",
      "[CV]  model__C=10.6073996426796, model__gamma=0.0010782720368618492, total=  10.0s\n",
      "[CV] model__C=5.173843032530275, model__gamma=0.03912018707260784 ....\n",
      "[CV]  model__C=5.173843032530275, model__gamma=0.03912018707260784, total=  11.1s\n",
      "[CV] model__C=5.173843032530275, model__gamma=0.03912018707260784 ....\n",
      "[CV]  model__C=5.173843032530275, model__gamma=0.03912018707260784, total=  11.1s\n",
      "[CV] model__C=5.173843032530275, model__gamma=0.03912018707260784 ....\n",
      "[CV]  model__C=5.173843032530275, model__gamma=0.03912018707260784, total=  11.2s\n",
      "[CV] model__C=8.757343573995623, model__gamma=0.026503976649874594 ...\n",
      "[CV]  model__C=8.757343573995623, model__gamma=0.026503976649874594, total=  11.2s\n",
      "[CV] model__C=8.757343573995623, model__gamma=0.026503976649874594 ...\n",
      "[CV]  model__C=8.757343573995623, model__gamma=0.026503976649874594, total=  11.6s\n",
      "[CV] model__C=8.757343573995623, model__gamma=0.026503976649874594 ...\n",
      "[CV]  model__C=8.757343573995623, model__gamma=0.026503976649874594, total=  11.8s\n",
      "[CV] model__C=3.993757431556655, model__gamma=0.0037627657376435165 ..\n",
      "[CV]  model__C=3.993757431556655, model__gamma=0.0037627657376435165, total=   9.9s\n",
      "[CV] model__C=3.993757431556655, model__gamma=0.0037627657376435165 ..\n",
      "[CV]  model__C=3.993757431556655, model__gamma=0.0037627657376435165, total=  10.1s\n",
      "[CV] model__C=3.993757431556655, model__gamma=0.0037627657376435165 ..\n",
      "[CV]  model__C=3.993757431556655, model__gamma=0.0037627657376435165, total=   7.3s\n",
      "[CV] model__C=5.908725733130206, model__gamma=0.04722302780443009 ....\n",
      "[CV]  model__C=5.908725733130206, model__gamma=0.04722302780443009, total=   6.4s\n",
      "[CV] model__C=5.908725733130206, model__gamma=0.04722302780443009 ....\n",
      "[CV]  model__C=5.908725733130206, model__gamma=0.04722302780443009, total=   8.2s\n",
      "[CV] model__C=5.908725733130206, model__gamma=0.04722302780443009 ....\n",
      "[CV]  model__C=5.908725733130206, model__gamma=0.04722302780443009, total=  11.8s\n",
      "[CV] model__C=6.339380352838232, model__gamma=0.004194342545426101 ...\n",
      "[CV]  model__C=6.339380352838232, model__gamma=0.004194342545426101, total=  10.3s\n",
      "[CV] model__C=6.339380352838232, model__gamma=0.004194342545426101 ...\n",
      "[CV]  model__C=6.339380352838232, model__gamma=0.004194342545426101, total=   5.7s\n",
      "[CV] model__C=6.339380352838232, model__gamma=0.004194342545426101 ...\n",
      "[CV]  model__C=6.339380352838232, model__gamma=0.004194342545426101, total=   9.1s\n",
      "[CV] model__C=9.323582401718916, model__gamma=0.011028101996364465 ...\n",
      "[CV]  model__C=9.323582401718916, model__gamma=0.011028101996364465, total=  10.7s\n",
      "[CV] model__C=9.323582401718916, model__gamma=0.011028101996364465 ...\n",
      "[CV]  model__C=9.323582401718916, model__gamma=0.011028101996364465, total=  11.0s\n",
      "[CV] model__C=9.323582401718916, model__gamma=0.011028101996364465 ...\n",
      "[CV]  model__C=9.323582401718916, model__gamma=0.011028101996364465, total=  11.6s\n",
      "[CV] model__C=7.485779608851653, model__gamma=0.03852395227919697 ....\n",
      "[CV]  model__C=7.485779608851653, model__gamma=0.03852395227919697, total=  11.9s\n",
      "[CV] model__C=7.485779608851653, model__gamma=0.03852395227919697 ....\n",
      "[CV]  model__C=7.485779608851653, model__gamma=0.03852395227919697, total=   6.6s\n",
      "[CV] model__C=7.485779608851653, model__gamma=0.03852395227919697 ....\n",
      "[CV]  model__C=7.485779608851653, model__gamma=0.03852395227919697, total=  11.9s\n",
      "[CV] model__C=5.32139202170268, model__gamma=0.00373396867904038 .....\n",
      "[CV]  model__C=5.32139202170268, model__gamma=0.00373396867904038, total=  10.0s\n",
      "[CV] model__C=5.32139202170268, model__gamma=0.00373396867904038 .....\n",
      "[CV]  model__C=5.32139202170268, model__gamma=0.00373396867904038, total=  10.0s\n",
      "[CV] model__C=5.32139202170268, model__gamma=0.00373396867904038 .....\n",
      "[CV]  model__C=5.32139202170268, model__gamma=0.00373396867904038, total=  10.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:  5.1min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score=nan,\n",
       "                   estimator=Pipeline(memory=None,\n",
       "                                      steps=[('scaler',\n",
       "                                              StandardScaler(copy=True,\n",
       "                                                             with_mean=True,\n",
       "                                                             with_std=True)),\n",
       "                                             ('model',\n",
       "                                              SVR(C=1.0, cache_size=200,\n",
       "                                                  coef0=0.0, degree=3,\n",
       "                                                  epsilon=0.1, gamma='scale',\n",
       "                                                  kernel='rbf', max_iter=-1,\n",
       "                                                  shrinking=True, tol=0.001,\n",
       "                                                  verbose=False))],\n",
       "                                      verbose=False),\n",
       "                   iid='deprecated', n_iter=10, n_jobs=None,\n",
       "                   param_distributions={'model__C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000022EDAD77B88>,\n",
       "                                        'model__gamma': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000022EDAD778C8>},\n",
       "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "                   return_train_score=False, scoring=None, verbose=2)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import reciprocal, uniform\n",
    "\n",
    "SVR_model = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('model', SVR())\n",
    "])\n",
    "\n",
    "param_dists = {\n",
    "    'model__gamma': reciprocal(0.001, 0.1),\n",
    "    'model__C': uniform(1,10)\n",
    "}\n",
    "\n",
    "rnd_search_cv = RandomizedSearchCV(SVR_model, param_dists, verbose=2, cv=3)\n",
    "rnd_search_cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('scaler',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('model',\n",
       "                 SVR(C=5.908725733130206, cache_size=200, coef0=0.0, degree=3,\n",
       "                     epsilon=0.1, gamma=0.04722302780443009, kernel='rbf',\n",
       "                     max_iter=-1, shrinking=True, tol=0.001, verbose=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_search_cv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37646952041010656\n",
      "0.6135711209062129\n"
     ]
    }
   ],
   "source": [
    "y_pred = rnd_search_cv.predict(X_test)\n",
    "mse = mean_squared_error(y_pred, y_test)\n",
    "rmse = np.sqrt(mse)\n",
    "print(mse)\n",
    "print(rmse)\n",
    "\n",
    "# Note that our original model performed better thanks to its gamma = 'auto' default option\n",
    "# To improce performance, further tuning of gamma would be required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
