Add courses 1 through 4
This commit is contained in:
tsb1995
parent
36fddde5c9
commit
7bdc2267a9
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,163 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"colab_type": "text",
|
||||
"id": "tOoyQ70H00_s"
|
||||
},
|
||||
"source": [
|
||||
"## Exercise 2\n",
|
||||
"In the course you learned how to do classificaiton using Fashion MNIST, a data set containing items of clothing. There's another, similar dataset called MNIST which has items of handwriting -- the digits 0 through 9.\n",
|
||||
"\n",
|
||||
"Write an MNIST classifier that trains to 99% accuracy or above, and does it without a fixed number of epochs -- i.e. you should stop training once you reach that level of accuracy.\n",
|
||||
"\n",
|
||||
"Some notes:\n",
|
||||
"1. It should succeed in less than 10 epochs, so it is okay to change epochs= to 10, but nothing larger\n",
|
||||
"2. When it reaches 99% or greater it should print out the string \"Reached 99% accuracy so cancelling training!\"\n",
|
||||
"3. If you add any additional variables, make sure you use the same names as the ones used in the class\n",
|
||||
"\n",
|
||||
"I've started the code for you below -- how would you finish it? "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"from os import path, getcwd, chdir\n",
|
||||
"\n",
|
||||
"# DO NOT CHANGE THE LINE BELOW. If you are developing in a local\n",
|
||||
"# environment, then grab mnist.npz from the Coursera Jupyter Notebook\n",
|
||||
"# and place it inside a local folder and edit the path to that location\n",
|
||||
"path = f\"{getcwd()}/../tmp2/mnist.npz\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"colab": {},
|
||||
"colab_type": "code",
|
||||
"id": "9rvXQGAA0ssC"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: train_mnist\n",
|
||||
"def train_mnist():\n",
|
||||
" # Please write your code only where you are indicated.\n",
|
||||
" # please do not remove # model fitting inline comments.\n",
|
||||
"\n",
|
||||
" # Setup callback such that it stops training after hitting 99% accuracy\n",
|
||||
" class myCallback(tf.keras.callbacks.Callback):\n",
|
||||
" def on_epoch_end(self, epoch, logs={}):\n",
|
||||
" if logs.get('acc')>0.99:\n",
|
||||
" print(\"Reached 99% accuracy so cancelling training!\")\n",
|
||||
" self.model.stop_training=True\n",
|
||||
"\n",
|
||||
" # Call your callbacks\n",
|
||||
" callbacks=myCallback()\n",
|
||||
"\n",
|
||||
" mnist = tf.keras.datasets.mnist\n",
|
||||
"\n",
|
||||
" (x_train, y_train),(x_test, y_test) = mnist.load_data(path=path)\n",
|
||||
" \n",
|
||||
" # Normalize our features\n",
|
||||
" x_train = x_train / 255.0\n",
|
||||
" x_test = x_test / 255.0\n",
|
||||
"\n",
|
||||
" model = tf.keras.models.Sequential([\n",
|
||||
" # Flatten our input to be a vector as opposed to a 28 x 28 matrix\n",
|
||||
" tf.keras.layers.Flatten(),\n",
|
||||
" \n",
|
||||
" # Setup first (and only) hidden layer with relu activation so that only positive values are accepted\n",
|
||||
" tf.keras.layers.Dense(1024, activation=tf.nn.relu),\n",
|
||||
" \n",
|
||||
" # Setup final output layer with size corrosponding to our 10 classes and softmax activation to\n",
|
||||
" # expedite our code for deciding which class a sample should be attributed to\n",
|
||||
" tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n",
|
||||
" ])\n",
|
||||
"\n",
|
||||
" model.compile(optimizer='adam',\n",
|
||||
" loss='sparse_categorical_crossentropy',\n",
|
||||
" metrics=['accuracy'])\n",
|
||||
" \n",
|
||||
" # model fitting\n",
|
||||
" history = model.fit( \n",
|
||||
" # Fit model to the training set (not our test set) and run for a maximum of 10 epochs\n",
|
||||
" # We include callbacks to stop the training when we hit our accuracy threshold\n",
|
||||
" x_train, y_train, epochs=6, callbacks=[callbacks] \n",
|
||||
" )\n",
|
||||
" # model fitting\n",
|
||||
" return history.epoch, history.history['acc'][-1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {},
|
||||
"colab_type": "code",
|
||||
"id": "9rvXQGAA0ssC"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING: Logging before flag parsing goes to stderr.\n",
|
||||
"W0930 19:29:01.265596 139898245306176 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
|
||||
"Instructions for updating:\n",
|
||||
"Call initializer instance with the dtype argument instead of passing it to the constructor\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1/6\n",
|
||||
"34016/60000 [================>.............] - ETA: 7s - loss: 0.2347 - acc: 0.9305"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train_mnist()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"coursera": {
|
||||
"course_slug": "introduction-tensorflow",
|
||||
"graded_item_id": "d6dew",
|
||||
"launcher_item_id": "FExZ4"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"colab_type": "text",
|
||||
"id": "iQjHqsmTAVLU"
|
||||
},
|
||||
"source": [
|
||||
"## Exercise 3\n",
|
||||
"In the videos you looked at how you would improve Fashion MNIST using Convolutions. For your exercise see if you can improve MNIST to 99.8% accuracy or more using only a single convolutional layer and a single MaxPooling 2D. You should stop training once the accuracy goes above this amount. It should happen in less than 20 epochs, so it's ok to hard code the number of epochs for training, but your training must end once it hits the above metric. If it doesn't, then you'll need to redesign your layers.\n",
|
||||
"\n",
|
||||
"I've started the code for you -- you need to finish it!\n",
|
||||
"\n",
|
||||
"When 99.8% accuracy has been hit, you should print out the string \"Reached 99.8% accuracy so cancelling training!\"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"from os import path, getcwd, chdir\n",
|
||||
"\n",
|
||||
"# DO NOT CHANGE THE LINE BELOW. If you are developing in a local\n",
|
||||
"# environment, then grab mnist.npz from the Coursera Jupyter Notebook\n",
|
||||
"# and place it inside a local folder and edit the path to that location\n",
|
||||
"path = f\"{getcwd()}/../tmp2/mnist.npz\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"config = tf.ConfigProto()\n",
|
||||
"config.gpu_options.allow_growth = True\n",
|
||||
"sess = tf.Session(config=config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: train_mnist_conv\n",
|
||||
"def train_mnist_conv():\n",
|
||||
" # Please write your code only where you are indicated.\n",
|
||||
" # please do not remove model fitting inline comments.\n",
|
||||
"\n",
|
||||
" # Setup callback to stop when we hit 99.8% accuracy\n",
|
||||
" class myCallback(tf.keras.callbacks.Callback):\n",
|
||||
" def on_epoch_end(self, epoch, logs={}):\n",
|
||||
" if (logs.get('acc')>0.998):\n",
|
||||
" print(\"Reached 99.8% accuracy so cancelling training!\")\n",
|
||||
" self.model.stop_training=True\n",
|
||||
" \n",
|
||||
" callbacks=myCallback()\n",
|
||||
"\n",
|
||||
" mnist = tf.keras.datasets.mnist\n",
|
||||
" (training_images, training_labels), (test_images, test_labels) = mnist.load_data(path=path)\n",
|
||||
" # Format inputs to be 4D vectors and normalizing \n",
|
||||
" training_images = training_images.reshape(60000, 28, 28, 1)\n",
|
||||
" training_images = training_images / 255.0\n",
|
||||
" test_images = test_images.reshape(10000, 28, 28, 1)\n",
|
||||
" test_images = test_images / 255.0\n",
|
||||
"\n",
|
||||
" model = tf.keras.models.Sequential([\n",
|
||||
" # Add a couple layers of convolution and a single layer of pooling\n",
|
||||
" # Only one layer of pooling due to such a high goal for accuracy\n",
|
||||
" tf.keras.layers.Conv2D(64, (3,3), activation='relu', input_shape=(28,28,1)),\n",
|
||||
" tf.keras.layers.MaxPooling2D(3,3),\n",
|
||||
" tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n",
|
||||
" \n",
|
||||
" # Flatten out input data\n",
|
||||
" tf.keras.layers.Flatten(),\n",
|
||||
" \n",
|
||||
" # Add two hidden layers of size 128 to provide complexity needed to reach goal accuracy\n",
|
||||
" tf.keras.layers.Dense(128, activation='relu'),\n",
|
||||
" tf.keras.layers.Dense(128, activation='relu'),\n",
|
||||
" tf.keras.layers.Dense(10, activation='softmax')\n",
|
||||
" ])\n",
|
||||
"\n",
|
||||
" model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
|
||||
" # model fitting\n",
|
||||
" history = model.fit(\n",
|
||||
" training_images, training_labels, epochs=20, callbacks=[callbacks]\n",
|
||||
" )\n",
|
||||
" # model fitting\n",
|
||||
" return history.epoch, history.history['acc'][-1]\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1/20\n",
|
||||
"11104/60000 [====>.........................] - ETA: 17s - loss: 0.3921 - acc: 0.8788"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "KeyboardInterrupt",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-8-1ff3c304aec3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_mnist_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
|
||||
"\u001b[0;32m<ipython-input-7-0b6012c60224>\u001b[0m in \u001b[0;36mtrain_mnist_conv\u001b[0;34m()\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;31m# model fitting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m history = model.fit(\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0mtraining_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 43\u001b[0m )\n\u001b[1;32m 44\u001b[0m \u001b[0;31m# model fitting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m 778\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 779\u001b[0m \u001b[0mvalidation_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m 781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 782\u001b[0m def evaluate(self,\n",
|
||||
"\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 362\u001b[0m \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 364\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 365\u001b[0m \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 3290\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3291\u001b[0m fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3292\u001b[0;31m run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m 3293\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3294\u001b[0m output_structure = nest.pack_sequence_as(\n",
|
||||
"\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1456\u001b[0m ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m 1457\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1458\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 1459\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1460\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"_, _ = train_mnist_conv()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"coursera": {
|
||||
"course_slug": "introduction-tensorflow",
|
||||
"graded_item_id": "ml06H",
|
||||
"launcher_item_id": "hQF8A"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"colab_type": "text",
|
||||
"id": "UncprnB0ymAE"
|
||||
},
|
||||
"source": [
|
||||
"Below is code with a link to a happy or sad dataset which contains 80 images, 40 happy and 40 sad. \n",
|
||||
"Create a convolutional neural network that trains to 100% accuracy on these images, which cancels training upon hitting training accuracy of >.999\n",
|
||||
"\n",
|
||||
"Hint -- it will work best with 3 convolutional layers."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tensorflow as tf\n",
|
||||
"import os\n",
|
||||
"import zipfile\n",
|
||||
"from os import path, getcwd, chdir\n",
|
||||
"\n",
|
||||
"# DO NOT CHANGE THE LINE BELOW. If you are developing in a local\n",
|
||||
"# environment, then grab happy-or-sad.zip from the Coursera Jupyter Notebook\n",
|
||||
"# and place it inside a local folder and edit the path to that location\n",
|
||||
"path = f\"{getcwd()}/../tmp2/happy-or-sad.zip\"\n",
|
||||
"\n",
|
||||
"zip_ref = zipfile.ZipFile(path, 'r')\n",
|
||||
"zip_ref.extractall(\"/tmp/h-or-s\")\n",
|
||||
"zip_ref.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: train_happy_sad_model\n",
|
||||
"def train_happy_sad_model():\n",
|
||||
" # Please write your code only where you are indicated.\n",
|
||||
" # please do not remove # model fitting inline comments.\n",
|
||||
"\n",
|
||||
" DESIRED_ACCURACY = 0.999\n",
|
||||
"\n",
|
||||
" class myCallback(tf.keras.callbacks.Callback):\n",
|
||||
" def on_epoch_end(self, epoch, logs={}):\n",
|
||||
" if logs.get('acc')>DESIRED_ACCURACY:\n",
|
||||
" print(\"Reached 100% accuracy so cancelling training!\")\n",
|
||||
" self.model.stop_training=True\n",
|
||||
"\n",
|
||||
" callbacks = myCallback()\n",
|
||||
" \n",
|
||||
" # This Code Block should Define and Compile the Model. Please assume the images are 150 X 150 in your implementation.\n",
|
||||
" model = tf.keras.models.Sequential([\n",
|
||||
" # First conv layer taking in 150 x 150 image input\n",
|
||||
" tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape=(150,150,3)),\n",
|
||||
" tf.keras.layers.MaxPooling2D(2,2),\n",
|
||||
" # Second conv layer\n",
|
||||
" tf.keras.layers.Conv2D(32, (3,3), activation='relu'),\n",
|
||||
" tf.keras.layers.MaxPooling2D(2,2),\n",
|
||||
" # Third conv layer\n",
|
||||
" tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n",
|
||||
" tf.keras.layers.MaxPooling2D(2,2),\n",
|
||||
" # Flatten results\n",
|
||||
" tf.keras.layers.Flatten(),\n",
|
||||
" # First hidden layer\n",
|
||||
" tf.keras.layers.Dense(512, activation='relu'),\n",
|
||||
" # Output layer\n",
|
||||
" tf.keras.layers.Dense(1, activation='sigmoid')\n",
|
||||
" ])\n",
|
||||
"\n",
|
||||
" from tensorflow.keras.optimizers import RMSprop\n",
|
||||
"\n",
|
||||
" model.compile(loss='binary_crossentropy', # Binary Crossentropy loss function to deal with two categories\n",
|
||||
" optimizer=RMSprop(lr=0.001),\n",
|
||||
" metrics=['acc'])\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # This code block should create an instance of an ImageDataGenerator called train_datagen \n",
|
||||
" # And a train_generator by calling train_datagen.flow_from_directory\n",
|
||||
"\n",
|
||||
" from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
|
||||
" \n",
|
||||
" # Rescale by a factor or 255\n",
|
||||
" train_datagen = ImageDataGenerator(rescale=1/255)\n",
|
||||
"\n",
|
||||
" # Please use a target_size of 150 X 150.\n",
|
||||
" train_generator = train_datagen.flow_from_directory(\n",
|
||||
" '/tmp/h-or-s', # Location of our happy/sad images\n",
|
||||
" target_size=(150,150), # resize images to 150 x 150\n",
|
||||
" batch_size=128, \n",
|
||||
" class_mode='binary')\n",
|
||||
" # Expected output: 'Found 80 images belonging to 2 classes'\n",
|
||||
"\n",
|
||||
" # This code block should call model.fit_generator and train for\n",
|
||||
" # a number of epochs.\n",
|
||||
" # model fitting\n",
|
||||
" history = model.fit_generator(\n",
|
||||
" train_generator,\n",
|
||||
" steps_per_epoch=8,\n",
|
||||
" epochs=15,\n",
|
||||
" verbose=1,\n",
|
||||
" callbacks=[callbacks])\n",
|
||||
" # model fitting\n",
|
||||
" return history.history['acc'][-1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found 80 images belonging to 2 classes.\n",
|
||||
"Epoch 1/15\n",
|
||||
"8/8 [==============================] - 5s 565ms/step - loss: 2.2942 - acc: 0.5000\n",
|
||||
"Epoch 2/15\n",
|
||||
"8/8 [==============================] - 3s 322ms/step - loss: 0.4900 - acc: 0.7937\n",
|
||||
"Epoch 3/15\n",
|
||||
"8/8 [==============================] - 3s 315ms/step - loss: 0.2577 - acc: 0.9047\n",
|
||||
"Epoch 4/15\n",
|
||||
"8/8 [==============================] - 2s 312ms/step - loss: 0.0923 - acc: 0.9734\n",
|
||||
"Epoch 5/15\n",
|
||||
"8/8 [==============================] - 3s 322ms/step - loss: 0.0560 - acc: 0.9766\n",
|
||||
"Epoch 6/15\n",
|
||||
"7/8 [=========================>....] - ETA: 0s - loss: 0.0178 - acc: 1.0000Reached 100% accuracy so cancelling training!\n",
|
||||
"8/8 [==============================] - 2s 312ms/step - loss: 0.0169 - acc: 1.0000\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"1.0"
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# The Expected output: \"Reached 99.9% accuracy so cancelling training!\"\"\n",
|
||||
"train_happy_sad_model()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"coursera": {
|
||||
"course_slug": "introduction-tensorflow",
|
||||
"graded_item_id": "1kAlw",
|
||||
"launcher_item_id": "PNLYD"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,668 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Gradient Checking\n",
|
||||
"\n",
|
||||
"Welcome to the final assignment for this week! In this assignment you will learn to implement and use gradient checking. \n",
|
||||
"\n",
|
||||
"You are part of a team working to make mobile payments available globally, and are asked to build a deep learning model to detect fraud--whenever someone makes a payment, you want to see if the payment might be fraudulent, such as if the user's account has been taken over by a hacker. \n",
|
||||
"\n",
|
||||
"But backpropagation is quite challenging to implement, and sometimes has bugs. Because this is a mission-critical application, your company's CEO wants to be really certain that your implementation of backpropagation is correct. Your CEO says, \"Give me a proof that your backpropagation is actually working!\" To give this reassurance, you are going to use \"gradient checking\".\n",
|
||||
"\n",
|
||||
"Let's do it!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Packages\n",
|
||||
"import numpy as np\n",
|
||||
"from testCases import *\n",
|
||||
"from gc_utils import sigmoid, relu, dictionary_to_vector, vector_to_dictionary, gradients_to_vector"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1) How does gradient checking work?\n",
|
||||
"\n",
|
||||
"Backpropagation computes the gradients $\\frac{\\partial J}{\\partial \\theta}$, where $\\theta$ denotes the parameters of the model. $J$ is computed using forward propagation and your loss function.\n",
|
||||
"\n",
|
||||
"Because forward propagation is relatively easy to implement, you're confident you got that right, and so you're almost 100% sure that you're computing the cost $J$ correctly. Thus, you can use your code for computing $J$ to verify the code for computing $\\frac{\\partial J}{\\partial \\theta}$. \n",
|
||||
"\n",
|
||||
"Let's look back at the definition of a derivative (or gradient):\n",
|
||||
"$$ \\frac{\\partial J}{\\partial \\theta} = \\lim_{\\varepsilon \\to 0} \\frac{J(\\theta + \\varepsilon) - J(\\theta - \\varepsilon)}{2 \\varepsilon} \\tag{1}$$\n",
|
||||
"\n",
|
||||
"If you're not familiar with the \"$\\displaystyle \\lim_{\\varepsilon \\to 0}$\" notation, it's just a way of saying \"when $\\varepsilon$ is really really small.\"\n",
|
||||
"\n",
|
||||
"We know the following:\n",
|
||||
"\n",
|
||||
"- $\\frac{\\partial J}{\\partial \\theta}$ is what you want to make sure you're computing correctly. \n",
|
||||
"- You can compute $J(\\theta + \\varepsilon)$ and $J(\\theta - \\varepsilon)$ (in the case that $\\theta$ is a real number), since you're confident your implementation for $J$ is correct. \n",
|
||||
"\n",
|
||||
"Lets use equation (1) and a small value for $\\varepsilon$ to convince your CEO that your code for computing $\\frac{\\partial J}{\\partial \\theta}$ is correct!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2) 1-dimensional gradient checking\n",
|
||||
"\n",
|
||||
"Consider a 1D linear function $J(\\theta) = \\theta x$. The model contains only a single real-valued parameter $\\theta$, and takes $x$ as input.\n",
|
||||
"\n",
|
||||
"You will implement code to compute $J(.)$ and its derivative $\\frac{\\partial J}{\\partial \\theta}$. You will then use gradient checking to make sure your derivative computation for $J$ is correct. \n",
|
||||
"\n",
|
||||
"<img src=\"images/1Dgrad_kiank.png\" style=\"width:600px;height:250px;\">\n",
|
||||
"<caption><center> <u> **Figure 1** </u>: **1D linear model**<br> </center></caption>\n",
|
||||
"\n",
|
||||
"The diagram above shows the key computation steps: First start with $x$, then evaluate the function $J(x)$ (\"forward propagation\"). Then compute the derivative $\\frac{\\partial J}{\\partial \\theta}$ (\"backward propagation\"). \n",
|
||||
"\n",
|
||||
"**Exercise**: implement \"forward propagation\" and \"backward propagation\" for this simple function. I.e., compute both $J(.)$ (\"forward propagation\") and its derivative with respect to $\\theta$ (\"backward propagation\"), in two separate functions. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: forward_propagation\n",
|
||||
"\n",
|
||||
"def forward_propagation(x, theta):\n",
|
||||
" \"\"\"\n",
|
||||
" Implement the linear forward propagation (compute J) presented in Figure 1 (J(theta) = theta * x)\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" x -- a real-valued input\n",
|
||||
" theta -- our parameter, a real number as well\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" J -- the value of function J, computed using the formula J(theta) = theta * x\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" J = theta * x\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" return J"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"J = 8\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x, theta = 2, 4\n",
|
||||
"J = forward_propagation(x, theta)\n",
|
||||
"print (\"J = \" + str(J))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Expected Output**:\n",
|
||||
"\n",
|
||||
"<table style=>\n",
|
||||
" <tr>\n",
|
||||
" <td> ** J ** </td>\n",
|
||||
" <td> 8</td>\n",
|
||||
" </tr>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Exercise**: Now, implement the backward propagation step (derivative computation) of Figure 1. That is, compute the derivative of $J(\\theta) = \\theta x$ with respect to $\\theta$. To save you from doing the calculus, you should get $dtheta = \\frac { \\partial J }{ \\partial \\theta} = x$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: backward_propagation\n",
|
||||
"\n",
|
||||
"def backward_propagation(x, theta):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes the derivative of J with respect to theta (see Figure 1).\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" x -- a real-valued input\n",
|
||||
" theta -- our parameter, a real number as well\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" dtheta -- the gradient of the cost with respect to theta\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" dtheta = x\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" return dtheta"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"dtheta = 2\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x, theta = 2, 4\n",
|
||||
"dtheta = backward_propagation(x, theta)\n",
|
||||
"print (\"dtheta = \" + str(dtheta))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Expected Output**:\n",
|
||||
"\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td> ** dtheta ** </td>\n",
|
||||
" <td> 2 </td>\n",
|
||||
" </tr>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Exercise**: To show that the `backward_propagation()` function is correctly computing the gradient $\\frac{\\partial J}{\\partial \\theta}$, let's implement gradient checking.\n",
|
||||
"\n",
|
||||
"**Instructions**:\n",
|
||||
"- First compute \"gradapprox\" using the formula above (1) and a small value of $\\varepsilon$. Here are the Steps to follow:\n",
|
||||
" 1. $\\theta^{+} = \\theta + \\varepsilon$\n",
|
||||
" 2. $\\theta^{-} = \\theta - \\varepsilon$\n",
|
||||
" 3. $J^{+} = J(\\theta^{+})$\n",
|
||||
" 4. $J^{-} = J(\\theta^{-})$\n",
|
||||
" 5. $gradapprox = \\frac{J^{+} - J^{-}}{2 \\varepsilon}$\n",
|
||||
"- Then compute the gradient using backward propagation, and store the result in a variable \"grad\"\n",
|
||||
"- Finally, compute the relative difference between \"gradapprox\" and the \"grad\" using the following formula:\n",
|
||||
"$$ difference = \\frac {\\mid\\mid grad - gradapprox \\mid\\mid_2}{\\mid\\mid grad \\mid\\mid_2 + \\mid\\mid gradapprox \\mid\\mid_2} \\tag{2}$$\n",
|
||||
"You will need 3 Steps to compute this formula:\n",
|
||||
" - 1'. compute the numerator using np.linalg.norm(...)\n",
|
||||
" - 2'. compute the denominator. You will need to call np.linalg.norm(...) twice.\n",
|
||||
" - 3'. divide them.\n",
|
||||
"- If this difference is small (say less than $10^{-7}$), you can be quite confident that you have computed your gradient correctly. Otherwise, there may be a mistake in the gradient computation. \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: gradient_check\n",
|
||||
"\n",
|
||||
"def gradient_check(x, theta, epsilon = 1e-7):\n",
|
||||
" \"\"\"\n",
|
||||
" Implement the backward propagation presented in Figure 1.\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" x -- a real-valued input\n",
|
||||
" theta -- our parameter, a real number as well\n",
|
||||
" epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" # Compute gradapprox using left side of formula (1). epsilon is small enough, you don't need to worry about the limit.\n",
|
||||
" ### START CODE HERE ### (approx. 5 lines)\n",
|
||||
" thetaplus = theta + epsilon # Step 1\n",
|
||||
" thetaminus = theta - epsilon # Step 2\n",
|
||||
" J_plus = forward_propagation(x, thetaplus) # Step 3\n",
|
||||
" J_minus = forward_propagation(x, thetaminus) # Step 4\n",
|
||||
" gradapprox = (J_plus - J_minus)/(2*epsilon) # Step 5\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" # Check if gradapprox is close enough to the output of backward_propagation()\n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" grad = backward_propagation(x,theta)\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" numerator = np.linalg.norm(grad - gradapprox) # Step 1'\n",
|
||||
" denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox) # Step 2'\n",
|
||||
" difference = numerator / denominator # Step 3'\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" if difference < 1e-7:\n",
|
||||
" print (\"The gradient is correct!\")\n",
|
||||
" else:\n",
|
||||
" print (\"The gradient is wrong!\")\n",
|
||||
" \n",
|
||||
" return difference"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The gradient is correct!\n",
|
||||
"difference = 2.91933588329e-10\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x, theta = 2, 4\n",
|
||||
"difference = gradient_check(x, theta)\n",
|
||||
"print(\"difference = \" + str(difference))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Expected Output**:\n",
|
||||
"The gradient is correct!\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td> ** difference ** </td>\n",
|
||||
" <td> 2.9193358103083e-10 </td>\n",
|
||||
" </tr>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Congrats, the difference is smaller than the $10^{-7}$ threshold. So you can have high confidence that you've correctly computed the gradient in `backward_propagation()`. \n",
|
||||
"\n",
|
||||
"Now, in the more general case, your cost function $J$ has more than a single 1D input. When you are training a neural network, $\\theta$ actually consists of multiple matrices $W^{[l]}$ and biases $b^{[l]}$! It is important to know how to do a gradient check with higher-dimensional inputs. Let's do it!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3) N-dimensional gradient checking"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"source": [
|
||||
"The following figure describes the forward and backward propagation of your fraud detection model.\n",
|
||||
"\n",
|
||||
"<img src=\"images/NDgrad_kiank.png\" style=\"width:600px;height:400px;\">\n",
|
||||
"<caption><center> <u> **Figure 2** </u>: **deep neural network**<br>*LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID*</center></caption>\n",
|
||||
"\n",
|
||||
"Let's look at your implementations for forward propagation and backward propagation. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def forward_propagation_n(X, Y, parameters):\n",
|
||||
" \"\"\"\n",
|
||||
" Implements the forward propagation (and computes the cost) presented in Figure 3.\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" X -- training set for m examples\n",
|
||||
" Y -- labels for m examples \n",
|
||||
" parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
|
||||
" W1 -- weight matrix of shape (5, 4)\n",
|
||||
" b1 -- bias vector of shape (5, 1)\n",
|
||||
" W2 -- weight matrix of shape (3, 5)\n",
|
||||
" b2 -- bias vector of shape (3, 1)\n",
|
||||
" W3 -- weight matrix of shape (1, 3)\n",
|
||||
" b3 -- bias vector of shape (1, 1)\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" cost -- the cost function (logistic cost for one example)\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" # retrieve parameters\n",
|
||||
" m = X.shape[1]\n",
|
||||
" W1 = parameters[\"W1\"]\n",
|
||||
" b1 = parameters[\"b1\"]\n",
|
||||
" W2 = parameters[\"W2\"]\n",
|
||||
" b2 = parameters[\"b2\"]\n",
|
||||
" W3 = parameters[\"W3\"]\n",
|
||||
" b3 = parameters[\"b3\"]\n",
|
||||
"\n",
|
||||
" # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID\n",
|
||||
" Z1 = np.dot(W1, X) + b1\n",
|
||||
" A1 = relu(Z1)\n",
|
||||
" Z2 = np.dot(W2, A1) + b2\n",
|
||||
" A2 = relu(Z2)\n",
|
||||
" Z3 = np.dot(W3, A2) + b3\n",
|
||||
" A3 = sigmoid(Z3)\n",
|
||||
"\n",
|
||||
" # Cost\n",
|
||||
" logprobs = np.multiply(-np.log(A3),Y) + np.multiply(-np.log(1 - A3), 1 - Y)\n",
|
||||
" cost = 1./m * np.sum(logprobs)\n",
|
||||
" \n",
|
||||
" cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)\n",
|
||||
" \n",
|
||||
" return cost, cache"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, run backward propagation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def backward_propagation_n(X, Y, cache):\n",
|
||||
" \"\"\"\n",
|
||||
" Implement the backward propagation presented in figure 2.\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" X -- input datapoint, of shape (input size, 1)\n",
|
||||
" Y -- true \"label\"\n",
|
||||
" cache -- cache output from forward_propagation_n()\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" gradients -- A dictionary with the gradients of the cost with respect to each parameter, activation and pre-activation variables.\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" m = X.shape[1]\n",
|
||||
" (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache\n",
|
||||
" \n",
|
||||
" dZ3 = A3 - Y\n",
|
||||
" dW3 = 1./m * np.dot(dZ3, A2.T)\n",
|
||||
" db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)\n",
|
||||
" \n",
|
||||
" dA2 = np.dot(W3.T, dZ3)\n",
|
||||
" dZ2 = np.multiply(dA2, np.int64(A2 > 0))\n",
|
||||
" dW2 = 1./m * np.dot(dZ2, A1.T)\n",
|
||||
" db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)\n",
|
||||
" \n",
|
||||
" dA1 = np.dot(W2.T, dZ2)\n",
|
||||
" dZ1 = np.multiply(dA1, np.int64(A1 > 0))\n",
|
||||
" dW1 = 1./m * np.dot(dZ1, X.T)\n",
|
||||
" db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)\n",
|
||||
" \n",
|
||||
" gradients = {\"dZ3\": dZ3, \"dW3\": dW3, \"db3\": db3,\n",
|
||||
" \"dA2\": dA2, \"dZ2\": dZ2, \"dW2\": dW2, \"db2\": db2,\n",
|
||||
" \"dA1\": dA1, \"dZ1\": dZ1, \"dW1\": dW1, \"db1\": db1}\n",
|
||||
" \n",
|
||||
" return gradients"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"source": [
|
||||
"You obtained some results on the fraud detection test set but you are not 100% sure of your model. Nobody's perfect! Let's implement gradient checking to verify if your gradients are correct."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**How does gradient checking work?**.\n",
|
||||
"\n",
|
||||
"As in 1) and 2), you want to compare \"gradapprox\" to the gradient computed by backpropagation. The formula is still:\n",
|
||||
"\n",
|
||||
"$$ \\frac{\\partial J}{\\partial \\theta} = \\lim_{\\varepsilon \\to 0} \\frac{J(\\theta + \\varepsilon) - J(\\theta - \\varepsilon)}{2 \\varepsilon} \\tag{1}$$\n",
|
||||
"\n",
|
||||
"However, $\\theta$ is not a scalar anymore. It is a dictionary called \"parameters\". We implemented a function \"`dictionary_to_vector()`\" for you. It converts the \"parameters\" dictionary into a vector called \"values\", obtained by reshaping all parameters (W1, b1, W2, b2, W3, b3) into vectors and concatenating them.\n",
|
||||
"\n",
|
||||
"The inverse function is \"`vector_to_dictionary`\" which outputs back the \"parameters\" dictionary.\n",
|
||||
"\n",
|
||||
"<img src=\"images/dictionary_to_vector.png\" style=\"width:600px;height:400px;\">\n",
|
||||
"<caption><center> <u> **Figure 2** </u>: **dictionary_to_vector() and vector_to_dictionary()**<br> You will need these functions in gradient_check_n()</center></caption>\n",
|
||||
"\n",
|
||||
"We have also converted the \"gradients\" dictionary into a vector \"grad\" using gradients_to_vector(). You don't need to worry about that.\n",
|
||||
"\n",
|
||||
"**Exercise**: Implement gradient_check_n().\n",
|
||||
"\n",
|
||||
"**Instructions**: Here is pseudo-code that will help you implement the gradient check.\n",
|
||||
"\n",
|
||||
"For each i in num_parameters:\n",
|
||||
"- To compute `J_plus[i]`:\n",
|
||||
" 1. Set $\\theta^{+}$ to `np.copy(parameters_values)`\n",
|
||||
" 2. Set $\\theta^{+}_i$ to $\\theta^{+}_i + \\varepsilon$\n",
|
||||
" 3. Calculate $J^{+}_i$ using to `forward_propagation_n(x, y, vector_to_dictionary(`$\\theta^{+}$ `))`. \n",
|
||||
"- To compute `J_minus[i]`: do the same thing with $\\theta^{-}$\n",
|
||||
"- Compute $gradapprox[i] = \\frac{J^{+}_i - J^{-}_i}{2 \\varepsilon}$\n",
|
||||
"\n",
|
||||
"Thus, you get a vector gradapprox, where gradapprox[i] is an approximation of the gradient with respect to `parameter_values[i]`. You can now compare this gradapprox vector to the gradients vector from backpropagation. Just like for the 1D case (Steps 1', 2', 3'), compute: \n",
|
||||
"$$ difference = \\frac {\\| grad - gradapprox \\|_2}{\\| grad \\|_2 + \\| gradapprox \\|_2 } \\tag{3}$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# GRADED FUNCTION: gradient_check_n\n",
|
||||
"\n",
|
||||
"def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):\n",
|
||||
" \"\"\"\n",
|
||||
" Checks if backward_propagation_n computes correctly the gradient of the cost output by forward_propagation_n\n",
|
||||
" \n",
|
||||
" Arguments:\n",
|
||||
" parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
|
||||
" grad -- output of backward_propagation_n, contains gradients of the cost with respect to the parameters. \n",
|
||||
" x -- input datapoint, of shape (input size, 1)\n",
|
||||
" y -- true \"label\"\n",
|
||||
" epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" # Set-up variables\n",
|
||||
" parameters_values, _ = dictionary_to_vector(parameters)\n",
|
||||
" grad = gradients_to_vector(gradients)\n",
|
||||
" num_parameters = parameters_values.shape[0]\n",
|
||||
" J_plus = np.zeros((num_parameters, 1))\n",
|
||||
" J_minus = np.zeros((num_parameters, 1))\n",
|
||||
" gradapprox = np.zeros((num_parameters, 1))\n",
|
||||
" \n",
|
||||
" # Compute gradapprox\n",
|
||||
" for i in range(num_parameters):\n",
|
||||
" \n",
|
||||
" # Compute J_plus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_plus[i]\".\n",
|
||||
" # \"_\" is used because the function you have to outputs two parameters but we only care about the first one\n",
|
||||
" ### START CODE HERE ### (approx. 3 lines)\n",
|
||||
" thetaplus = np.copy(parameters_values) # Step 1\n",
|
||||
" thetaplus[i][0] += epsilon # Step 2\n",
|
||||
" J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaplus)) # Step 3\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" # Compute J_minus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_minus[i]\".\n",
|
||||
" ### START CODE HERE ### (approx. 3 lines)\n",
|
||||
" thetaminus = np.copy(parameters_values) # Step 1\n",
|
||||
" thetaminus[i][0] -= epsilon # Step 2 \n",
|
||||
" J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaminus)) # Step 3\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" # Compute gradapprox[i]\n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" gradapprox[i] = (J_plus[i] - J_minus[i]) / (2*epsilon)\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
" \n",
|
||||
" # Compare gradapprox to backward propagation gradients by computing difference.\n",
|
||||
" ### START CODE HERE ### (approx. 1 line)\n",
|
||||
" numerator = np.linalg.norm(grad - gradapprox) # Step 1'\n",
|
||||
" denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox) # Step 2'\n",
|
||||
" difference = numerator / denominator # Step 3'\n",
|
||||
" ### END CODE HERE ###\n",
|
||||
"\n",
|
||||
" if difference > 2e-7:\n",
|
||||
" print (\"\\033[93m\" + \"There is a mistake in the backward propagation! difference = \" + str(difference) + \"\\033[0m\")\n",
|
||||
" else:\n",
|
||||
" print (\"\\033[92m\" + \"Your backward propagation works perfectly fine! difference = \" + str(difference) + \"\\033[0m\")\n",
|
||||
" \n",
|
||||
" return difference"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"metadata": {
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[92mYour backward propagation works perfectly fine! difference = 1.18855520355e-07\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"X, Y, parameters = gradient_check_n_test_case()\n",
|
||||
"\n",
|
||||
"cost, cache = forward_propagation_n(X, Y, parameters)\n",
|
||||
"gradients = backward_propagation_n(X, Y, cache)\n",
|
||||
"difference = gradient_check_n(parameters, gradients, X, Y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**Expected output**:\n",
|
||||
"\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td> ** There is a mistake in the backward propagation!** </td>\n",
|
||||
" <td> difference = 0.285093156781 </td>\n",
|
||||
" </tr>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"It seems that there were errors in the `backward_propagation_n` code we gave you! Good that you've implemented the gradient check. Go back to `backward_propagation` and try to find/correct the errors *(Hint: check dW2 and db1)*. Rerun the gradient check when you think you've fixed it. Remember you'll need to re-execute the cell defining `backward_propagation_n()` if you modify the code. \n",
|
||||
"\n",
|
||||
"Can you get gradient check to declare your derivative computation correct? Even though this part of the assignment isn't graded, we strongly urge you to try to find the bug and re-run gradient check until you're convinced backprop is now correctly implemented. \n",
|
||||
"\n",
|
||||
"**Note** \n",
|
||||
"- Gradient Checking is slow! Approximating the gradient with $\\frac{\\partial J}{\\partial \\theta} \\approx \\frac{J(\\theta + \\varepsilon) - J(\\theta - \\varepsilon)}{2 \\varepsilon}$ is computationally costly. For this reason, we don't run gradient checking at every iteration during training. Just a few times to check if the gradient is correct. \n",
|
||||
"- Gradient Checking, at least as we've presented it, doesn't work with dropout. You would usually run the gradient check algorithm without dropout to make sure your backprop is correct, then add dropout. \n",
|
||||
"\n",
|
||||
"Congrats, you can be confident that your deep learning model for fraud detection is working correctly! You can even use this to convince your CEO. :) \n",
|
||||
"\n",
|
||||
"<font color='blue'>\n",
|
||||
"**What you should remember from this notebook**:\n",
|
||||
"- Gradient checking verifies closeness between the gradients from backpropagation and the numerical approximation of the gradient (computed using forward propagation).\n",
|
||||
"- Gradient checking is slow, so we don't run it in every iteration of training. You would usually run it only to make sure your code is correct, then turn it off and use backprop for the actual learning process. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"coursera": {
|
||||
"course_slug": "deep-neural-network",
|
||||
"graded_item_id": "n6NBD",
|
||||
"launcher_item_id": "yfOsE"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue