notebooks/explaining/saliency-gradient-backpropagation-smooth.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"3-saliency-smoothgrad.ipynb","provenance":[{"file_id":"1SQ9Vtu5a-YWR0urOdJBadcW8C4gUDy6d","timestamp":1613482411118}],"collapsed_sections":["D72-2dlD-BzW"],"toc_visible":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.5"}},"cells":[{"cell_type":"markdown","metadata":{"id":"_AbalcTKgGgw"},"source":["# Saliency using Gradient Backpropagation\n","\n","Implement the saliency algorithm described in https://arxiv.org/pdf/1312.6034.pdf."]},{"cell_type":"code","metadata":{"id":"h4UfPUkM-EBY","executionInfo":{"status":"ok","timestamp":1614086894909,"user_tz":180,"elapsed":1029,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["REPETITIONS = 50\n","NOISE = .2"],"execution_count":1,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"D72-2dlD-BzW"},"source":["## Setup"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_psmrM7LN_WG","executionInfo":{"status":"ok","timestamp":1614086894910,"user_tz":180,"elapsed":1024,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}},"outputId":"25189241-4303-4360-8d20-0bac04f62345"},"source":["from google.colab import drive\n","drive.mount('/content/drive')\n","\n","! cp -r '/content/drive/MyDrive/Colab Notebooks/cs-no/6.grads/config' ."],"execution_count":2,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"scrolled":true,"id":"ctkvP4pGN2VA","executionInfo":{"status":"ok","timestamp":1614086897287,"user_tz":180,"elapsed":3397,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["import tensorflow as tf\n","from config import config, data, model, utils\n","\n","from config.utils import to_image, normalize\n","\n","utils.setup_clean_image_plotting()"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"1qVL7R8WDtiF","executionInfo":{"status":"ok","timestamp":1614086897291,"user_tz":180,"elapsed":3395,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["from tensorflow.python.keras.applications import mobilenet_v2\n","\n","config.model.architecture = tf.keras.applications.MobileNetV2\n","config.model.module = mobilenet_v2\n","config.model.decode = tf.keras.applications.mobilenet_v2.decode_predictions\n","config.model.preprocess = tf.keras.applications.mobilenet_v2.preprocess_input\n","config.model.decompress = lambda x: (x + 1.) * 127.5\n","\n","config.data.image_size = (224, 224)\n","config.data.input_shape = (config.data.batch_size, 224, 224, 3)"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"id":"FkQBp8jd2ZzP","executionInfo":{"status":"ok","timestamp":1614086897293,"user_tz":180,"elapsed":3392,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["images = data.load_images()\n","x = model.preprocess(images)"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"G8lb2uvNDzhR","executionInfo":{"status":"ok","timestamp":1614086899380,"user_tz":180,"elapsed":5475,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["nn = model.build()\n","\n","logits = nn(x, training=False)\n","preds = tf.argmax(logits, axis=1)\n","probs = tf.nn.softmax(logits)\n","decoded = model.decode(probs.numpy(), top=2)"],"execution_count":6,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"vp9V1IiY9xUw"},"source":["## SmoothGrad Backpropagation"]},{"cell_type":"code","metadata":{"id":"u0P1zhNR9svb","executionInfo":{"status":"ok","timestamp":1614086899381,"user_tz":180,"elapsed":5472,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["def activation_loss(y, units):\n","    return tf.gather(y, units, axis=1, batch_dims=1)\n","\n","@tf.function\n","def gradients(inputs, units):\n","    with tf.GradientTape(watch_accessed_variables=True) as tape:\n","        tape.watch(inputs)\n","        y = nn(inputs)\n","        loss = activation_loss(y, units)\n","    \n","    grads = tape.gradient(loss, inputs, unconnected_gradients=tf.UnconnectedGradients.ZERO)\n","\n","    return loss, grads"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"ezoUa8gN93WQ","executionInfo":{"status":"ok","timestamp":1614086899382,"user_tz":180,"elapsed":5469,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["@tf.function\n","def smooth_gradients(inputs, units, num_samples=50, noise=.2):\n","    x = tf.repeat(inputs, num_samples, axis=0)\n","    x += tf.random.normal(x.shape, mean=0, stddev=noise/2)\n","\n","    y = tf.repeat(units, num_samples)\n","    loss, grads = gradients(x, y)\n","    \n","    grads = tf.reshape(grads, (-1, num_samples, *grads.shape[1:]))\n","    return loss, tf.reduce_mean(grads, axis=1)"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"8gfyx09jCLcG","executionInfo":{"status":"ok","timestamp":1614087012274,"user_tz":180,"elapsed":92694,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}}},"source":["_, g = gradients(x, preds)\n","\n","sa = normalize(tf.reduce_sum(tf.abs(g), axis=-1))\n","sp = normalize(tf.reduce_sum(tf.nn.relu(g), axis=-1))\n","sn = normalize(tf.reduce_sum(tf.nn.relu(-g), axis=-1))\n","\n","sg = tf.concat([smooth_gradients(x[ix:ix+1], preds[ix:ix+1], REPETITIONS, NOISE)[1]\n","                for ix in range(len(x))],\n","               axis=0)\n","\n","a = normalize(tf.reduce_sum(tf.abs(g), axis=-1))\n","p = normalize(tf.reduce_sum(tf.nn.relu(g), axis=-1))\n","n = normalize(tf.reduce_sum(tf.nn.relu(-g), axis=-1))\n","\n","sa = normalize(tf.reduce_sum(tf.abs(sg), axis=-1))\n","sp = normalize(tf.reduce_sum(tf.nn.relu(sg), axis=-1))\n","sn = normalize(tf.reduce_sum(tf.nn.relu(-sg), axis=-1))"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"output_embedded_package_id":"1vcYdd58dNFlLC2H_vYVhOF4q1mx9Aw2j"},"id":"XpBgS08wK9Ym","executionInfo":{"status":"ok","timestamp":1614087396123,"user_tz":180,"elapsed":45724,"user":{"displayName":"Lucas","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64","userId":"01670254553878112810"}},"outputId":"ee670104-c8c0-4796-b1fd-6c91207744a0"},"source":["utils.plot(sum(zip(to_image(images),\n","                   a.numpy(),\n","                   sa.numpy(),\n","                   to_image(images * sa[..., tf.newaxis])), ()),\n","           titles=['original',\n","                   'absolute gradients',\n","                   'smooth abs gradients',\n","                   'original * smooth abs gradients'],\n","           rows=len(images))"],"execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{}}]}]}