alzeihmer-vit.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\nimport copy\nimport warnings\nwarnings.filterwarnings('ignore')\nimport tensorflow as tf\nimport cv2\nimport keras\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing.image import load_img, img_to_array\nimport matplotlib\nimport matplotlib.pylab as plt\nimport numpy as np\nimport seaborn as sns\nimport shap\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom keras.applications.vgg16 import VGG16,preprocess_input\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-24T14:39:27.747585Z","iopub.execute_input":"2022-02-24T14:39:27.747834Z","iopub.status.idle":"2022-02-24T14:39:34.625695Z","shell.execute_reply.started":"2022-02-24T14:39:27.747761Z","shell.execute_reply":"2022-02-24T14:39:34.624924Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"W = 32 # The default size for ResNet is 224 but resize to .5 to save memory size\nH = 32 # The default size for ResNet is 224 but resize to .5 to save memory size\n\nlabel_to_class = {\n    'MildDemented': 0,\n    'ModerateDemented': 1,\n    'NonDemented': 2,\n    'VeryMildDemented':3\n    \n\n}\nclass_to_label = {v: k for k, v in label_to_class.items()}\nn_classes = len(label_to_class)\n\ndef get_images(dir_name='../input/alzheimers-dataset-4-class-of-images/Alzheimer_s Dataset', label_to_class=label_to_class):\n    \"\"\"read images / labels from directory\"\"\"\n    \n    Images = []\n    Classes = []\n    \n    for j in ['/train','/test']:\n        for label_name in os.listdir(dir_name+str(j)):\n            cls = label_to_class[label_name]\n\n            for img_name in os.listdir('/'.join([dir_name+str(j), label_name])):\n                img = load_img('/'.join([dir_name+str(j), label_name, img_name]), target_size=(W, H))\n                img = img_to_array(img)\n\n                Images.append(img)\n                Classes.append(cls)\n            \n    Images = np.array(Images, dtype=np.float32)\n    Classes = np.array(Classes, dtype=np.float32)\n    Images, Classes = shuffle(Images, Classes, random_state=0)\n    \n    return Images, Classes","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:39:34.627364Z","iopub.execute_input":"2022-02-24T14:39:34.627597Z","iopub.status.idle":"2022-02-24T14:39:34.640010Z","shell.execute_reply.started":"2022-02-24T14:39:34.627555Z","shell.execute_reply":"2022-02-24T14:39:34.639277Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"## get images / labels\n\nImages, Classes = get_images()\n\nImages.shape, Classes.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:39:34.647936Z","iopub.execute_input":"2022-02-24T14:39:34.648257Z","iopub.status.idle":"2022-02-24T14:40:02.157973Z","shell.execute_reply.started":"2022-02-24T14:39:34.648225Z","shell.execute_reply":"2022-02-24T14:40:02.157168Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"## split train / test\n\nindices_train, indices_test = train_test_split(list(range(Images.shape[0])), train_size=0.8, test_size=0.2, shuffle=True)\n\nx_train = Images[indices_train]\ny_train = Classes[indices_train]\nx_test = Images[indices_test]\ny_test = Classes[indices_test]\n\nx_train.shape, y_train.shape, x_test.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:02.160035Z","iopub.execute_input":"2022-02-24T14:40:02.160492Z","iopub.status.idle":"2022-02-24T14:40:02.198666Z","shell.execute_reply.started":"2022-02-24T14:40:02.160452Z","shell.execute_reply":"2022-02-24T14:40:02.197866Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"from keras.utils import np_utils","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:02.199909Z","iopub.execute_input":"2022-02-24T14:40:02.200237Z","iopub.status.idle":"2022-02-24T14:40:02.204596Z","shell.execute_reply.started":"2022-02-24T14:40:02.200199Z","shell.execute_reply":"2022-02-24T14:40:02.203878Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"y_train = keras.utils.np_utils.to_categorical(y_train, n_classes)\ny_test = keras.utils.np_utils.to_categorical(y_test, n_classes)\n\ny_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:02.206029Z","iopub.execute_input":"2022-02-24T14:40:02.206328Z","iopub.status.idle":"2022-02-24T14:40:02.216522Z","shell.execute_reply.started":"2022-02-24T14:40:02.206282Z","shell.execute_reply":"2022-02-24T14:40:02.215617Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"x_train.shape, y_train.shape, x_test.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:02.218169Z","iopub.execute_input":"2022-02-24T14:40:02.218549Z","iopub.status.idle":"2022-02-24T14:40:02.225902Z","shell.execute_reply.started":"2022-02-24T14:40:02.218514Z","shell.execute_reply":"2022-02-24T14:40:02.225209Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"pip install -U tensorflow-addons","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:02.227341Z","iopub.execute_input":"2022-02-24T14:40:02.227616Z","iopub.status.idle":"2022-02-24T14:40:20.709056Z","shell.execute_reply.started":"2022-02-24T14:40:02.227556Z","shell.execute_reply":"2022-02-24T14:40:20.708258Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:20.712532Z","iopub.execute_input":"2022-02-24T14:40:20.712765Z","iopub.status.idle":"2022-02-24T14:40:21.021370Z","shell.execute_reply.started":"2022-02-24T14:40:20.712737Z","shell.execute_reply":"2022-02-24T14:40:21.020625Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"num_classes = 4\ninput_shape = (32, 32, 3)","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:21.022473Z","iopub.execute_input":"2022-02-24T14:40:21.022710Z","iopub.status.idle":"2022-02-24T14:40:21.026996Z","shell.execute_reply.started":"2022-02-24T14:40:21.022678Z","shell.execute_reply":"2022-02-24T14:40:21.026362Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"x_train.shape, y_train.shape, x_test.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:21.028148Z","iopub.execute_input":"2022-02-24T14:40:21.028543Z","iopub.status.idle":"2022-02-24T14:40:21.049964Z","shell.execute_reply.started":"2022-02-24T14:40:21.028508Z","shell.execute_reply":"2022-02-24T14:40:21.049239Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.001\nweight_decay = 0.0001\nbatch_size = 256\nnum_epochs = 100\nimage_size = 72  # We'll resize input images to this size\npatch_size = 6  # Size of the patches to be extract from the input images\nnum_patches = (image_size // patch_size) ** 2\nprojection_dim = 64\nnum_heads = 4\ntransformer_units = [\n    projection_dim * 2,\n    projection_dim,\n]  # Size of the transformer layers\ntransformer_layers = 8\nmlp_head_units = [2048, 1024] ","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:21.051408Z","iopub.execute_input":"2022-02-24T14:40:21.051951Z","iopub.status.idle":"2022-02-24T14:40:21.058150Z","shell.execute_reply.started":"2022-02-24T14:40:21.051914Z","shell.execute_reply":"2022-02-24T14:40:21.057502Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        layers.Normalization(),\n        layers.Resizing(image_size, image_size),\n        layers.RandomFlip(\"horizontal\"),\n        layers.RandomRotation(factor=0.02),\n        layers.RandomZoom(\n            height_factor=0.2, width_factor=0.2\n        ),\n    ],\n    name=\"data_augmentation\",\n)\n# Compute the mean and the variance of the training data for normalization.\ndata_augmentation.layers[0].adapt(x_train)","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:21.059006Z","iopub.execute_input":"2022-02-24T14:40:21.061623Z","iopub.status.idle":"2022-02-24T14:40:24.245807Z","shell.execute_reply.started":"2022-02-24T14:40:21.061584Z","shell.execute_reply":"2022-02-24T14:40:24.245021Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"def mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = layers.Dense(units, activation=tf.nn.gelu)(x)\n        x = layers.Dropout(dropout_rate)(x)\n    return x\n","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:24.248353Z","iopub.execute_input":"2022-02-24T14:40:24.249335Z","iopub.status.idle":"2022-02-24T14:40:24.256021Z","shell.execute_reply.started":"2022-02-24T14:40:24.249294Z","shell.execute_reply":"2022-02-24T14:40:24.255332Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"class Patches(layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n\n    def call(self, images):\n        batch_size = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images=images,\n            sizes=[1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        patches = tf.reshape(patches, [batch_size, -1, patch_dims])\n        return patches","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:24.257527Z","iopub.execute_input":"2022-02-24T14:40:24.257831Z","iopub.status.idle":"2022-02-24T14:40:24.266501Z","shell.execute_reply.started":"2022-02-24T14:40:24.257762Z","shell.execute_reply":"2022-02-24T14:40:24.265578Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(4, 4))\nimage = x_train[np.random.choice(range(x_train.shape[0]))]\nplt.imshow(image.astype(\"uint8\"))\nplt.axis(\"off\")\n\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\n\nn = int(np.sqrt(patches.shape[1]))\nplt.figure(figsize=(4, 4))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 3))\n    plt.imshow(patch_img.numpy().astype(\"uint8\"))\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:24.267955Z","iopub.execute_input":"2022-02-24T14:40:24.268287Z","iopub.status.idle":"2022-02-24T14:40:29.039848Z","shell.execute_reply.started":"2022-02-24T14:40:24.268250Z","shell.execute_reply":"2022-02-24T14:40:29.039154Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"class PatchEncoder(layers.Layer):\n    def __init__(self, num_patches, projection_dim):\n        super(PatchEncoder, self).__init__()\n        self.num_patches = num_patches\n        self.projection = layers.Dense(units=projection_dim)\n        self.position_embedding = layers.Embedding(\n            input_dim=num_patches, output_dim=projection_dim\n        )\n\n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.num_patches, delta=1)\n        encoded = self.projection(patch) + self.position_embedding(positions)\n        return encoded","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:29.041130Z","iopub.execute_input":"2022-02-24T14:40:29.041713Z","iopub.status.idle":"2022-02-24T14:40:29.047846Z","shell.execute_reply.started":"2022-02-24T14:40:29.041675Z","shell.execute_reply":"2022-02-24T14:40:29.046843Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"def create_vit_classifier():\n    inputs = layers.Input(shape=input_shape)\n    # Augment data.\n    augmented = data_augmentation(inputs)\n    # Create patches.\n    patches = Patches(patch_size)(augmented)\n    # Encode patches.\n    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)\n\n    # Create multiple layers of the Transformer block.\n    for _ in range(transformer_layers):\n        # Layer normalization 1.\n        x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n        # Create a multi-head attention layer.\n        attention_output = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection_dim, dropout=0.1\n        )(x1, x1)\n        # Skip connection 1.\n        x2 = layers.Add()([attention_output, encoded_patches])\n        # Layer normalization 2.\n        x3 = layers.LayerNormalization(epsilon=1e-6)(x2)\n        # MLP.\n        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)\n        # Skip connection 2.\n        encoded_patches = layers.Add()([x3, x2])\n\n    # Create a [batch_size, projection_dim] tensor.\n    representation = layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n    representation = layers.Flatten()(representation)\n    representation = layers.Dropout(0.5)(representation)\n    # Add MLP.\n    features = mlp(representation, hidden_units=mlp_head_units, dropout_rate=0.5)\n    # Classify outputs.\n    logits = layers.Dense(num_classes)(features)\n    # Create the Keras model.\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:40:29.049221Z","iopub.execute_input":"2022-02-24T14:40:29.049542Z","iopub.status.idle":"2022-02-24T14:40:29.060608Z","shell.execute_reply.started":"2022-02-24T14:40:29.049508Z","shell.execute_reply":"2022-02-24T14:40:29.059784Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"def run_experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        learning_rate=learning_rate, weight_decay=weight_decay\n    )\n\n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.CategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.TopKCategoricalAccuracy(5, name=\"top-5-accuracy\"),\n        ],\n    )\n\n    checkpoint_filepath = \"/tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,\n    )\n\n    history = model.fit(\n        x=x_train,\n        y=y_train,\n        batch_size=batch_size,\n        epochs=num_epochs,\n        validation_split=0.1,\n        callbacks=[checkpoint_callback],\n    )\n\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, top_5_accuracy = model.evaluate(x_test, y_test)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test top 5 accuracy: {round(top_5_accuracy * 100, 2)}%\")\n\n    return history\n\n\nvit_classifier = create_vit_classifier()\nhistory = run_experiment(vit_classifier)","metadata":{"execution":{"iopub.status.busy":"2022-02-24T14:51:46.020071Z","iopub.execute_input":"2022-02-24T14:51:46.020367Z","iopub.status.idle":"2022-02-24T14:59:31.882526Z","shell.execute_reply.started":"2022-02-24T14:51:46.020331Z","shell.execute_reply":"2022-02-24T14:59:31.881426Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}