interpolation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GlobalAveragePooling2D, MaxPooling2D\n",
    "from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.convolutional import UpSampling2D, Conv2D\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "from scipy.misc import imread\n",
    "from scipy.misc import imresize\n",
    "from scipy.misc import imsave\n",
    "import glob\n",
    "import random\n",
    "\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import applications\n",
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.utils import to_categorical\n",
    "#from keras.applications.resnet50 import preprocess_input\n",
    "\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.resnet50 import preprocess_input, decode_predictions\n",
    "#from keras.applications.vgg19 import preprocess_input as preprocess_vgg\n",
    "from keras.applications.inception_v3 import preprocess_input as inception_preprocess\n",
    "\n",
    "#from keras.applications.inception_resnet_v2 import preprocess_input\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "import copy\n",
    "import cv2\n",
    "\n",
    "from keras.models import *\n",
    "from keras.layers import *\n",
    "from keras.optimizers import *\n",
    "\n",
    "import numpy as np \n",
    "import os\n",
    "import numpy as np\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras import backend as keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../data/'\n",
    "savepath = '../data/'\n",
    "#images = glob.glob(path+'new_data/*.npy')\n",
    "tile_sizes = np.loadtxt(path+'tile_sizes.txt', dtype='int')\n",
    "tile_sizes = tile_sizes[10:800]\n",
    "images_sampled = {}\n",
    "for tile in tile_sizes:\n",
    "    for i in range(1, 500):\n",
    "        images_sampled.setdefault(tile[0]*30+i, []).append(tile[1])\n",
    "print(len(images_sampled.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def myGenerator(batch_size):\n",
    "    s = 192\n",
    "    while True:\n",
    "        index_list = random.sample(images_sampled.keys(), batch_size)\n",
    "        alldata_x = []\n",
    "        alldata_y = []\n",
    "        for i in index_list:\n",
    "            frame = path+'sources/nparrs_384/frame'+str(i)+'.npy'\n",
    "            frame = np.load(frame)\n",
    "            tile_index = random.choice(images_sampled[i])\n",
    "            \n",
    "            tmp = imresize(frame[tile_index], (s, 192))\n",
    "            temp  = imresize(frame[tile_index], (24, 24))\n",
    "            temp  = imresize(temp, (s, 192))\n",
    "            \n",
    "            alldata_x.append(temp)\n",
    "            #alldata_y.append(frame[tile_index])\n",
    "            alldata_y.append(tmp)\n",
    "        \n",
    "        alldata_x = np.array(alldata_x)\n",
    "        alldata_y = np.array(alldata_y)\n",
    "        \n",
    "        alldata_y = (alldata_y.astype(np.float32) - 127.5) / 127.5\n",
    "        alldata_x = alldata_x.astype(np.float32)/255.0\n",
    "        \n",
    "        yield alldata_x, alldata_y\n",
    "\n",
    "x = myGenerator(500)\n",
    "xtrain, ytrain = next(x)\n",
    "print('xtrain shape:',xtrain.shape)\n",
    "print('ytrain shape:',ytrain.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy \n",
    "import math\n",
    "import cv2\n",
    "import time\n",
    "\n",
    "def psnr(img1, img2):\n",
    "    mse = numpy.mean( (img1 - img2) ** 2 )\n",
    "    if mse == 0:\n",
    "        return 100\n",
    "    PIXEL_MAX = 255.0\n",
    "    return 20 * math.log10(PIXEL_MAX / math.sqrt(mse))\n",
    "\n",
    "data = []\n",
    "for i in range(5):\n",
    "    cv2.imwrite(path+'xface' + str(i) + '.jpg', xtrain[i])\n",
    "    cv2.imwrite(path+'yface' + str(i) + '.jpg', ytrain[i])\n",
    "    img1 = cv2.imread(path+'xface' + str(i) + '.jpg')\n",
    "    img2 = cv2.imread(path+'yface' + str(i) + '.jpg')\n",
    "    d = psnr(img1, img2)\n",
    "    print (d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def psnr2(target, ref):\n",
    "    # assume RGB image\n",
    "    target_data = numpy.array(target, dtype=float)\n",
    "    ref_data = numpy.array(ref, dtype=float)\n",
    "\n",
    "    diff = ref_data - target_data\n",
    "    diff = diff.flatten('C')\n",
    "\n",
    "    rmse = math.sqrt(numpy.mean(diff ** 2.))\n",
    "\n",
    "    return 20 * math.log10(255. / rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for i in range(100):\n",
    "    img1 = cv2.imread(path+'test/frame_real'+str(i)+'.png')\n",
    "    img2 = imresize(img1, (1, 1))\n",
    "    img2 = imresize(img2, (384, 384))\n",
    "    d = psnr(img1, img2)\n",
    "    data.append(d)\n",
    "    print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Video Quality Metrics\n",
    "Copyright (c) 2014 Alex Izvorski <aizvorski@gmail.com>\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify\n",
    "it under the terms of the GNU General Public License as published by\n",
    "the Free Software Foundation, either version 3 of the License, or\n",
    "(at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful,\n",
    "but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
    "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
    "GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License\n",
    "along with this program.  If not, see <http://www.gnu.org/licenses/>.\n",
    "\"\"\"\n",
    "\n",
    "import numpy\n",
    "from scipy.ndimage import gaussian_filter\n",
    "\n",
    "from numpy.lib.stride_tricks import as_strided as ast\n",
    "\n",
    "\"\"\"\n",
    "Hat tip: http://stackoverflow.com/a/5078155/1828289\n",
    "\"\"\"\n",
    "def block_view(A, block=(3, 3)):\n",
    "    \"\"\"Provide a 2D block view to 2D array. No error checking made.\n",
    "    Therefore meaningful (as implemented) only for blocks strictly\n",
    "    compatible with the shape of A.\"\"\"\n",
    "    # simple shape and strides computations may seem at first strange\n",
    "    # unless one is able to recognize the 'tuple additions' involved ;-)\n",
    "    shape = (A.shape[0]/ block[0], A.shape[1]/ block[1])+ block\n",
    "    strides = (block[0]* A.strides[0], block[1]* A.strides[1])+ A.strides\n",
    "    return ast(A, shape= shape, strides= strides)\n",
    "\n",
    "\n",
    "def ssim(img1, img2, C1=0.01**2, C2=0.03**2):\n",
    "\n",
    "    bimg1 = block_view(img1, (4,4))\n",
    "    bimg2 = block_view(img2, (4,4))\n",
    "    s1  = numpy.sum(bimg1, (-1, -2))\n",
    "    s2  = numpy.sum(bimg2, (-1, -2))\n",
    "    ss  = numpy.sum(bimg1*bimg1, (-1, -2)) + numpy.sum(bimg2*bimg2, (-1, -2))\n",
    "    s12 = numpy.sum(bimg1*bimg2, (-1, -2))\n",
    "\n",
    "    vari = ss - s1*s1 - s2*s2\n",
    "    covar = s12 - s1*s2\n",
    "\n",
    "    ssim_map =  (2*s1*s2 + C1) * (2*covar + C2) / ((s1*s1 + s2*s2 + C1) * (vari + C2))\n",
    "    return numpy.mean(ssim_map)\n",
    "\n",
    "# FIXME there seems to be a problem with this code\n",
    "def ssim_exact(img1, img2, sd=1.5, C1=0.01**2, C2=0.03**2):\n",
    "\n",
    "    mu1 = gaussian_filter(img1, sd)\n",
    "    mu2 = gaussian_filter(img2, sd)\n",
    "    mu1_sq = mu1 * mu1\n",
    "    mu2_sq = mu2 * mu2\n",
    "    mu1_mu2 = mu1 * mu2\n",
    "    sigma1_sq = gaussian_filter(img1 * img1, sd) - mu1_sq\n",
    "    sigma2_sq = gaussian_filter(img2 * img2, sd) - mu2_sq\n",
    "    sigma12 = gaussian_filter(img1 * img2, sd) - mu1_mu2\n",
    "\n",
    "    ssim_num = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2))\n",
    "\n",
    "    ssim_den = ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2))\n",
    "\n",
    "    ssim_map = ssim_num / ssim_den\n",
    "    return numpy.mean(ssim_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for i in range(100):\n",
    "    img1 = cv2.imread(path+'test/frame_real'+str(i)+'.png')\n",
    "    img2 = imresize(img1, (1, 1))\n",
    "    img2 = imresize(img2, (384, 384))\n",
    "    d = ssim_exact(img1/25, img2)\n",
    "    data.append(d)\n",
    "    print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}