ai_gatys_styletransfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enter main\n",
      "1\n",
      "2.16099e+12\n",
      "1.34829e+11\n",
      "9.60001e+10\n",
      "7.96799e+10\n",
      "7.04823e+10\n",
      "6.45035e+10\n",
      "6.03926e+10\n",
      "5.74574e+10\n",
      "5.52695e+10\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-955c58416abe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mniterations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 159\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    160\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mniterations\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m         \u001b[0mresult_img\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvggnet\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'inputimage'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    715\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 717\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    718\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    913\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 915\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    916\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m--> 965\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    970\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    971\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 972\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    973\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    974\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m    952\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m    953\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 954\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m    955\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    956\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import scipy.misc\n",
    "import scipy.io\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "\n",
    "#image width, height and number of channels\n",
    "imWidth = 400\n",
    "imHeight = 300\n",
    "nchannels = 3\n",
    "\n",
    "vgg19_path = 'imagenet-vgg-verydeep-19.mat'\n",
    "\n",
    "#alpha is the weight given to the content loss and beta to the style loss\n",
    "alpha = 1\n",
    "beta = 500\n",
    "\n",
    "#get the weights and biases from vgg model\n",
    "def getWeights(vgg19_layers, layerid):\n",
    "    weight = vgg19_layers[layerid][0][0][0][0][0]\n",
    "    bias = vgg19_layers[layerid][0][0][0][0][1]\n",
    "    return weight, bias\n",
    "    \n",
    "#create vgg model using weights which are loaded from pretrained vgg19 model mat file\n",
    "def create_tfvgg():   \n",
    "    vgg19 = scipy.io.loadmat(vgg19_path)\n",
    "    \n",
    "    #three classes in struct: 'classes', 'layers' and 'normalization'\n",
    "    #43 structs in 'layers' (1x43 struct)\n",
    "    vgg19_layers = vgg19['layers'][0]\n",
    "    \n",
    "    #vggnet \n",
    "    vggnet = {}\n",
    "    vggnet['inputimage'] = tf.Variable(np.zeros((1, imHeight, imWidth, 3)).astype('float32'))\n",
    "    \n",
    "    weights = getWeights( vgg19_layers, 0 )\n",
    "    vggnet['conv1_1'] = tf.nn.relu(tf.nn.conv2d(vggnet['inputimage'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 2 )\n",
    "    vggnet['conv1_2'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv1_1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    vggnet['pool1'] = tf.nn.avg_pool(vggnet['conv1_2'], ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')\n",
    "    weights = getWeights( vgg19_layers, 5 )\n",
    "    vggnet['conv2_1'] = tf.nn.relu(tf.nn.conv2d(vggnet['pool1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 7 )\n",
    "    vggnet['conv2_2'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv2_1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    vggnet['pool2'] = tf.nn.avg_pool(vggnet['conv2_2'], ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')\n",
    "    weights = getWeights( vgg19_layers, 10 )\n",
    "    vggnet['conv3_1'] = tf.nn.relu(tf.nn.conv2d(vggnet['pool2'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 12 )\n",
    "    vggnet['conv3_2'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv3_1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 14 )\n",
    "    vggnet['conv3_3'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv3_2'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 16 )\n",
    "    vggnet['conv3_4'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv3_3'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    vggnet['pool3'] = tf.nn.avg_pool(vggnet['conv3_4'], ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')\n",
    "    weights = getWeights( vgg19_layers, 19 )\n",
    "    vggnet['conv4_1'] = tf.nn.relu(tf.nn.conv2d(vggnet['pool3'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 21 )\n",
    "    vggnet['conv4_2'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv4_1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 23 )\n",
    "    vggnet['conv4_3'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv4_2'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 25 )\n",
    "    vggnet['conv4_4'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv4_3'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    vggnet['pool4'] = tf.nn.avg_pool(vggnet['conv4_4'], ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')\n",
    "    weights = getWeights( vgg19_layers, 28 )\n",
    "    vggnet['conv5_1'] = tf.nn.relu(tf.nn.conv2d(vggnet['pool4'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 30 )\n",
    "    vggnet['conv5_2'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv5_1'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 32 )\n",
    "    vggnet['conv5_3'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv5_2'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    weights = getWeights( vgg19_layers, 34 )\n",
    "    vggnet['conv5_4'] = tf.nn.relu(tf.nn.conv2d(vggnet['conv5_3'], weights[0], strides=[1, 1, 1, 1], padding='SAME')+ weights[1])\n",
    "    vggnet['pool5'] = tf.nn.avg_pool(vggnet['conv5_4'], ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')\n",
    "    return vggnet\n",
    "\n",
    "\n",
    "#subroutine to save image\n",
    "def saveImage(path,image):\n",
    "    image = image + np.array([103.939, 116.779, 123.68]).reshape((1,1,1,3))\n",
    "    image = image[0]\n",
    "    image = np.clip(image, 0, 255).astype('uint8')\n",
    "    scipy.misc.imsave(path, image)\n",
    "\n",
    "#def main():\n",
    "niterations = 1000\n",
    "sourcePath = 'images/sac.jpg'\n",
    "stylePath = 'images/starry_night2.jpg'\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "print('1')\n",
    "vggnet = create_tfvgg();\n",
    "#generate uniform random white noise\n",
    "xImage = np.random.uniform(-50, 50, (1, imHeight,imWidth, 3)).astype('float32')\n",
    "\n",
    "# read source and style image and perform mean subtraction - \n",
    "#The input images should be zero-centered by mean pixel (rather than mean image) subtraction\n",
    "source = scipy.misc.imread(sourcePath)\n",
    "source = scipy.misc.imresize(source, (imHeight, imWidth))\n",
    "source = np.reshape(source,((1,)+source.shape))\n",
    "#source = source[::-1]\n",
    "source = source - np.array([103.939, 116.779, 123.68]).reshape((1,1,1,3))\n",
    "\n",
    "style = scipy.misc.imread(stylePath)\n",
    "style = scipy.misc.imresize(style, (imHeight, imWidth))\n",
    "style = np.reshape(style,((1,)+style.shape))\n",
    "#style = style[::-1]\n",
    "style = style - np.array([103.939, 116.779, 123.68]).reshape((1,1,1,3))\n",
    "\n",
    "\n",
    "\n",
    "#source loss\n",
    "sess.run([vggnet['inputimage'].assign(source)])\n",
    "p = sess.run(vggnet['conv4_2'])\n",
    "x = vggnet['conv4_2']\n",
    "M = p.shape[1]*p.shape[2]\n",
    "N = p.shape[3]\n",
    "sourceloss = (0.5) * tf.reduce_sum(tf.pow((x - p),2))\n",
    "\n",
    "\n",
    "#style loss\n",
    "sess.run([vggnet['inputimage'].assign(style)])\n",
    "##conv layers and their 'wl's\n",
    "convlayers = [('conv1_1',1.),('conv2_1',1.5),('conv3_1',2.),('conv4_1',2.5),('conv5_1',3.)]\n",
    "# convlayers = [('conv1_1', 0.5), ('conv2_1', 1.0), ('conv3_1', 1.5), ('conv4_1', 3.0), ('conv5_1', 4.0)]\n",
    "styleloss = 0\n",
    "for i in range(len(convlayers)):\n",
    "    a = sess.run(vggnet[convlayers[i][0]])\n",
    "    M = a.shape[1]* a.shape[2]\n",
    "    N = a.shape[3]\n",
    "    aMat = np.reshape(a, (M,N))\n",
    "    A = np.dot(aMat.T,aMat)\n",
    "    \n",
    "    g = vggnet[convlayers[i][0]]\n",
    "    greshaped = tf.reshape(g, (M,N))\n",
    "    G = tf.matmul(tf.transpose(greshaped), greshaped)\n",
    "    \n",
    "    loss = (1./(4*N*N*M*M)) * tf.reduce_sum(tf.pow(G - A, 2))\n",
    "    styleloss = styleloss + convlayers[i][1] * loss\n",
    "\n",
    "totalloss = alpha * sourceloss + beta * styleloss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(2.0)\n",
    "train = optimizer.minimize(totalloss)\n",
    "\n",
    "sess.run(tf.initialize_all_variables())\n",
    "sess.run(vggnet['inputimage'].assign(xImage))\n",
    "\n",
    "if not os.path.exists(OUTOUT_DIR):\n",
    "  os.mkdir(OUTOUT_DIR)\n",
    "\n",
    "for i in range(niterations):\n",
    "    sess.run(train)\n",
    "    if i%100 ==0 or i==niterations-1:\n",
    "        result_img = sess.run(vggnet['inputimage'])\n",
    "        print sess.run(totalloss)\n",
    "        saveImage(os.path.join(OUTOUT_DIR,'%s.png'%(str(i).zfill(4))),result_img)\n",
    "\n",
    "saveImage(os.path.join(OUTOUT_DIR,'final.png'),result_img)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}