06_MINIST_CNN_Basic_for_TensorFlow_Practice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST CNN Basic for TensorFlow Practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST Basic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- (784, 0) 으로 되어 있는 이미지를 (28, 28)로 변환"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img = mnist.train.images[0].reshape(28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x124586940>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADY1JREFUeJzt3WuMHXUZx/HfY2kDQcNFcbOhlbXlVuFFhYVIJEaRGiAm\nxYQUN0EqGFdISSgpiQRJ7AteGNNaTEgka2gsRqoSBQox2ktIalMRWlJ3uSlo2rSl9EKh3QaCUh5f\n7KAL7PzP4czMmdl9vp9ks+fMM5cnJ/vbmXNmzvzN3QUgno/V3QCAehB+ICjCDwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBHdfNjZkZlxMCFXN3a2e+Qnt+M7vCzP5uZi+b2R1F1gWgu6zTa/vNbJqkf0ia\nL2m3pKclDbj784ll2PMDFevGnv9iSS+7+7/c/d+Sfi1pQYH1AeiiIuE/XdKucc93Z9Pex8wGzWyr\nmW0tsC0AJav8Az93H5I0JHHYDzRJkT3/Hkmzxj2fmU0DMAkUCf/Tks4ys8+a2QxJ35S0tpy2AFSt\n48N+d3/HzG6R9CdJ0yStcvfnSusMQKU6PtXX0cZ4zw9UrisX+QCYvAg/EBThB4Ii/EBQhB8IivAD\nQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrw\nA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IquMhuiXJzHZIGpV0TNI77t5fRlMAqlco/JmvuPvB\nEtYDoIs47AeCKhp+l7TOzLaZ2WAZDQHojqKH/Ze6+x4z+7Sk9Wb2ortvGj9D9k+BfwxAw5i7l7Mi\ns2WSjrr78sQ85WwMQC53t3bm6/iw38xONLNPvPdY0tckPdvp+gB0V5HD/h5JD5vZe+t50N3/WEpX\nACpX2mF/WxvjsB+oXOWH/QAmN8IPBEX4gaAIPxAU4QeCIvxAUGV8qw81u+GGG3JrrU7lvvbaa8n6\n3Llzk/UtW7Yk65s3b07WUR/2/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1JQ5zz8wMJCsX3DBBcl6\n6lx505188skdL3vs2LFkfcaMGcn6W2+9lay/+eabubWRkZHksgsXLkzWDxw4kKwjjT0/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwQ1qW7dvWLFitzarbfemlx22rRpRTaNGjzxxBPJeqtrO/bt21dmO5MG\nt+4GkET4gaAIPxAU4QeCIvxAUIQfCIrwA0G1PM9vZqskfV3Sfnc/P5t2qqTfSOqTtEPSQnd/veXG\nCp7n37VrV25t5syZyWWHh4eT9VbfS69Sq3vbP/LII13q5KObP39+sn799dfn1vr6+gptu9V1ANde\ne21ubSrfC6DM8/y/kHTFB6bdIWmju58laWP2HMAk0jL87r5J0qEPTF4gaXX2eLWkq0vuC0DFOn3P\n3+Pue7PHr0rqKakfAF1S+B5+7u6p9/JmNihpsOh2AJSr0z3/PjPrlaTs9/68Gd19yN373b2/w20B\nqECn4V8raVH2eJGkR8tpB0C3tAy/ma2R9BdJ55jZbjP7jqQfSZpvZi9Jujx7DmASmVTf5z/77LNz\na+edd15y2Q0bNiTro6OjHfWEtNmzZ+fWHn/88eSyc+fOLbTt22+/PbeWujfEZMf3+QEkEX4gKMIP\nBEX4gaAIPxAU4QeCmlSn+jC1XHPNNcn6Qw89VGj9Bw8ezK2ddtpphdbdZJzqA5BE+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0EVHq4LSLn55ptzaxdddFGl\n2z7++ONzaxdeeGFy2W3btpXdTuOw5weCIvxAUIQfCIrwA0ERfiAowg8ERfiBoFret9/MVkn6uqT9\n7n5+Nm2ZpO9KOpDNdqe7/6HlxrhvfyV6e3tza9ddd11y2SVLlpTdzvukejNr6/bylThy5EiyftJJ\nJ3Wpk/KVed/+X0i6YoLpK919XvbTMvgAmqVl+N19k6RDXegFQBcVec9/i5kNm9kqMzultI4AdEWn\n4f+ZpDmS5knaK2lF3oxmNmhmW81sa4fbAlCBjsLv7vvc/Zi7vyvp55IuTsw75O797t7faZMAytdR\n+M1s/Ee435D0bDntAOiWll/pNbM1kr4s6VNmtlvSDyV92czmSXJJOyR9r8IeAVSgZfjdfWCCyfdX\n0EtYl19+ebLe6rvng4ODubXZs2d31NNUt2rVqrpbqB1X+AFBEX4gKMIPBEX4gaAIPxAU4QeC4tbd\nJTjzzDOT9fvuuy9Zv+yyy5L1Kr/6unPnzmT99ddfL7T+u+66K7f29ttvJ5e99957k/Vzzjmno54k\n6ZVXXul42amCPT8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBMV5/jbddtttubXFixcnl50zZ06yfvTo\n0WT9jTfeSNbvueee3Fqr89lbtmxJ1ltdB1Clw4cPF1p+dHQ0t/bYY48VWvdUwJ4fCIrwA0ERfiAo\nwg8ERfiBoAg/EBThB4LiPH+bLrnkktxaq/P4a9euTdZXrMgd7UyStGnTpmR9spo3b16yfsYZZxRa\nf+p+AS+++GKhdU8F7PmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKiW5/nNbJakByT1SHJJQ+7+UzM7\nVdJvJPVJ2iFpobsXu8l7g9100025teHh4eSyd999d9ntTAmtxjvo6ekptP4NGzYUWn6qa2fP/46k\npe7+OUlfkLTYzD4n6Q5JG939LEkbs+cAJomW4Xf3ve7+TPZ4VNILkk6XtEDS6my21ZKurqpJAOX7\nSO/5zaxP0ucl/VVSj7vvzUqvauxtAYBJou1r+83s45J+J2mJux8ZP36cu7uZec5yg5IGizYKoFxt\n7fnNbLrGgv8rd/99NnmfmfVm9V5J+yda1t2H3L3f3fvLaBhAOVqG38Z28fdLesHdfzKutFbSouzx\nIkmPlt8egKqY+4RH6/+fwexSSX+WNCLp3WzynRp73/9bSZ+RtFNjp/oOtVhXemMIZfny5cn60qVL\nk/VWtzS/8sorc2tPPvlkctnJzN3bGtO95Xt+d98sKW9lX/0oTQFoDq7wA4Ii/EBQhB8IivADQRF+\nICjCDwTFrbtRqZGRkdzaueeeW2jd69atS9an8rn8MrDnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGg\nOM+PSvX19eXWjjsu/ed3+PDhZH3lypWdtIQMe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrz/Chk\nYGAgWT/hhBNya6Ojo8llBwfTo7zxff1i2PMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFDm7ukZzGZJ\nekBSjySXNOTuPzWzZZK+K+lANuud7v6HFutKbwyNM3369GT9qaeeStZT9+Zfs2ZNctkbb7wxWcfE\n3N3ama+di3zekbTU3Z8xs09I2mZm67PaSndf3mmTAOrTMvzuvlfS3uzxqJm9IOn0qhsDUK2P9J7f\nzPokfV7SX7NJt5jZsJmtMrNTcpYZNLOtZra1UKcAStV2+M3s45J+J2mJux+R9DNJcyTN09iRwYqJ\nlnP3IXfvd/f+EvoFUJK2wm9m0zUW/F+5++8lyd33ufsxd39X0s8lXVxdmwDK1jL8ZmaS7pf0grv/\nZNz03nGzfUPSs+W3B6Aq7Xza/0VJ35I0Ymbbs2l3Shows3kaO/23Q9L3KukQtWp1KvjBBx9M1rdv\n355bW79+fW4N1Wvn0/7NkiY6b5g8pw+g2bjCDwiK8ANBEX4gKMIPBEX4gaAIPxBUy6/0lroxvtIL\nVK7dr/Sy5weCIvxAUIQfCIrwA0ERfiAowg8ERfiBoLo9RPdBSTvHPf9UNq2JmtpbU/uS6K1TZfZ2\nRrszdvUinw9t3GxrU+/t19TemtqXRG+dqqs3DvuBoAg/EFTd4R+qefspTe2tqX1J9NapWnqr9T0/\ngPrUvecHUJNawm9mV5jZ383sZTO7o44e8pjZDjMbMbPtdQ8xlg2Dtt/Mnh037VQzW29mL2W/Jxwm\nrabelpnZnuy1225mV9XU2ywze8LMnjez58zs1mx6ra9doq9aXreuH/ab2TRJ/5A0X9JuSU9LGnD3\n57vaSA4z2yGp391rPydsZl+SdFTSA+5+fjbtx5IOufuPsn+cp7j79xvS2zJJR+seuTkbUKZ3/MjS\nkq6W9G3V+Nol+lqoGl63Ovb8F0t62d3/5e7/lvRrSQtq6KPx3H2TpEMfmLxA0urs8WqN/fF0XU5v\njeDue939mezxqKT3Rpau9bVL9FWLOsJ/uqRd457vVrOG/HZJ68xsm5kN1t3MBHqyYdMl6VVJPXU2\nM4GWIzd30wdGlm7Ma9fJiNdl4wO/D7vU3S+QdKWkxdnhbSP52Hu2Jp2uaWvk5m6ZYGTp/6nztet0\nxOuy1RH+PZJmjXs+M5vWCO6+J/u9X9LDat7ow/veGyQ1+72/5n7+p0kjN080srQa8No1acTrOsL/\ntKSzzOyzZjZD0jclra2hjw8xsxOzD2JkZidK+pqaN/rwWkmLsseLJD1aYy/v05SRm/NGllbNr13j\nRrx2967/SLpKY5/4/1PSD+roIaev2ZL+lv08V3dvktZo7DDwPxr7bOQ7kj4paaOklyRtkHRqg3r7\npaQRScMaC1pvTb1dqrFD+mFJ27Ofq+p+7RJ91fK6cYUfEBQf+AFBEX4gKMIPBEX4gaAIPxAU4QeC\nIvxAUIQfCOq/esVX4lsZQ0YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1244e5780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(img, cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img = img.reshape(-1, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 3x3 크기에 1개 Fature(색상)을 갖는 필터 5에 대한 layer 생성\n",
    "W1 = tf.Variable(tf.random_normal([3,3,1,5], stddev=0.01))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 입력값 img에 필터 layer 를 Stride 2로 하고 padding을 줘서 필터 Layer로 인한 사이즈 축소가 발생하지 않도록 함.\n",
    "- 단, stride 크기가 2이므로 사이즈가 절반으로 줄어듬"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv2d = tf.nn.conv2d(img, W1, strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Conv2D:0\", shape=(1, 14, 14, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(conv2d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv2d_img = conv2d.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv2d_img = np.swapaxes(conv2d_img, 0, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABcCAYAAAB+6068AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAECRJREFUeJztnWtsVGUax/9vsRcqrQwt0CJQEIqIiKKIEhss7goqXlY/\n6K5ImogCHzYxxpgFL1njB11NvEQhREVSMIYVL1FAcANYJbWKBS8IAgW5rCAX6ZXW0o723Q9MZ8/7\nnNOZ6VzOTM/8fwlp/2dOz/v2P3MeTt/L8yitNQghhPR9MpLdAUIIIfGBAZ0QQjwCAzohhHgEBnRC\nCPEIDOiEEOIRGNAJIcQjMKATQohHYEAnhBCPEFNAV0rdpJTap5Q6oJRaFK9O9WXoiTP0xQ49sUNP\nYuO8aH9QKdUPwFIANwI4CqBWKbVWa/1jTz+TlZWlc3Jyom0y5dFaQykFrXUDgGJE4MmAAQN0QUGB\ne51MAoHdyF0AShHBZ6WwsFCPHDnSxR66T289AYDs7Gydm5vrUg/dx7JrfQx6EVO87Ek3zc3Np7XW\ng8OdF3VABzAVwAGt9UEAUEr9G8AdAHo0PycnB1OmTImhydSmubkZhw4dQmNj4yGtdWcknhQUFOCx\nxx5zr5NJ4ODBg3j++edbI/2sjBw5El988YWbXXSdbdu2YcaMGRF7AgC5ubmYMWOGW110nfr6etTU\n1MDv9/fKk+nTp7vVxaSxbt26I5GcF8uQy4UAfrboo4FjBkqp+Uqp7Uqp7Z2dnTE0l/p0dHRA/AUS\n1pPW1lbX+pcsGhsbAcD65tt8sXpy+vRpN7uXFH755RcgjCeA6UtHR4db3UsKZ8+eRUaGEZLSPqb0\nloRPimqtX9daT9FaT8nKykp0c30CqycDBgxIdndSAqsnhYWFye5OymD1JTs7O9ndSQkYU3omloB+\nDMAIix4eOJa2ZGdn4+zZs9ZDae8JAPh8PgCw3nlp78uwYcMAemKQk5ODrq4u66G096S3xBLQawGU\nKqVGK6WyAPwVwNr4dKtvkpeXh/b2dgDIoif/p6SkBABy+Fn5P1dddRVATwx8Ph+6urpAT6In6oCu\ntf4dwN8B/AfAHgBrtNa749WxvkhGRgbGjRsHAONAT4L069cPAP4LflaCnHfeeQA9McjIyED//v0B\nehI1saxygdZ6A4ANceqLJwgsQdyltfbucp7oaKYnNuiJIDMzE1rrccnuR1+FO0UJIcQjMKATQohH\nYEAnhBCPwIBOCCEeIaZJ0VgZO3asoeWur6amprDXOHr0qKEt+SAAAEopQx8+fNjQEydONLTYqeY6\nmZmZhj506JChI9kZF1hVEuSbb74JeX5gZUGQWbNmGVp66Dbh3tNIqKmpMbTcoXv11VcbevBgM23G\n77//3us2E82VV15paLlJLZL756KLLjK03Lz022+/GXrJkiWGvvBCcyNnYPVO0gjsSg4iPYlkc5bc\n2Pb999+HvOa0adMM/eOPZqYCsbY+ofAJnRBCPAIDOiGEeAQGdEII8QgM6IQQ4hFcncHIy8tDeXl5\nUL/44ovG67fccouhA/kuDA4cOGDoyy+/3NBy0mP8+PGGXrNmjaHXrjVTRUydOtXWZiInSnNycozJ\n4crKSuN1OQnqlMy/ubnZ0BdffLGhZQ76888/39Dyffjwww8NvWzZMlubiZwoPXPmDLZs2RLUzz33\nnPH6vn37DO00+SffMzmp+ccffxha/j533XWXoZcuXWro/Px8p64nFJ/PZ/TrhhtuMF7/+eefQ2rA\nPkEuvZM+ycniOXPmGHrz5s1hep3YidLm5masW7cuqO+5556QbV933XW2awwZMsTQx48fD/kzx46F\nzhcmP1tykQKQuIlSPqETQohHYEAnhBCPwIBOCCEewdUx9K6uLmNMeNWqVcbrgwYNMrQciwKA4cOH\nG7q2tjZkm3LM+fHHH7f1ycq2bdts1xgzZkzINmIhNzfX2CAydOhQ43U5Fiw3HgHADz/8YGi5EUcy\nefJkQy9evNjQcrPIZ599ZrtGImtbaq2Nz8mDDz5ovC43fjiNUY4YMcLQ4ca8v/32W0Pff//9hpbj\n+PJzBDjPb8STzs7O7tJ1AICZM2car585c8bQTvMcJ0+eNLTcFCORm2ruu+8+Q8tygZs2bbJdQ74X\n8aSoqAgVFRVBLT+XLS0thna6N+TmKbkxSCLH5eU8nhzHX7Fihe0aiapUxid0QgjxCAzohBDiERjQ\nCSHEI7g6ht7W1mYkSZIJkxKBKNpsG0OTa7SdKs7v3bs3/h0L0NDQYFsbn2h27zareskx6LKyMkPv\n2LHDdo1EjqHn5+fj5ptvTtj1nZC/z6WXXmro5cuXG1quUwec903Ek7a2Nnz11VdBHSh3mFCKi4sN\nLZPZNTQ0GPrZZ5+1XSORY+jt7e3GmHe48e94INepy3mGefPmGVrOxwD2/TDxgk/ohBDiERjQCSHE\nIzCgE0KIR0huNvooGDVqlKHlWnW5tl0WKmhvbze0zN3y0Ucf2dqUuVFSDZm/Rua72b59u6Hb2toM\nXVpaamg5pp6KxR3CIedK5P6Cd955x9ByLb/U0jNZeARI/Bh6PJBzA3IcXv4OEyZMMLRc152VlWVo\np1xIqY4siiHn3WQRELmGXOaLkvcLx9AJIYT0GgZ0QgjxCAzohBDiEVJqDF3mTJHjloB97POKK64w\n9JNPPmlomftl/vz5hn7ttdcMLfMyAPYxNTeReVU+//xz2zlyXblcS3/ZZZcZWua4tuaoB+xrid0s\nchsJct7k3XfftZ0j82fI9ck5OTmGnj59uqFlHnqZR7yoqCiyzrqIHMudNGmS7RxZANyadx4AXnnl\nFUPLfDA+n8/Q11xzjaGd9izIz7CbyDXjcg4NsPdPzhvI+0PmW5L1Ay644AJDv/XWWxH1NR7wCZ0Q\nQjwCAzohhHgEBnRCCPEIKTWG7vf7DV1fX287Z+PGjYaWNRHXr19v6OrqakPLNaTW3BgAjNzk3biR\nc6Yn5HpqOT4HAHfffbeh5Vr9n376ydBy3a3M471//35Dy5qLyebNN9809HvvvWc7R+btlnMj1jqu\ngL3O6ty5cw0t63MmKp91LMj1z/J+AmDLkSPX28s9CXIN9jPPPGPoDz74oNf9dJO8vDxDO60JlzUG\n5L4TGXNWr15taDlOL+cunGreJgo+oRNCiEdgQCeEEI8QNqArpVYopU4ppXZZjg1SSm1SSu0PfPWF\nuoYX2bNnD6qrq/H1118Hj/n9fnz33XcAMDEdfVm1ahUeffRRPP3008FjbW1tePnll4E09WTBggUo\nKSkx0jQ3NDTg1ltvBdLUkx07duDjjz/G5s2bg8c6OztRXV2NlpYWpKMn8SKSMfRKAEsAWJOkLAKw\nRWv9L6XUooD+R6ydOXHihKGdaiI+9dRTvbrm9ddfb2hZH/PIkSOGlmOGPVFcXIzhw4djz549xrV8\nPh8aGxt3AdiCOPjiNI8g6ejoMPS+fftCni9zwO/cudPQx44dM7S1ZmMopk2bhvLyclRWVgaPffLJ\nJxg/fjz27t0bN0/kOOhDDz1kO8dpvXEo5Hiz9ODee+81dEFBQUTXnTt3LhYuXGjURX3hhRdQXl6O\nTz/9NG6eAMDhw4cNXVVVZTtH5iYfNmxYyGsG/uMJIveGyDzxo0ePDtdNlJSUYMyYMUaOobq6Ogwe\nPBh+vx9NTU1x82TgwIGGXrJkie2c3tYMDjy0Bbn99tsNLesJONUpThRhn9C11lsBNIjDdwBYGfh+\nJYC/xLlfKc/AgQNtxWJPnz5t3XCSdr6UlpbaJlh37txpLUScdp6UlZXZip+vX78ec+bM6ZZp50lh\nYaFtIvL48eMoKSnplmnnSbyIdgx9qNa6e2r3BIChPZ2olJqvlNqulNoud995Db/fb8182KMvVk9a\nW1td618yaGlpsa7MicgTWUnea5w6dcpaCSji+0f+JeYlOjo6rLt3GVOiJOZJUX1uXZ0O8frrWusp\nWuspMtWmlwnli9WTVFz+ligi9cSpDKBX6c39I9MkexXGlOiJNqCfVEoVA0Dg66n4danvkpmZGRzP\npi/nyM/PR3NzMwB60s2QIUOCa5fpyTmys7ODOZPoSfREu7FoLYAKAP8KfLVXhYiCRPz5JBP2ywmN\na6+91tBOE0mRUlhYaJ3YjYsv8s9sWbAjGixjlQCAO++809Byg0r//v2jbmvSpEn48ssvu2VcPJFz\nF72dAHVCFvaVE8svvfSSoWMpfDx79my8/fbb3TJu94983+JRnHnmzJmGlknPZNKy2bNnR9VOUVGR\ndYFCwjzp7QSoE3L8X3okJ4rr6upibjNSwgZ0pdRqAOUACpVSRwH8E+cC+Rql1DwARwDc3fMVvMnu\n3bvR1NQEv9+PmpoajBo1CiUlJdi1axcATATQhDTzZfny5airq0NraysWLVqE2267DbNmzcIbb7wB\npKknFRUV2Lp1K+rr6zF27Fg88cQTeOSRR7p3oqalJ7W1tfj111/R2dmJjRs34pJLLsG4ceNQW1vb\nXRHpz0gzT+JF2ICutf5bDy/9Kc596VPIUl7dTJ48GVVVVbu01n92uUtJ54EHHnA8/vDDD2PhwoVp\n6cnKlSsdj2/YsAG5ublp6YlM39xNWVkZqqqq0NjYmHaexAvuFCWEEI+QUsm54kFGhvl/1Kuvvmpo\n+WTt9SVyTjQ0mNsKZMJ+6yYYID5j1KmGLJIhN+UsW7bM0JFuOPMaMrHU+++/b2inIhpeQxZ4WbBg\ngaFlUXa5mclN+IROCCEegQGdEEI8AgM6IYR4BM+NoUtkgnuZ7EoWLvAicl7Bsi4cgD0BkxfHzCXS\nE2vmP8CerEsWwEgXFi9ebGh5P6Va8ZNE0K9fP0OvWbPG0DfeeKOhZdF2N+ETOiGEeAQGdEII8QgM\n6IQQ4hGULEKc0MaU+hXnUgUUAkj1BeCx9LFEax1R0hV6YqePeQJE38+IPQH6nC/0xE7C7x9XA3qw\nUaW2a62nhD8zebjdR3qS/Paihb7YoSd23Ogjh1wIIcQjMKATQohHSFZAfz1J7fYGt/tIT5LfXrTQ\nFzv0xE7C+5iUMXRCCCHxh0MuhBDiEVwN6Eqpm5RS+5RSB5RSi9xsOxRKqRVKqVNKqV2WY4OUUpuU\nUvsDX30JbD/lfKEnduiJM8n0hZ6YuBbQlVL9ACwFcDOACQD+ppSa4Fb7YagEcJM4tgjAFq11KYAt\nAR13UtiXStATSSXoiROVSIIv9MSOm0/oUwEc0Fof1Fp3Avg3gDtcbL9HtNZbATSIw3cA6K4fthLA\nXxLUfEr6Qk/s0BNnkugLPRG4GdAvBGBNbXg0cCxVGaq1Ph74/gSAoaFOjoG+5As9sUNPnHHDF3oi\n4KRoBOhzS4G4HMgCPbFDT5yhL3YS5YmbAf0YgBEWPTxwLFU5qZQqBoDA11MJaqcv+UJP7NATZ9zw\nhZ4I3AzotQBKlVKjlVJZAP4KYK2L7feWtQAqAt9XAPgoQe30JV/oiR164owbvtATidbatX8AbgFQ\nB+AnAI+72XaYfq0GcByAH+fG4eYBKMC5mej9ADYDGJROvtATetIXfKEn5j/uFCWEEI/ASVFCCPEI\nDOiEEOIRGNAJIcQjMKATQohHYEAnhBCPwIBOCCEegQGdEEI8AgM6IYR4hP8BixVURgxUiHoAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1244e5518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, one_img in enumerate(conv2d_img):\n",
    "    plt.subplot(1, 5, i+1), plt.imshow(one_img.reshape(14, 14), cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Max Pooling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Kernel은 2x2 사이즈로 pooling을 하고 Stride : 2, padding을 줘서 pooling Layer로 인한 사이즈 축소가 발생하지 않도록 함.\n",
    "- 단, stride 로 인하여 사이즈가 절반으로 줄어듬"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pool = tf.nn.max_pool(conv2d, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"MaxPool:0\", shape=(1, 7, 7, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pool_img = pool.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 7, 7, 5)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pool_img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 첫번째 축과 마지막 축을 바꿔줌\n",
    "pool_img = np.swapaxes(pool_img, 0, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 7, 7, 1)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pool_img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAABcCAYAAABOZ1+dAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACPFJREFUeJzt3c9rVXcax/HPEzVo2whqYtXGDJ1iieIgpMWFij+gFGdR\nXAmdfyAouHGhDoqIq45d1cVswlBKwWJdGOyi9MeipdqNjT9K1bGDU5y2YonSMPgDkpg+szCOqefk\n+z3JPefe+zXvF4jJfY7n+/Dx5OF6PT/M3QUASEdLoxsAAEwNgxsAEsPgBoDEMLgBIDEMbgBIDIMb\nABLD4AaAxDC4ASAxDG4ASMzsKnZqZjPickx3t6LbzpRMJN12944iG7a3t3tXV1fV/TTchQsXCmfC\ncZJvpuRSdKYUGtxmtlXSUUmzJP3D3f9WQ29PDTP7XmTypDYzu6YCmXR1demrr76qU1uN09bWNsSx\nkkEmNYh+VGJmsyT9XdKfJa2S9BczW1V1Y4kgk6x/iUz+b2xsTJK6xLHyJDKpQZHPuNdKuubuP7j7\niKTjkrZV21YayCTXCJk8NjAwIEnDHCsZZFKDIoP7BUk/Tfj+5/HXfsfMes1swMwGymouIWSSFc3k\n9u3bDWirvm7evClJIxNeyuQyQ4+TYCbSjM2lkNLOKnH3Pnd/1d1fLWufqSOTrImZtLe3N7qdpsBx\nko9cJldkcN+QtHzC953jr+ExMskiE0lLly6VpNYJL5HLQ2RSgyKD+xtJK8zsRTNrlfSmpI+qbSsN\nZJKrlUwee+WVVyRpLsdKBpnUIHo6oLs/MLNdkj7Vw1N33nX3y6E/M2/ePK1YsWLS+vDwcLSxwcHB\nYH1oaCi6jzoonElLS4vmzp07af3+/fslt9YwL0v6pwpk0gzmzJkTrI+Ojta0/9mzZ0vSjyp4rMyf\nP1/r16+fdH8LFy6Mrrls2bJg/ZlnngnWDx8+HF2jBIUzKaKlJf4etLOzM9zQjz8G6xs3bgzW63lq\na6HzuN39Y0kfV9xLctz95Ub30IQu8Zlkxn/JJINMasAl7wCQGAY3ACSGwQ0AiWFwA0BiGNwAkBgG\nNwAkhsENAImp5EEKHR0d2rFjx6T1JUuWRPdx/vz5YH3RokXB+vgVa5OKnUxftu7ubn344YeT1i9e\nvBjdxzvvvBOs79u3L1jfvn17sG5W+LkQpfj11191/PjxSeuxi2Mkad26dTX10NraGqzHjrOyLV++\nPPj3fOXKleg+Tp48Gaxv3bo1WO/r6wvWe3t7oz2UraurSwcOHJi0XuSGZbNmzQrWY/sIXUAnSWfO\nnIn28Ntvv0W3KYJ33ACQGAY3ACSGwQ0AiWFwA0BiGNwAkBgGNwAkhsENAIkxdy9/p2Y177S7uztY\nj904fcOGDcH6uXPnoj3EtnH3wic+l5HJggULgvU1a9YE67t37w7Wt20r5UHb54reZ7mnp8frefP5\nPG1tbcH6nTt3ylijcCZlHCd79+4N1o8cORKsx85n7ujomHJPOQpnIpWTS8xzzz0XrO/atStYf+ut\nt6JrxK6VKDpTeMcNAIlhcANAYhjcAJAYBjcAJIbBDQCJYXADQGIY3ACQmEruxx2zevXq6DaHDh0K\n1s+ePRusv/3228H6G2+8Ee2hnl577bXoNp2dncF6LJOenp4p9dRoY2Nj0W1Onz4drG/evDlYX7Zs\n2VRaariDBw9Gt9m5c2ew/tJLLwXrRe4r3WxC9/9/ZNOmTcH68PBwsP7LL78E6/v374/2UBbecQNA\nYhjcAJAYBjcAJIbBDQCJYXADQGIY3ACQGAY3ACSmIffjbm1tje5jZGQkWO/r6wvW+/v7g/XYfZgl\n6cSJE8F6mffjjt1rW5KGhoaC9dj9w99///1g/ejRo9EeCijtftyx+yNL0t27d4t3luPLL78M1mPn\ngRdR5v24y/jZuXz5crAeu9f9ypUroz0UUOr9uOfMmRPdx+joaNHlcn399dfB+qlTp6L7iF1fUnSm\nFLoAx8yuS7ojaUzSg6kE/jQzs+9EJk/6E7lkkEkWmdRgKldObnH38KMxZh4yyUcuWWSSRSbTxGfc\nAJCYooPbJX1mZufMrDdvAzPrNbMBMxsor72mRyb5Js1lYiaxZxs+ZQpl0ojGGoifn2kq+lHJBne/\nYWaLJX1uZlfd/Xf/q+TufZL6pPo82LMZuHsPmWRcDeUyMZOenh4yEceJ+PmZskLvuN39xvjvg5L6\nJa2tsqmUkEnGqEQuTyCTLDKpQXRwm9mzZtb26GtJr0u6VHVjqSCTjBaJXB65d++eRCZ5yKQGRT4q\neV5Sv5k92v4Dd/+k0q4SYWbfikye1E0ujw0ODkpkkodMatCQC3DKsHjx4mC9u7s7WA9d+FFUmRfg\nlOG9994L1vfs2ROs37p1q4w2SrsApwyx43v8DUmlyrwApwxffPFFsL5ly5aqW5BKvgCnDNu3bw/W\nYxc/HTt2rOYeis4UTgcEgMQwuAEgMQxuAEgMgxsAEsPgBoDEMLgBIDEMbgBITFXncd+S9J8JL7VL\navY7Ck21xz+4e0fRjWdIJtIUciGTrJxMprtmvfHzk1VZJpUM7swiZgPNfqP0evdIJo1fbzoa0SO5\nNH696aiyRz4qAYDEMLgBIDH1GtzhJ/s2h3r3SCaNX286GtEjuTR+vemorMe6fMYNACgPH5UAQGIq\nHdxmttXMvjeza2b21yrXqoWZXTez78zsYtXPtyOTSddr+lzIJItM8lWei7tX8kvSLEn/lvRHSa2S\nvpW0qqr1auz1uqT2OqxDJgnnQiZk0iy5VPmOe62ka+7+g7uPSDouaVuF66WATPKRSxaZZJHJuCoH\n9wuSfprw/c/jrzUjl/SZmZ0zs94K1yGTfKnkQiZZZJKv0lyKPHNyJtjg7jfMbLGkz83sqrtX+0yt\n5kcmWWSSRSb5Ks2lynfcNyQtn/B95/hrTcfdb4z/PiipXw//SVYFMsmXRC5kkkUm+arOpcrB/Y2k\nFWb2opm1SnpT0kcVrjctZvasmbU9+lrS65IuVbQcmeRr+lzIJItM8tUjl8o+KnH3B2a2S9Knevi/\nwe+6++Wq1qvB85L6x5/2PVvSB+7+SRULkUm+RHIhkywyyVd5Llw5CQCJ4cpJAEgMgxsAEsPgBoDE\nMLgBIDEMbgBIDIMbABLD4AaAxDC4ASAx/wNukkKv2BuYpwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1248d5470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, one_img in enumerate(pool_img):\n",
    "    plt.subplot(1, 5, i+1), plt.imshow(one_img.reshape(7, 7), cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST CNN  Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 입력 그래프 작성\n",
    "X = tf.placeholder(tf.float32, [None, 784])\n",
    "X_img = tf.reshape(X, [-1, 28, 28, 1])\n",
    "Y = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 첫번째 Conv Layer 작성\n",
    "- Shape = (?, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"MaxPool_4:0\", shape=(?, 14, 14, 32), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Filter 생성 : 3 x 3, 32개, #feature:1(Color)\n",
    "W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))\n",
    "\n",
    "# Convolutional Layer 생성 : 28 x 28, 32개, stride = 1, Padding은 같도록 => 크기 변환 없음\n",
    "L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')\n",
    "L1 = tf.nn.relu(L1)\n",
    "# Pooling Layer 생성 : 2 x 2, stride = 2, padding은 같도록 => stride=2 이므로 절반 사이즈로 줄어듬\n",
    "L1 = tf.nn.max_pool(L1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME' )\n",
    "print(L1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 두번째 Conv Layer 작성\n",
    "- Shape = (?, 14, 14, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"MaxPool_6:0\", shape=(?, 7, 7, 64), dtype=float32)\n",
      "Tensor(\"Reshape_7:0\", shape=(?, 3136), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Filter 생성 : 3 x 3, 64개, #features: 32\n",
    "W2 = tf.Variable(tf.random_normal([3, 3, 32, 64], stddev=0.01))\n",
    "\n",
    "# Conv Leyer 생성 : 14 x 14, 64개, Stride= 1, padding은 같도록 => 크기 변환 없음\n",
    "L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "L2 = tf.nn.relu(L2)\n",
    "\n",
    "# Pooling Layer 생성 : 2 x x, stride = 2, padding은 같도록 => stride=2이므로 절반 사이즈로 줄어듬\n",
    "L2 = tf.nn.max_pool(L2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "print(L2)\n",
    "# Reshape를 통해 배열을 펼쳐준다.\n",
    "L2 = tf.reshape(L2, [-1, 7*7*64])\n",
    "print(L2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 처음의 MNIST 이미지 (?, 784)를 2개의 Conv Layer를 통과시켜 (?, 3136)으로 만들어줌"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fully Connected(FC, Dense) Layer 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Variable W3 already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:\n\n  File \"<ipython-input-53-37a922edc653>\", line 1, in <module>\n    W3 = tf.get_variable(\"W3\", shape=[7*7*64, 10], initializer=tf.contrib.layers.xavier_initializer())\n  File \"/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-59-a3222f543e4f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 가중치 W3의 초기값은 적절한 값을 설정해주는 Xabier Initialization을 사용하여 설정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mW3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"W3\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxavier_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_normal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mhypothesis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mL2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m   1201\u001b[0m       \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m       \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_getter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1203\u001b[0;31m       constraint=constraint)\n\u001b[0m\u001b[1;32m   1204\u001b[0m get_variable_or_local_docstring = (\n\u001b[1;32m   1205\u001b[0m     \"\"\"%s\n",
      "\u001b[0;32m/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m   1090\u001b[0m           \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1091\u001b[0m           \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_getter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1092\u001b[0;31m           constraint=constraint)\n\u001b[0m\u001b[1;32m   1093\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1094\u001b[0m   def _get_partitioned_variable(self,\n",
      "\u001b[0;32m/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m    423\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m           \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m           constraint=constraint)\n\u001b[0m\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m   def _get_partitioned_variable(\n",
      "\u001b[0;32m/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, constraint)\u001b[0m\n\u001b[1;32m    392\u001b[0m           \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m           use_resource=use_resource, constraint=constraint)\n\u001b[0m\u001b[1;32m    395\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource, constraint)\u001b[0m\n\u001b[1;32m    740\u001b[0m                          \u001b[0;34m\"reuse=tf.AUTO_REUSE in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    741\u001b[0m                          \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 742\u001b[0;31m                              name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m    743\u001b[0m       \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    744\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Variable W3 already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:\n\n  File \"<ipython-input-53-37a922edc653>\", line 1, in <module>\n    W3 = tf.get_variable(\"W3\", shape=[7*7*64, 10], initializer=tf.contrib.layers.xavier_initializer())\n  File \"/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/Users/Eddy/.pyenv/versions/3.5.1/envs/dss/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n"
     ]
    }
   ],
   "source": [
    "# 가중치 W3의 초기값은 적절한 값을 설정해주는 Xabier Initialization을 사용하여 설정\n",
    "W3 = tf.get_variable(\"W3\", shape=[7*7*64, 10], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b = tf.Variable(tf.random_normal([10]))\n",
    "hypothesis = tf.matmul(L2, W3)+b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=hypothesis, labels=Y))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 세션 열기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 학습 시작"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_epochs = 15\n",
    "batch_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:0001 cost = 0.431947107\n",
      "Epoch:0002 cost = 0.350243576\n",
      "Epoch:0003 cost = 0.299538578\n",
      "Epoch:0004 cost = 0.252721545\n",
      "Epoch:0005 cost = 0.215396845\n",
      "Epoch:0006 cost = 0.185032157\n",
      "Epoch:0007 cost = 0.161252189\n",
      "Epoch:0008 cost = 0.143773863\n",
      "Epoch:0009 cost = 0.131097625\n",
      "Epoch:0010 cost = 0.119831594\n",
      "Epoch:0011 cost = 0.111174227\n",
      "Epoch:0012 cost = 0.104551473\n",
      "Epoch:0013 cost = 0.098748127\n",
      "Epoch:0014 cost = 0.093671276\n",
      "Epoch:0015 cost = 0.089466243\n",
      "Training Finished\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0\n",
    "    total_batch = int(mnist.train.num_examples / batch_size)\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        feed_dict = {X: batch_xs, Y: batch_ys}\n",
    "        c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)\n",
    "        avg_cost += c / total_batch\n",
    "    print('Epoch:{:04d} cost = {:.9f}'.format(epoch+1, avg_cost))\n",
    "print('Training Finished')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 테스트 모델"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.9734\n"
     ]
    }
   ],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print('Accuracy', sess.run(accuracy, feed_dict={X: mnist.test.images, Y:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}