ner_train.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ner_train.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ThanhDucPham/LSTM-NLP-Vietnamese-Tensorflow-/blob/master/ner_train.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "JVBYotTxyRIZ",
        "colab_type": "code",
        "outputId": "43493659-4213-495d-9a18-ce85414c5ff8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "E: Package 'python-software-properties' has no installation candidate\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XOPjIPsJyakm",
        "colab_type": "code",
        "outputId": "e04719ab-41da-4ed1-cc34-dfb1e3459c4c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hls7dmEFgafT",
        "colab_type": "code",
        "outputId": "ddcbcc9a-2127-4838-d549-b05539127142",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "cd /content/drive/My Drive/AI_COLAB/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/AI_COLAB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "x7k7hzSMiNOd",
        "colab_type": "code",
        "outputId": "546aa792-c831-4bf6-b394-e993a86a5629",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install pyvi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyvi in /usr/local/lib/python3.6/dist-packages (0.0.9.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from pyvi) (0.20.3)\n",
            "Requirement already satisfied: sklearn-crfsuite in /usr/local/lib/python3.6/dist-packages (from pyvi) (0.3.6)\n",
            "Requirement already satisfied: scipy>=0.13.3 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->pyvi) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->pyvi) (1.14.6)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (0.8.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (1.11.0)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (4.28.1)\n",
            "Requirement already satisfied: python-crfsuite>=0.8.3 in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->pyvi) (0.9.6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "AFoNBuOwlOmP",
        "colab_type": "code",
        "outputId": "f3a920c1-db04-42bd-ba5a-833290f6acfa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2100
        }
      },
      "cell_type": "code",
      "source": [
        "from NER.data_utils import prepare_data, score\n",
        "from NER.ner_model import NERModel\n",
        "from NER.config import Config\n",
        "\n",
        "# create instance of config\n",
        "config = Config()\n",
        "\n",
        "# build model\n",
        "model = NERModel(config)\n",
        "model.build()\n",
        "\n",
        "# create datasets\n",
        "train_sents, train_labels, length_sentences = prepare_data(config.filename_train,config.vocab_words, colum=[0,3])\n",
        "dev_sents, dev_labels, length_sentences_dev = prepare_data(config.filename_dev, config.vocab_words,colum=[0,3])\n",
        "# train model\n",
        "model.train(train_sents, train_labels, dev_sents,dev_labels)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading projection weights from /content/drive/My Drive/AI_COLAB/DATA/model_trained/VNCorpus_W2VN_big.bin\n",
            "{'kw': {}, 'mode': 'rb', 'uri': '/content/drive/My Drive/AI_COLAB/DATA/model_trained/VNCorpus_W2VN_big.bin'}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "check:  /content/drive/My Drive/AI_COLAB/DATA/model_trained/VNCorpus_W2VN_big.bin\n",
            "loading model word2vec...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loaded (14902, 300) matrix from /content/drive/My Drive/AI_COLAB/DATA/model_trained/VNCorpus_W2VN_big.bin\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Finish in 0.26 sec.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:154: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:154: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:164: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:164: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:168: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /content/drive/My Drive/AI_COLAB/NER/ner_model.py:168: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:110: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /content/drive/My Drive/AI_COLAB/NER/base_model.py:57: get_or_create_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please switch to tf.train.get_or_create_global_step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "From /content/drive/My Drive/AI_COLAB/NER/base_model.py:57: get_or_create_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please switch to tf.train.get_or_create_global_step\n",
            "Initializing tf session\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "len_sentence:  [   31     4    46   141   879  1260   126    40    22    49     0   434\n",
            "  9186     0  1296    20   744 10761  7839     0     5  1128     0    31\n",
            "     4     0   168   126    49   568   434   328    25     0  1288     0\n",
            "  3012   454   134     0     2  1581     0  1157   125   167     0     0\n",
            "     0     0     0   104    13  4471  5583  1963     0    40  2517   161\n",
            "    26    57 11222  1163     0     9  2517   454    56    82  1029    77\n",
            "     0 13821     0     0     0     0    49   630   477     5     0     0\n",
            "   336  2508   187    56   235    13  9962     5   742 12898     0     0\n",
            "   518  4179     5  2497     0  3286     0     0     0   132     0     0\n",
            "  1791   188   336    40  3682    28  2862    92   126    50 13821     0\n",
            "     0     0]\n",
            "label_sentence:  [0 0 0 0 0 0 0 0 0 0 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 3 0 3 0 3 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 3 6 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 2 5 0 0 0 0 1 4 4 0 1 0 0 3 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 3 6 0]\n",
            "total:  12870\n",
            "len_sentence:  [    6   416   290     9   169    39     6   235   290    21    60   249\n",
            "     0  2726   118   284    15   284   141    72    39  1813 13943    20\n",
            "  2129  1024  1360     0]\n",
            "label_sentence:  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0]\n",
            "total:  1678\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 4.9566 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 45.82 - f1 59.01\n",
            "new best score!\n",
            "Epoch 2 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 3.1828 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 53.41 - f1 65.12\n",
            "new best score!\n",
            "Epoch 3 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 2.6500 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 57.99 - f1 68.03\n",
            "new best score!\n",
            "Epoch 4 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 2.3411 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 63.62 - f1 70.56\n",
            "new best score!\n",
            "Epoch 5 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 2.0831 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 59.90 - f1 69.70\n",
            "Epoch 6 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.8500 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 64.21 - f1 71.31\n",
            "new best score!\n",
            "Epoch 7 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.6799 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 63.94 - f1 71.08\n",
            "Epoch 8 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.5844 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 65.21 - f1 72.32\n",
            "new best score!\n",
            "Epoch 9 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.4999 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 65.71 - f1 72.77\n",
            "new best score!\n",
            "Epoch 10 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.4309 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 66.44 - f1 72.99\n",
            "new best score!\n",
            "Epoch 11 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.3083 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 66.03 - f1 72.54\n",
            "Epoch 12 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.2762 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 66.71 - f1 73.16\n",
            "new best score!\n",
            "Epoch 13 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.2343 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 67.39 - f1 73.87\n",
            "new best score!\n",
            "Epoch 14 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.1143 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 68.53 - f1 73.43\n",
            "Epoch 15 out of 15\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "628/644 [============================>.] - ETA: 4s - train loss: 1.0724 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "acc 67.17 - f1 73.40\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "j7fbjGAmlb-Y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}