Mixture_Lucene_Doc2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 960M (CNMeM is enabled with initial size: 50.0% of memory, cuDNN not available)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import lucene\n",
    "\n",
    "from java.io import File \n",
    "from org.apache.lucene.analysis.standard import StandardAnalyzer \n",
    "from org.apache.lucene.document import Document, Field \n",
    "from org.apache.lucene.index import IndexWriter, IndexWriterConfig \n",
    "from org.apache.lucene.store import SimpleFSDirectory \n",
    "from org.apache.lucene.util import Version\n",
    "from org.apache.lucene.search import IndexSearcher \n",
    "from org.apache.lucene.index import IndexReader \n",
    "from org.apache.lucene.queryparser.classic import QueryParser \n",
    "\n",
    "from org.apache.lucene import document, store, util\n",
    "import numpy as np\n",
    "import csv\n",
    "\n",
    "import gensim\n",
    "import time\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "23917\n"
     ]
    }
   ],
   "source": [
    "# PATHS \n",
    "luceneIndexPath = '/home/tarun/PE/lucene/luceneIndexDirectory/'\n",
    "corpus = '/home/tarun/PE/newCorpus/'\n",
    "trainingFilePath = '/home/tarun/PE/Dataset/training_set.tsv'\n",
    "\n",
    "lucene.initVM()\n",
    "\n",
    "# ANALYZER\n",
    "analyzer = StandardAnalyzer(util.Version.LUCENE_CURRENT) \n",
    "\n",
    "# DIRECTORY\n",
    "directory = SimpleFSDirectory(File(luceneIndexPath))\n",
    "\n",
    "\n",
    "# INDEX WRITER\n",
    "writerConfig = IndexWriterConfig(util.Version.LUCENE_CURRENT, analyzer) \n",
    "writer = IndexWriter(directory, writerConfig)\n",
    "\n",
    "print writer.numDocs()\n",
    "# INDEXING ALL DOCUMENTS/ARTICLES IN THE CORPUS\n",
    "for fileName in os.listdir(corpus):\n",
    "    document = Document()\n",
    "    article = os.path.join(corpus, fileName)\n",
    "    content = open(article, 'r').read()\n",
    "    document.add(Field(\"text\", content, Field.Store.YES, Field.Index.ANALYZED))\n",
    "    writer.addDocument(document)\n",
    "print writer.numDocs()\n",
    "writer.close()\n",
    "\n",
    "# INDEX READER\n",
    "reader = IndexReader.open(directory)\n",
    "searcher = IndexSearcher(reader)\n",
    "\n",
    "# QUERYING FOR A QUESTION\n",
    "queryParser = QueryParser(util.Version.LUCENE_CURRENT, \"text\", analyzer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MySentences(object):\n",
    "    def __init__(self, dirname):\n",
    "        self.dirname = dirname\n",
    " \n",
    "    def __iter__(self):\n",
    "        for fname in os.listdir(self.dirname):\n",
    "            for line in open(os.path.join(self.dirname, fname)):\n",
    "                yield line.split()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences = MySentences('/home/tarun/PE/newCorpus/') # a memory-friendly iterator\n",
    "model = gensim.models.Word2Vec(sentences, workers=8) # min_count=5, size=100,\n",
    "model.save('word2vecModel')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "model = gensim.models.Word2Vec.load('word2vecModel')\n",
    "\n",
    "failed = 0\n",
    "\n",
    "def getSimilarity(sentence1, sentence2):\n",
    "    global failed\n",
    "    tokens1 = nltk.word_tokenize(sentence1)\n",
    "    tokens2 = nltk.word_tokenize(sentence2)\n",
    "    try:\n",
    "        tokens1 = filter(lambda x: x in model.vocab, tokens1)\n",
    "        tokens2 = filter(lambda x: x in model.vocab, tokens2)\n",
    "        nonStopWords1 = [word for word in tokens1 if word not in stopwords.words('english')]\n",
    "        nonStopWords2 = [word for word in tokens2 if word not in stopwords.words('english')]\n",
    "        wordSimScores = []\n",
    "        for word1 in nonStopWords1:\n",
    "            for word2 in nonStopWords2:\n",
    "                wordSimScores.append(model.similarity(word1, word2))\n",
    "        if (len(wordSimScores) == 0):\n",
    "            return 0\n",
    "        return np.mean(wordSimScores)\n",
    "    except:\n",
    "        failed+=1\n",
    "    return 0\n",
    "print failed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "0 1.21593475342e-05\n"
     ]
    }
   ],
   "source": [
    "#DocText with options\n",
    "import pickle\n",
    "with open('/home/tarun/PE/Dataset/final_test_set.pkl', 'rb') as f:\n",
    "    test = pickle.load(f)\n",
    "\n",
    "listOfHits = [1,2,3,4,5,6]\n",
    "\n",
    "for eachNum in listOfHits:\n",
    "    print eachNum\n",
    "    submissionFileName = \"luceneModelTest\" + str(eachNum) + \".csv\"\n",
    "    answers = ['A', 'B', 'C', 'D']\n",
    "    submissionFile = open(submissionFileName, \"w\")\n",
    "    writer = csv.writer(submissionFile, delimiter=',')\n",
    "    writer.writerow(['id', 'correctAnswer'])\n",
    "    checkRows = []\n",
    "    num = 0\n",
    "    tic = time.time()\n",
    "    for row in test:\n",
    "        if(num % 1000 == 0):\n",
    "            print num, time.time() - tic\n",
    "        num = num + 1\n",
    "        question = row[1]\n",
    "        answerScores = []\n",
    "        query = queryParser.parse(queryParser.escape(question))\n",
    "        hits = searcher.search(query, eachNum)\n",
    "        docsScores = [hit.score for hit in hits.scoreDocs]\n",
    "        docTexts = \"\"\n",
    "        for hit in hits.scoreDocs:\n",
    "            doc_id = hit.doc\n",
    "            #print doc_id, hit.toString()\n",
    "            docT = searcher.doc(hit.doc)\n",
    "            docText = docT.get(\"text\").encode(\"utf-8\")\n",
    "            docTexts = docTexts + docText\n",
    "\n",
    "        for option in [row[2], row[3], row[4], row[5]]:\n",
    "            # escape for handling special characters like \"/\"\n",
    "            \n",
    "            similarity = getSimilarity(docTexts, option) #docTexts with options\n",
    "            answerScores.append(similarity)\n",
    "            if similarity == 0:\n",
    "                checkRows.append(test.index(row))\n",
    "\n",
    "        writer.writerow([row[0], answers[answerScores.index(np.max(answerScores))]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#DocText with options and question with options... weight of both (sum)\n",
    "import pickle\n",
    "with open('/home/tarun/PE/Dataset/final_test_set.pkl', 'rb') as f:\n",
    "    test = pickle.load(f)\n",
    "\n",
    "listOfHits = [1,2,3,4,5,6]\n",
    "\n",
    "for eachNum in listOfHits:\n",
    "    print eachNum\n",
    "    submissionFileName = \"luceneModelTest\" + str(eachNum) + \".csv\"\n",
    "    answers = ['A', 'B', 'C', 'D']\n",
    "    submissionFile = open(submissionFileName, \"w\")\n",
    "    writer = csv.writer(submissionFile, delimiter=',')\n",
    "    writer.writerow(['id', 'correctAnswer'])\n",
    "    checkRows = []\n",
    "    for row in test:\n",
    "        question = row[1]\n",
    "        answerScores = []\n",
    "        for option in [row[2], row[3], row[4], row[5]]:\n",
    "            # escape for handling special characters like \"/\"\n",
    "            query = queryParser.parse(queryParser.escape(question))\n",
    "            hits = searcher.search(query, eachNum)\n",
    "            docsScores = [hit.score for hit in hits.scoreDocs]\n",
    "            docTexts = \"\"\n",
    "            for hit in hits.scoreDocs:\n",
    "                doc_id = hit.doc\n",
    "                #print doc_id, hit.toString()\n",
    "                docT = searcher.doc(hit.doc)\n",
    "                docText = docT.get(\"text\").encode(\"utf-8\")\n",
    "                docTexts = docTexts + docText\n",
    "            \n",
    "            similarity1 = getSimilarity(docTexts, option) #docTexts with options\n",
    "            similarity2 = getSimilarity(question, option)\n",
    "            similarity = similarity1 + similarity2\n",
    "            answerScores.append(similarity)\n",
    "            if similarity == 0:\n",
    "                checkRows.append(test.index(row))\n",
    "\n",
    "        writer.writerow([row[0], answers[answerScores.index(np.max(answerScores))]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}