SMLR07. Compare one wiki article to a list of articles (comparewikilist.py).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikionly #script name is wikionly (no summary), class name is wiki\n",
    "import re as re\n",
    "import nltk\n",
    "# nltk.download('wordnet')\n",
    "from nltk.corpus import wordnet\n",
    "import math\n",
    "\n",
    "#Input two Wikipedia articles to compute similarity percentage\n",
    "class similar:\n",
    "    def __init__(self,text1,text2,verbose=1):\n",
    "        \"\"\"To start, assign var = comparewiki.similar('arg1','arg2', verbose=1). \n",
    "        arg1 and arg2 are names of the wikipedia articles.\n",
    "        verbose=1 prints the probability score and mathematical calculation. \n",
    "        verbose=2 additionally prints array of words for each article\n",
    "        verbose=0 disables any logs.\n",
    "        To get values in a list for storage, use .ans(). To get the 40 common words for comparison, use .words()\"\"\"\n",
    "\n",
    "        self.wn = nltk.corpus.wordnet #the corpus reader\n",
    "        self.verbose = verbose # Verbose/log level of detail\n",
    "\n",
    "        #Error handling: check if both arguments input are string format\n",
    "        checkstr = False\n",
    "        if isinstance(text1, str) == True:\n",
    "            if isinstance(text2, list) == True:\n",
    "                self.text1 = text1\n",
    "                self.text2 = text2\n",
    "                checkstr = True\n",
    "            else:\n",
    "                print('Error! The second argument is not a list format!')\n",
    "                return\n",
    "        else:\n",
    "            print('Error! The first argument is not a string format!')\n",
    "            return\n",
    "        \n",
    "        #Run internal wikipedia python file for processing for both wiki titles\n",
    "        if checkstr == True:\n",
    "            self.wiki1 = wikionly.wiki(text1)\n",
    "            # Get list of 40 common words for first article (string)\n",
    "            # This means that article won't have to keep calling commonwords function in percent and iterate function\n",
    "            self.dotn01 = ('.','n','.','0','1')\n",
    "            self.wiki1list = []\n",
    "            for key in self.wiki1.commonwords(40):\n",
    "                self.wiki1slice = list(key)\n",
    "                for letter in self.dotn01:\n",
    "                    self.wiki1slice.append(letter)\n",
    "                self.wiki1slice = ''.join(self.wiki1slice)\n",
    "                self.wiki1list.append(self.wiki1slice)\n",
    "            \n",
    "    def iterate(self):\n",
    "        # List that stores list of results\n",
    "        self.matrix = []\n",
    "        # Index for getting write article from self.text2 list\n",
    "        self.index = 0\n",
    "        \n",
    "        for article in self.text2:\n",
    "            self.wiki2 = wikionly.wiki(article)\n",
    "            # Call function that calculates percentage\n",
    "            pct = self.percent(self.wiki1,self.wiki2,self.verbose)\n",
    "            # Call the function that shows list of words for both Wiki sites\n",
    "            # Only can be used if self.percent has been called and list/arrays for articles are created\n",
    "            if self.verbose == 2:\n",
    "                print(self.words())\n",
    "            self.matrix.append(self.ans(self.index))\n",
    "            self.index += 1\n",
    "            \n",
    "        return self.matrix\n",
    "                \n",
    "    \n",
    "    #Retrieve top 40 common words from wiki page, slice up and append .n01 for NLTK usage\n",
    "    def percent(self,input1,input2,verbose):\n",
    "\n",
    "        self.wiki2list = []\n",
    "        for key in self.wiki2.commonwords(40):\n",
    "            self.wiki2slice = list(key)\n",
    "            for letter in self.dotn01:\n",
    "                self.wiki2slice.append(letter)\n",
    "            self.wiki2slice = ''.join(self.wiki2slice)\n",
    "            self.wiki2list.append(self.wiki2slice)\n",
    "        \n",
    "        #count and sum for calculating similarity\n",
    "        self.count = 0\n",
    "        self.sum = 0\n",
    "        #A count for the ranking of the word (how often it appears in both wiki passages)\n",
    "        self.topten1 = 0\n",
    "        self.topten2 = 0\n",
    "\n",
    "        #For words that are 1-10th and 11-21st in popularity, if both wiki pages have the word, they get more points\n",
    "        for word1 in self.wiki1list:\n",
    "            #Reset self.topten2\n",
    "            self.topten2 = 0\n",
    "            self.topten1 += 1\n",
    "            for word2 in self.wiki2list:\n",
    "                self.topten2 += 1\n",
    "                #reinitialize to zero to prevent old sums from going into maxsum\n",
    "                self.sum1 = 0\n",
    "                self.sum2 = 0\n",
    "                self.sum3 = 0\n",
    "                self.sum4 = 0\n",
    "                self.maxsum = 0\n",
    "                \n",
    "                if self.topten1 < 11 and self.topten2 < 11:\n",
    "                    self.expvalue = 4.5\n",
    "                elif self.topten1 < 21 and self.topten2 < 21:\n",
    "                    self.expvalue = 2.5\n",
    "                else:\n",
    "                    self.expvalue = 1.5\n",
    "                \n",
    "                #Main algorithm for calculating score of words\n",
    "                try:\n",
    "                    if re.findall(r\"\\d+.n.01\", word1) == [] and re.findall(r\"\\d+.n.01\", word2) == []: #check both words not numbers\n",
    "                        #since words have many meanings, for every pair of words, use top two meanings n.01 and n.02 for comparison\n",
    "                        #two for loops will check every permutation pair of words between wiki pages, two meanings for each word, \n",
    "                        #Take the max similarity value taken for computation of similarity index\n",
    "                        #e.g. money.n.01 may have highest value with value.n.02 because value.n.01 has the obvious meaning of worth/significance and secondary for money\n",
    "                        word11 = word1.replace('n.01','n.02')\n",
    "                        word22 = word2.replace('n.01','n.02')\n",
    "                        #print(word11,word22)\n",
    "                        self.x = self.wn.synset(word1)\n",
    "                        self.y = self.wn.synset(word2)\n",
    "                        #get default similarity value of 1st definitions of word\n",
    "                        self.sum1 = self.x.path_similarity(self.y) * math.exp(self.expvalue * self.x.path_similarity(self.y)) + 10 * math.log(0.885+self.x.path_similarity(self.y))\n",
    "                        try: #get 2nd definitions of words and their similarity values, if it exist\n",
    "                            self.xx = self.wn.synset(word11)\n",
    "                            self.yy = self.wn.synset(word22)\n",
    "                            self.sum2 = self.xx.path_similarity(self.y) * math.exp(self.expvalue * self.xx.path_similarity(self.y)) + 10 * math.log(0.89+self.xx.path_similarity(self.y))\n",
    "                            self.sum3 = self.x.path_similarity(self.yy) * math.exp(self.expvalue * self.x.path_similarity(self.yy)) + 10 * math.log(0.89+self.x.path_similarity(self.yy))\n",
    "                            self.sum4 = self.xx.path_similarity(self.yy) * math.exp(self.expvalue * self.xx.path_similarity(self.yy)) + 10 * math.log(0.89+self.xx.path_similarity(self.yy))\n",
    "                        except:\n",
    "                            continue\n",
    "                        self.maxsum = max(self.sum1,self.sum2,self.sum3,self.sum4) #get the max similarity value between 2 words x 2 meanings = 4 comparisons\n",
    "                        #print(word1, word2, self.maxsum)\n",
    "                        self.sum += self.maxsum\n",
    "                        self.count += 1\n",
    "                except:\n",
    "                    if word1 == word2 and re.findall(r\"\\d+.n.01\", word1) == []: #remove years/numbers being counted as match yyyy.n.01\n",
    "                        self.sum += math.exp(self.expvalue) + 10 * math.log(1.89)\n",
    "                        self.count += 1\n",
    "                    else:\n",
    "                        continue\n",
    "\n",
    "        #Print the results and implement ceiling if the percent exceeds 100% or drops below 0%\n",
    "        if self.count != 0:\n",
    "            self.pct = round(self.sum/self.count*100)\n",
    "            if self.pct > 100:\n",
    "                self.pct = 100\n",
    "            elif self.pct < 0:\n",
    "                self.pct = 0\n",
    "            if self.verbose >= 1:\n",
    "                print('Probability of topics being related is ' + str(self.pct) + '%')\n",
    "                print('Count is ' + str(self.count) + ' and sum is ' + str(self.sum))\n",
    "        else:\n",
    "            if self.verbose >= 1:\n",
    "                print('No relation index can be calculated as words are all foreign')\n",
    "            \n",
    "        return self.pct\n",
    "        \n",
    "    #Print out list of common words for both Wiki articles\n",
    "    def words(self):\n",
    "        print(self.wiki1list)\n",
    "        print('\\n')\n",
    "        print(self.wiki2list)\n",
    "        \n",
    "    #Outputs list of results [Article 1, Article 2, Percentage, Yes/No] that can be put into a dataframe\n",
    "    def ans(self, index=0):\n",
    "        self.listans = [self.text1,self.text2[index],self.pct]\n",
    "        if self.pct > 49:\n",
    "            self.listans.append('Yes')\n",
    "        else:\n",
    "            self.listans.append('No')\n",
    "        \n",
    "        if self.verbose == 2:\n",
    "            self.listans.append(self.wiki1list)\n",
    "            self.listans.append(self.wiki2list)\n",
    "        \n",
    "        return self.listans\n",
    "    \n",
    "    def help(self):\n",
    "        print(\"To start, assign var = comparewiki.similar('arg1','arg2', verbose=1). arg1 and arg2 are names of the wikipedia articles, while verbose=1 prints the probability score and mathematical calculation. verbose=2 additionally prints array of words for each article, and verbose=0 disables any logs. To get values in a list for storage, use .ans(). To get the 40 common words for comparison, use .words()\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Probability of topics being related is 95%\n",
      "Count is 511 and sum is 486.3398758941804\n",
      "Probability of topics being related is 96%\n",
      "Count is 442 and sum is 423.64113859834\n",
      "Probability of topics being related is 81%\n",
      "Count is 486 and sum is 395.91641808510576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['Armin van Buuren', 'Tiesto', 95, 'Yes'],\n",
       " ['Armin van Buuren', 'Martin Garrix', 96, 'Yes'],\n",
       " ['Armin van Buuren', 'Swedish House Mafia', 81, 'Yes']]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similar('Armin van Buuren',['Tiesto','Martin Garrix','Swedish House Mafia']).iterate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = similar('Armin van Buuren',['Tiesto','Martin Garrix','Swedish House Mafia'], verbose=0).iterate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Armin van Buuren', 'Tiesto', '95', 'Yes'],\n",
       "       ['Armin van Buuren', 'Martin Garrix', '96', 'Yes'],\n",
       "       ['Armin van Buuren', 'Swedish House Mafia', '81', 'Yes']],\n",
       "      dtype='<U19')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.array(array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Armin van Buuren</td>\n",
       "      <td>Tiesto</td>\n",
       "      <td>95</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Armin van Buuren</td>\n",
       "      <td>Martin Garrix</td>\n",
       "      <td>96</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Armin van Buuren</td>\n",
       "      <td>Swedish House Mafia</td>\n",
       "      <td>81</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0                    1   2    3\n",
       "0  Armin van Buuren               Tiesto  95  Yes\n",
       "1  Armin van Buuren        Martin Garrix  96  Yes\n",
       "2  Armin van Buuren  Swedish House Mafia  81  Yes"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(np.array(array))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}