Text_Summarizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Summarizer:  Producing a concise and fluent summary while preserving key information and overall meaning\n",
    "\n",
    "Extractive Summarization attempts to summarize articles by selecting a subset of words that retain the most important points. This approach weights the important part of sentences and uses the same to form the summary.\n",
    "Sentences are weighted and ranked based on importance and similarity among each other. Cosine similarity is primarily used to measure similarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.cluster.util import cosine_distance\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_article(file_name):\n",
    "    file = codecs.open(file_name, \"r\", encoding='utf-8')  # handles accentuated characters\n",
    "    filedata = file.readlines()\n",
    "    article = filedata[0].split(\". \")    # split the text by sentences using \". \"\n",
    "    \n",
    "    sentences = []\n",
    "    for sentence in article:             # iterate thru sentences, printing each and generate list of wards for each sentence\n",
    "        #print(sentence)\n",
    "        sentences.append(sentence.replace(\"[^a-zA-Z]\", \" \").split(\" \"))    # replace any non character by \" \"\n",
    "    #sentences.pop()   ##### systematically eliminate last sentence of the text from the returned sentences??\n",
    "    \n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_similarity(sentence_1, sentence_2, stopwords=None):\n",
    "    if stopwords is None:\n",
    "        stopwords = []     # create an empty list to avoid error below\n",
    " \n",
    "    sentence_1 = [w.lower() for w in sentence_1]\n",
    "    sentence_2 = [w.lower() for w in sentence_2]\n",
    "\n",
    "    all_words = list(set(sentence_1 + sentence_2))  # create total vocabulary of unique words for the two sentences compared\n",
    "\n",
    "    vector1 = [0] * len(all_words)                  # prepare one-hot vectors for each sentence over all vocab\n",
    "    vector2 = [0] * len(all_words)\n",
    "\n",
    "    # build the vector for the first sentence\n",
    "    for w in sentence_1:\n",
    "        if w in stopwords:\n",
    "            continue \n",
    "        vector1[all_words.index(w)] += 1           # list.index(element) returns the index of the given element in the list\n",
    "\n",
    "    # build the vector for the second sentence\n",
    "    for w in sentence_2:\n",
    "        if w in stopwords:\n",
    "            continue\n",
    "        vector2[all_words.index(w)] += 1\n",
    "\n",
    "    return 1 - cosine_distance(vector1, vector2)   # Cosine = 0 for similar sentences => returns 1 if perfectly similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_similarity_matrix(sentences, stop_words):\n",
    "    # Create an empty similarity matrix\n",
    "    similarity_matrix = np.zeros((len(sentences), len(sentences)))  # create a square matrix with dim the num of sentences\n",
    " \n",
    "    for idx1 in range(len(sentences)):\n",
    "        for idx2 in range(len(sentences)):\n",
    "            if idx1 == idx2: #ignore if both are same sentences (diagonal of the square matrix)\n",
    "                continue\n",
    "            # similarity of each sentence to all other sentences in the text is measured and logged in the matrix\n",
    "            similarity_matrix[idx1][idx2] = sentence_similarity(sentences[idx1], sentences[idx2], stop_words)\n",
    "\n",
    "    return similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_summary(file_name, top_n=5, show=False):\n",
    "    #stop_words = stopwords.words('english')\n",
    "    stop_words = stopwords.words('french')\n",
    "    summarize_text = []\n",
    "    \n",
    "    # Step 1 - Read text and tokenize\n",
    "    sentences =  read_article(file_name)\n",
    "    print(\"number of sentences in text : \", len(sentences))\n",
    "    \n",
    "    # Step 2 - Generate Similary Matrix across sentences\n",
    "    sentence_similarity_matrix = build_similarity_matrix(sentences, stop_words)\n",
    "    \n",
    "    # Step 3 - Rank sentences in similarity matrix. let’s convert the similarity matrix into a graph. \n",
    "    # The nodes of this graph will represent the sentences and the edges will represent the similarity scores between\n",
    "    # the sentences. On this graph, we will apply the PageRank algorithm to arrive at the sentence rankings.\n",
    "    sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_matrix)\n",
    "    scores = nx.pagerank(sentence_similarity_graph)\n",
    "    \n",
    "    # Step 4 - Sort the rank and pick top sentences extract the top N sentences based on their rankings for summary generation\n",
    "    ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)\n",
    "    if show :\n",
    "        print(\"Indexes of top ranked_sentence order are \", ranked_sentence)\n",
    "    # extract the top N sentences based on their rankings for summary generation\n",
    "    for i in range(top_n):\n",
    "        summarize_text.append(\" \".join(ranked_sentence[i][1]))\n",
    "    \n",
    "    # Step 5 - Output the summarize text\n",
    "    print(\"Summarize Text: \\n\", \". \".join(summarize_text)+'.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sentences in text :  36\n",
      "Summarize Text: \n",
      " Le port du masque n’est donc pas utile. Rendre obligatoire le port du masque dans certains lieux extérieurs, relève sans doute plus du principe de précaution que d’une exigence scientifique. À l’instar de Paris, de plus en plus de municipalités rendent obligatoire le port du masque dans les rues et les zones les plus densément occupées. Après Lille, Nice ou encore Toulouse, c’est désormais au tour de Paris de rejoindre la liste grandissante des municipalités rendant obligatoire le port du masque dans certaines rues et certains quartiers. En revanche, sur une terrasse bondée ou lors d’un rassemblement festif très dense, le port du masque est à recommander.\n"
     ]
    }
   ],
   "source": [
    "# let's begin\n",
    "generate_summary( \"covid.txt\", 5, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sentences in text :  22\n",
      "Summarize Text: \n",
      " Le 28 juillet, les trois hommes sont mis en examen pour «tentative d'homicide volontaire en bande organisée», «recel en bande organisée de vol, transport, acquisition, détention d'armes de catégorie B en réunion» et «association de malfaiteurs en vue de la commission de crimes et délits punis de 10 ans d'emprisonnement». Selon le parquet de Paris, les deux jeunes militaires arrêtés le 24 juillet à Créteil (Val-de-Marne) semblaient viser une femme de 54 ans. Les 30 et 31 juillet, deux autres hommes sont à leur tour placés en garde à vue.\n"
     ]
    }
   ],
   "source": [
    "generate_summary( \"dgse.txt\", 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sentences in text :  21\n",
      "Summarize Text: \n",
      " Malgré les menées de l'armée pour étouffer cette affaire, le premier jugement condamnant Dreyfus est cassé par la Cour de cassation au terme d'une enquête minutieuse, et un nouveau conseil de guerre a lieu à Rennes en 1899. À cette date, l'opinion comme la classe politique française est unanimement défavorable à Dreyfus. Le même mois, Mathieu Dreyfus porte plainte auprès du ministère de la Guerre contre Walsin Esterhazy.\n"
     ]
    }
   ],
   "source": [
    "generate_summary( \"dreyfus.txt\", 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sentences in text :  11\n",
      "Summarize Text: \n",
      " The limited study is available for abstractive summarization as it requires a deeper understanding of the text as compared to the extractive approach. It’s good to understand Cosine similarity to make the best use of code you are going to see. Since we will be representing our sentences as the bunch of vectors, we can use it to find the similarity among sentences.\n"
     ]
    }
   ],
   "source": [
    "generate_summary( \"summarize.txt\", 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of sentences in text :  16\n",
      "Summarize Text: \n",
      " So I'm not the one to strike up a conversation about the weather and know that in the next few minutes I have to go and try to win a tennis match. I have not a lot of friends away from the courts.' When she said she is not really close to a lot of players, is that something strategic that she is doing? Is it different on the men's tour than the women's tour? 'No, not at all. I think everyone just thinks because we're tennis players we should be the greatest of friends. When I'm on the courts or when I'm on the court playing, I'm a competitor and I want to beat every single person whether they're in the locker room or across the net. I think just because you're in the same sport doesn't mean that you have to be friends with everyone just because you're categorized, you're a tennis player, so you're going to get along with tennis players.\n"
     ]
    }
   ],
   "source": [
    "generate_summary( \"maria.txt\", 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Format de la Cellule Texte Brut",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}