Add files via upload

Briefly.ipynb

@@ -0,0 +1,556 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Briefly\n",
+    "\n",
+    "\n",
+    "### __ Problem Statement __\n",
+    "- Obtain news from google news articles\n",
+    "- Sammarize the articles within 60 words\n",
+    "- Obtain keywords from the articles\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "##### Importing all the necessary libraries required to run the following code "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from gnewsclient import gnewsclient   # for fetching google news\n",
+    "from newspaper import Article         # to obtain text from news articles\n",
+    "from transformers import pipeline     # to summarize text\n",
+    "import spacy                          # for named entity recognition\n",
+    "import spacy.displacy as displacy     # display keywords"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Load sshleifer/distilbart-cnn-12-6 model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_model():                           \n",
+    "    model = pipeline('summarization')\n",
+    "    return model\n",
+    "data = gnewsclient.NewsClient(max_results=0)\n",
+    "nlp = spacy.load(\"en_core_web_lg\") "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Obtain urls and it's content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def getNews(topic,location):                \n",
+    "    count=0\n",
+    "    contents=[]\n",
+    "    titles=[]\n",
+    "    authors=[]\n",
+    "    urls=[]\n",
+    "    data = gnewsclient.NewsClient(language='english',location=location,topic=topic,max_results=10) \n",
+    "    news = data.get_news()  \n",
+    "    for item in news:\n",
+    "        url=item['link']\n",
+    "        article = Article(url)\n",
+    "        try:\n",
+    "            article.download()\n",
+    "            article.parse()\n",
+    "            temp=item['title'][::-1]\n",
+    "            index=temp.find(\"-\")\n",
+    "            temp=temp[:index-1][::-1]\n",
+    "            urls.append(url)\n",
+    "            contents.append(article.text)\n",
+    "            titles.append(item['title'][:-index-1])    \n",
+    "            authors.append(temp)\n",
+    "            count+=1\n",
+    "            if(count==5):\n",
+    "                break\n",
+    "        except:\n",
+    "            continue \n",
+    "    return contents,titles,authors,urls         "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Summarizes the content- minimum word limit 30 and maximum 60"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def getNewsSummary(contents,summarizer):   \n",
+    "    summaries=[]     \n",
+    "    for content in contents:\n",
+    "        minimum=len(content.split())\n",
+    "        summaries.append(summarizer(content,max_length=60,min_length=min(30,minimum),do_sample=False,truncation=True)[0]['summary_text'])   \n",
+    "    return summaries"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Named Entity Recognition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Obtain 4 keywords from content (person,organisation or geopolitical entity) \n",
+    "def generateKeyword(contents):            \n",
+    "    keywords=[]\n",
+    "    words=[]      \n",
+    "    labels=[\"PERSON\",\"ORG\",\"GPE\"]\n",
+    "    for content in contents:\n",
+    "        doc=nlp(content)\n",
+    "        keys=[]\n",
+    "        limit=0\n",
+    "        for ent in doc.ents:\n",
+    "            key=ent.text.upper()\n",
+    "            label=ent.label_\n",
+    "            if(key not in words and key not in keywords and label in labels): \n",
+    "                keys.append(key)\n",
+    "                limit+=1\n",
+    "                for element in key.split():\n",
+    "                    words.append(element)\n",
+    "            if(limit==4):\n",
+    "                keywords.append(keys)\n",
+    "                break                           \n",
+    "    return keywords\n",
+    "  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Displaying keywords "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def printKeywords(keywords):\n",
+    "    for keyword in keywords:\n",
+    "        print(keyword)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Displaying the Summary with keywords in it highlighted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def printSummary(summaries,titles):\n",
+    "    colors = {\"ORG\": \"linear-gradient(90deg, #aa9cfc, #fc9ce7)\"}\n",
+    "    options = {\"ents\": [\"PERSON\",\"ORG\",\"GPE\",\"NORP\",\"PERCENT\"],\"colors\": colors} \n",
+    "    for summary,title in zip(summaries,titles):\n",
+    "        doc=nlp(summary) \n",
+    "        print('\\033[1m' + title.upper() + '\\033[0m\\n')\n",
+    "        displacy.render(doc, style=\"ent\", options=options,jupyter=True)\n",
+    "        print(\"\\n\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to sshleifer/distilbart-cnn-12-6 (https://huggingface.co/sshleifer/distilbart-cnn-12-6)\n"
+     ]
+    }
+   ],
+   "source": [
+    "summarizer=load_model() "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "contents,titles,authors,urls=getNews(\"Sports\",\"India\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries=getNewsSummary(contents,summarizer)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "keywords=generateKeyword(contents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['DWAYNE BRAVO', 'SRI LANKA', 'ICC', 'THE WEST INDIES']\n",
+      "['VIRAT KOHLI', 'INDIA', 'SCOTLAND', 'SUPER 12']\n",
+      "['AUSTRALIA', 'AFGHANISTAN', 'CRICKET AUSTRALIA', 'CRICBUZZ STAFF •']\n",
+      "['GARY STEAD', 'TRENT BOULT', 'COLIN DE GRANDHOMME', 'BLACKCAPS']\n",
+      "[\"VIRAT KOHLI'S\", 'TEAM INDIA', 'DHONI', 'UAE']\n"
+     ]
+    }
+   ],
+   "source": [
+    "printKeywords(keywords)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1mT20 WORLD CUP 2021: WEST INDIES AND CHENNAI SUPER KINGS ALL-ROUNDER DWAYNE BRAVO TO RETIRE AFTER SHOWPIECE... \u001b[0m\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\"> \n",
+       "<mark class=\"entity\" style=\"background: #c887fb; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    West Indies\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">NORP</span>\n",
+       "</mark>\n",
+       " all-rounder \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Dwayne Bravo\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " will hang his boots at the end of the ICC T20 World Cup 2021 . \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Bravo\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " told \n",
+       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    ICC\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
+       "</mark>\n",
+       " on the post-match Facebook Live show that he will be drawing the curtains on his international career . \n",
+       "<mark class=\"entity\" style=\"background: #c887fb; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    West Indies\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">NORP</span>\n",
+       "</mark>\n",
+       " lost to \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Sri Lanka\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " by 20 runs in</div></span>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\u001b[1mHAPPY BIRTHDAY VIRAT KOHLI: INDIAN CRICKET TEAM CAPTAIN TURNS 33 \u001b[0m\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\"> \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    India\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " captain \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Virat Kohli\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " turns 33 on Friday . The 33-year-old is currently leading the national side in the ongoing T20 World Cup . \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    India\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " lost their first two games in the tournament before winning the third . \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    India\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " sit in fourth spot in the Group 2 points table .</div></span>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\u001b[1mONE-OFF TEST VS AFGHANISTAN POSTPONED, CONFIRMS CRICKET AUSTRALIA | CRICBUZZ.COM - CRICBUZZ \u001b[0m\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\"> \n",
+       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Cricket Australia's\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
+       "</mark>\n",
+       " one-off Test against \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Afghanistan\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " has officially been postponed . The historic Test has been hanging in the balance since the CA revealed that they wouldn't support the \n",
+       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Taliban\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
+       "</mark>\n",
+       " government's stance against the inclusion of women in sports . Instead of cancelling the Test match, CA has vowed to</div></span>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\u001b[1mNEW ZEALAND INCLUDE FIVE SPINNERS FOR INDIA TOUR, TRENT BOULT OPTS OUT CITING BUBBLE FATIGUE \u001b[0m\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\"> \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    New Zealand\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " name five spinners in 15-man squad for two-Test series against \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    India\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " . Senior pacer \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Trent Boult\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " and fast-bowling all-rounder \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Colin de Grandhomme\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " will miss tour due to bio-bubble fatigue . \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Ajaz Patel\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       ", \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Will Somerville\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " and</div></span>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\u001b[1m‘THERE ARE MANY CANDIDATES BUT HE’S THE BEST': SEHWAG PICKS NEXT INDIA CAPTAIN AFTER KOHLI STEPS DOWN AT END OF T20 WC \u001b[0m\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\"> \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Virat Kohli\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " set to step down as T20I captain after this World Cup in \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    UAE\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " and \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Oman\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " . Many experts are anticipating his deputy \n",
+       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Rohit Sharma\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
+       "</mark>\n",
+       " to fill up the position . Former \n",
+       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    India\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
+       "</mark>\n",
+       " opener \n",
+       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Virender Sehwag\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
+       "</mark>\n",
+       " backed \n",
+       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
+       "    Rohit\n",
+       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
+       "</mark>\n",
+       " as the ideal candidate .</div></span>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "printSummary(summaries,titles)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

Briefly.py

@@ -0,0 +1,119 @@
+import streamlit as st                      #Web App
+from gnewsclient import gnewsclient         # for fetching google news
+from newspaper import Article               # to obtain text from news articles
+from transformers import pipeline           # to summarize text
+import spacy                                # to obtain keyword
+from annotated_text import annotated_text   # to display keywords
+
+
+# Load sshleifer/distilbart-cnn-12-6 model
+@st.cache(allow_output_mutation=True)
+def load_model():                           
+    model = pipeline('summarization')
+    return model
+
+data = gnewsclient.NewsClient(max_results=0) 
+
+
+# obtain urls and it's content
+def getNews(topic,location):                
+    count=0
+    contents=[]
+    titles=[]
+    authors=[]
+    urls=[]
+    data = gnewsclient.NewsClient(language='english',location=location,topic=topic,max_results=10) 
+    news = data.get_news()  
+    for item in news:
+        url=item['link']
+        article = Article(url)
+        try:
+            article.download()
+            article.parse()
+            temp=item['title'][::-1]
+            index=temp.find("-")
+            temp=temp[:index-1][::-1]
+            urls.append(url)
+            contents.append(article.text)
+            titles.append(item['title'][:-index-1])    
+            authors.append(temp)
+            count+=1
+            if(count==5):
+                break
+        except:
+            continue 
+    return contents,titles,authors,urls 
+
+ 
+ # Summarizes the content- minimum word limit 30 and maximum 60
+def getNewsSummary(contents,summarizer):   
+    summaries=[]     
+    for content in contents:
+        minimum=len(content.split())
+        summaries.append(summarizer(content,max_length=60,min_length=min(30,minimum),do_sample=False,truncation=True)[0]['summary_text'])   
+    return summaries
+
+
+# Obtain 4 keywords from content (person,organisation or geopolitical entity)
+def generateKeyword(contents):            
+    keywords=[]
+    words=[]
+    nlp = spacy.load("en_core_web_lg")    
+    labels=["PERSON","ORG","GPE"]
+    for content in contents:
+        doc=nlp(content)
+        keys=[]
+        limit=0
+        for ent in doc.ents:
+            key=ent.text.upper()
+            label=ent.label_
+            if(key not in words and key not in keywords and label in labels): 
+                keys.append(key)
+                limit+=1
+                for element in key.split():
+                    words.append(element)
+            if(limit==4):
+                keywords.append(keys)
+                break                           
+    return keywords
+
+
+# Display title,author and summary in streamlit
+def DisplaySummary(titles,authors,summaries,keywords,urls):
+    for i in range(5):
+        if(i+1<=len(summaries) and i+1<=len(keywords)):
+            st.text("")
+            st.subheader(f'[{titles[i]}] ({urls[i]})')
+            st.markdown(f'<b>{authors[i]}</b>',unsafe_allow_html=True)
+            st.write(summaries[i])
+            if(len(keywords[i])==4):
+                annotated_text("KEYWORDS :",(keywords[i][0],"","#faa")," ",(keywords[i][1],"","#faa")," ",(keywords[i][2],"","#faa")," ",(keywords[i][3],"","#faa"))
+            elif(len(keywords[i])==3):
+                annotated_text("KEYWORDS :",(keywords[i][0],"","#faa")," ",(keywords[i][1],"","#faa")," ",(keywords[i][2],"","#faa"))  
+            elif(len(keywords[i])==2):
+                annotated_text("KEYWORDS :",(keywords[i][0],"","#faa")," ",(keywords[i][1],"","#faa"))  
+            elif(len(keywords[i])==1):
+                annotated_text("KEYWORDS :",(keywords[i][0],"","#faa"))            
+        st.text("")
+        st.text("")
+
+
+def main(): 
+    summarizer=load_model()                 
+    st.title('Briefly')
+    with st.expander('Read trending news in less than 60 words...', expanded=True):
+        with st.form(key='form1'):
+            topic=st.selectbox('Category:',data.topics[2:]+["World"])
+            location=st.selectbox('Location:',data.locations)        
+            submit_button=st.form_submit_button() 
+                          
+    if submit_button:
+        with st.spinner('Fetching news...'):            
+            contents,titles,authors,urls=getNews(topic,location)
+            summaries=getNewsSummary(contents,summarizer)
+            keywords=generateKeyword(contents)
+        DisplaySummary(titles,authors,summaries,keywords,urls)
+
+
+if __name__ == '__main__':
+    main()

Procfile

@@ -0,0 +1 @@
+web: sh setup.sh && streamlit run Briefly.py

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+gnewsclient
+transformers
+newspaper3k
+spacy 
+annotated_text

setup.sh

@@ -0,0 +1,13 @@
+mkdir -p ~/.streamlit/
+
+echo "\
+[general]\n\
+email = \"[email protected]\"\n\
+" > ~/.streamlit/credentials.toml
+
+echo "\
+[server]\n\
+headless = true\n\
+enableCORS=false\n\
+port = $PORT\n\
+" > ~/.streamlit/config.toml