updated notebook and ' fix in app

app.py

@@ -49,6 +49,10 @@ def get_youtube_data(url):
 
     title, author = data["title"], data["author_name"]
 
+    # ' is a reserved character
+    title = title.replace("'", "")
+    author = author.replace("'", "")
+
     df = pd.DataFrame(raw)
 
     df['end'] = df['start'] + df['duration']

notebook.ipynb

@@ -2,352 +2,70 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 17,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "No transcript found\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from youtube_transcript_api import YouTubeTranscriptApi\n",
-    "from nltk.tokenize import TextTilingTokenizer  \n",
+    "from nltk.tokenize import TextTilingTokenizer \n",
+    "from youtubesearchpython import VideosSearch\n",
+    "from semantic_search import SemanticSearch \n",
     "import pandas as pd\n",
+    "import gradio as gr\n",
     "import numpy as np\n",
     "import requests\n",
+    "import tiktoken\n",
+    "import openai\n",
     "import json\n",
-    "\n",
-    "url = \"https://www.youtube.com/watch?v=z7-K1zmBu-8\"\n",
-    "video_id = url.split(\"=\")[1]\n",
-    "\n",
-    "try:\n",
-    "    raw = YouTubeTranscriptApi.get_transcript(video_id)\n",
-    "except:\n",
-    "    try:\n",
-    "        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)\n",
-    "        for transcript in transcript_list:\n",
-    "            raw = transcript.translate('en').fetch()\n",
-    "            break\n",
-    "    except:\n",
-    "        print(\"No transcript found\")\n",
-    "        raw = []\n",
-    "\n",
-    "response = requests.get(f\"https://noembed.com/embed?dataType=json&url={url}\")\n",
-    "data = json.loads(response.content)\n",
-    "\n",
-    "title, author = data[\"title\"], data[\"author_name\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 42,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[{'text': '[Music]', 'start': 2.19, 'duration': 3.5},\n",
-       " {'text': '[Music]', 'start': 18.73, 'duration': 3.07},\n",
-       " {'text': '[Applause]', 'start': 27.71, 'duration': 3.289},\n",
-       " {'text': '[Music]', 'start': 33.68, 'duration': 7.01},\n",
-       " {'text': '[Laughter] [Music] [Music] [', 'start': 36.05, 'duration': 4.64},\n",
-       " {'text': 'Applause]', 'start': 59.97, 'duration': 3.2},\n",
-       " {'text': '[Music]', 'start': 68.78, 'duration': 3.12},\n",
-       " {'text': 'Recently, the', 'start': 72.18, 'duration': 3.0},\n",
-       " {'text': \"issue of sexual assault by celebrities has not stopped.  It's\",\n",
-       "  'start': 79.26,\n",
-       "  'duration': 3.24},\n",
-       " {'text': \"true that it happened to me. It's\",\n",
-       "  'start': 87.74,\n",
-       "  'duration': 3.96},\n",
-       " {'text': 'reallyembarrassing', 'start': 96.299, 'duration': 3.721},\n",
-       " {'text': \"[Music] It's disastrous\", 'start': 98.16, 'duration': 4.099},\n",
-       " {'text': '[Music]', 'start': 102.36, 'duration': 3.18},\n",
-       " {'text': \"There's also a part where feminists are obsessed with men's genitals. I think their germinating\",\n",
-       "  'start': 111.6,\n",
-       "  'duration': 4.5},\n",
-       " {'text': 'power is a really scary part.  I',\n",
-       "  'start': 113.82,\n",
-       "  'duration': 4.259},\n",
-       " {'text': 'think this castration will happen more often as they get castrated.',\n",
-       "  'start': 121.259,\n",
-       "  'duration': 3.441},\n",
-       " {'text': 'In fact, feminism was popular at the time,',\n",
-       "  'start': 133.58,\n",
-       "  'duration': 7.659},\n",
-       " {'text': 'but thanks to its popularity,', 'start': 137.58, 'duration': 8.04},\n",
-       " {'text': 'a lot of', 'start': 141.239, 'duration': 8.041},\n",
-       " {'text': 'things happened, such as scolding, ridicule, insults, and',\n",
-       "  'start': 145.62,\n",
-       "  'duration': 5.04},\n",
-       " {'text': 'insults against', 'start': 149.28, 'duration': 2.28},\n",
-       " {'text': 'men.  I', 'start': 150.66, 'duration': 2.7},\n",
-       " {'text': \"just couldn't stay there. Well, the\",\n",
-       "  'start': 153.36,\n",
-       "  'duration': 5.459},\n",
-       " {'text': 'pepper is 3 cm.', 'start': 160.2, 'duration': 5.58},\n",
-       " {'text': 'Besides, all men are potential rape',\n",
-       "  'start': 162.36,\n",
-       "  'duration': 7.019},\n",
-       " {'text': 'criminals. Men are useless. Men stopped',\n",
-       "  'start': 165.78,\n",
-       "  'duration': 6.599},\n",
-       " {'text': 'trusting women.  If you', 'start': 176.34, 'duration': 4.92},\n",
-       " {'text': 'reach there, you may be hit by the #MeToo movement, so I',\n",
-       "  'start': 184.92,\n",
-       "  'duration': 3.86},\n",
-       " {'text': 'think there are a lot of them right now. I think',\n",
-       "  'start': 200.28,\n",
-       "  'duration': 2.539},\n",
-       " {'text': 'there may be a little more than in other countries.',\n",
-       "  'start': 221.76,\n",
-       "  'duration': 4.8},\n",
-       " {'text': '[Applause]', 'start': 238.29, 'duration': 3.23},\n",
-       " {'text': '[Music]', 'start': 243.27, 'duration': 7.169},\n",
-       " {'text': 'Personally, I', 'start': 245.78, 'duration': 8.019},\n",
-       " {'text': \"would say it's content that only has these emotions. As an\",\n",
-       "  'start': 250.439,\n",
-       "  'duration': 6.061},\n",
-       " {'text': 'example, I said that we need to strongly pass the anti-discrimination law, but',\n",
-       "  'start': 253.799,\n",
-       "  'duration': 4.801},\n",
-       " {'text': 'this is actually an expression', 'start': 258.6, 'duration': 3.89},\n",
-       " {'text': 'dictatorship class.  Guys', 'start': 259.919, 'duration': 5.111},\n",
-       " {'text': '[Applause]', 'start': 262.49, 'duration': 4.149},\n",
-       " {'text': '[Music] I think you', 'start': 265.03, 'duration': 3.949},\n",
-       " {'text': \"'re talking a lot, but I want to hear it. Oh, you're not going to listen. You're so mean.\",\n",
-       "  'start': 268.979,\n",
-       "  'duration': 5.041},\n",
-       " {'text': 'For example, I went to the bathroom, and I',\n",
-       "  'start': 295.02,\n",
-       "  'duration': 2.7},\n",
-       " {'text': 'saw that it was unisex, but I want it to be safe with the door open.  I',\n",
-       "  'start': 297.72,\n",
-       "  'duration': 4.8},\n",
-       " {'text': 'think I was able to see well how men and women cut off harmony at the source. My',\n",
-       "  'start': 354.18,\n",
-       "  'duration': 2.6},\n",
-       " {'text': 'girlfriend is', 'start': 357.96, 'duration': 4.5},\n",
-       " {'text': 'not satisfied with something like that.',\n",
-       "  'start': 359.759,\n",
-       "  'duration': 4.081},\n",
-       " {'text': 'Could anyone who really thought about gender equality talk about peeing and taking a shower?  I think',\n",
-       "  'start': 371.539,\n",
-       "  'duration': 3.541},\n",
-       " {'text': '[Music]', 'start': 378.61, 'duration': 3.159},\n",
-       " {'text': 'In fact, many companies are', 'start': 384.479, 'duration': 3.321},\n",
-       " {'text': 'paying for it,', 'start': 388.979, 'duration': 4.44},\n",
-       " {'text': 'or in the labor market, there', 'start': 390.12, 'duration': 6.74},\n",
-       " {'text': 'is employment discrimination without any reason for gender inequality.',\n",
-       "  'start': 393.419,\n",
-       "  'duration': 3.441},\n",
-       " {'text': 'Evil men must disappear', 'start': 450.539, 'duration': 7.141},\n",
-       " {'text': 'Because of these millitons, there are many such people',\n",
-       "  'start': 454.56,\n",
-       "  'duration': 7.139},\n",
-       " {'text': 'in Korean society, and Ha Tae-kyung is a',\n",
-       "  'start': 457.68,\n",
-       "  'duration': 4.019},\n",
-       " {'text': 'representative abolitionist.  It',\n",
-       "  'start': 472.979,\n",
-       "  'duration': 7.521},\n",
-       " {'text': 'means that I oppose giving privileges to Han Gender.',\n",
-       "  'start': 503.099,\n",
-       "  'duration': 5.641},\n",
-       " {'text': 'Korean feminists are now', 'start': 504.3, 'duration': 8.58},\n",
-       " {'text': 'understood as', 'start': 508.74, 'duration': 6.5},\n",
-       " {'text': 'discriminatory against men.  In such a society,',\n",
-       "  'start': 528.42,\n",
-       "  'duration': 4.56},\n",
-       " {'text': 'I', 'start': 534.8, 'duration': 6.479},\n",
-       " {'text': 'think we should pay more attention to the serious discrimination against women.',\n",
-       "  'start': 537.18,\n",
-       "  'duration': 4.099},\n",
-       " {'text': 'Recently,', 'start': 544.08, 'duration': 3.0},\n",
-       " {'text': 'when I look at the political situation in Korea, I feel that it is now retreating.',\n",
-       "  'start': 551.459,\n",
-       "  'duration': 4.741},\n",
-       " {'text': '19% of female lawmakers in the National Assembly',\n",
-       "  'start': 559.62,\n",
-       "  'duration': 4.44},\n",
-       " {'text': 'are now 19%.', 'start': 561.06, 'duration': 3.0},\n",
-       " {'text': 'Why do people who send me messages like this send me messages like this',\n",
-       "  'start': 592.14,\n",
-       "  'duration': 3.84},\n",
-       " {'text': 'when they come every day?  It', 'start': 593.76, 'duration': 4.92},\n",
-       " {'text': \"'s necessary, but it seems that there\",\n",
-       "  'start': 615.3,\n",
-       "  'duration': 3.479},\n",
-       " {'text': 'are many cases where the target is directed at women,',\n",
-       "  'start': 618.779,\n",
-       "  'duration': 3.74},\n",
-       " {'text': \"but if there's a motto that I\", 'start': 645.42, 'duration': 5.82},\n",
-       " {'text': \"personally take while leading this group, let's\",\n",
-       "  'start': 647.12,\n",
-       "  'duration': 5.32},\n",
-       " {'text': \"create a world where feminists don't have to choose feminism.  I\",\n",
-       "  'start': 657.899,\n",
-       "  'duration': 3.721},\n",
-       " {'text': 'choose', 'start': 665.339, 'duration': 5.161},\n",
-       " {'text': \"feminism because I think you're watching.\",\n",
-       "  'start': 676.019,\n",
-       "  'duration': 4.081},\n",
-       " {'text': 'As a person, I live to protect the woman I love.',\n",
-       "  'start': 686.959,\n",
-       "  'duration': 5.701},\n",
-       " {'text': \"I think I'm about the level of a director who creates a hero.\",\n",
-       "  'start': 697.82,\n",
-       "  'duration': 6.94},\n",
-       " {'text': 'Well,', 'start': 701.04, 'duration': 3.72},\n",
-       " {'text': 'one day,', 'start': 707.16, 'duration': 10.41},\n",
-       " {'text': '[Music] We were', 'start': 727.75, 'duration': 4.46},\n",
-       " {'text': 'humiliated like', 'start': 730.2, 'duration': 5.139},\n",
-       " {'text': 'this.  I', 'start': 736.7, 'duration': 4.84},\n",
-       " {'text': \"think there are so many messages in this very short video. First of all, I think there's\",\n",
-       "  'start': 741.54,\n",
-       "  'duration': 2.64},\n",
-       " {'text': 'enough room for it to be interpreted as',\n",
-       "  'start': 744.18,\n",
-       "  'duration': 2.54},\n",
-       " {'text': 'sexual', 'start': 756.36, 'duration': 4.979},\n",
-       " {'text': 'harassment.', 'start': 762.66, 'duration': 5.1},\n",
-       " {'text': 'But I really', 'start': 777.42, 'duration': 4.38},\n",
-       " {'text': 'had no intention of interfering with the event.',\n",
-       "  'start': 780.3,\n",
-       "  'duration': 3.42},\n",
-       " {'text': 'It was a chance to share, but I',\n",
-       "  'start': 791.04,\n",
-       "  'duration': 7.28},\n",
-       " {'text': \"think I couldn't give you a good answer after hearing that conversation.\",\n",
-       "  'start': 795.12,\n",
-       "  'duration': 3.2},\n",
-       " {'text': '[Music] I was', 'start': 802.19, 'duration': 2.649},\n",
-       " {'text': 'very surprised. I', 'start': 803.76, 'duration': 2.04},\n",
-       " {'text': 'went with my faith,', 'start': 805.8, 'duration': 4.26},\n",
-       " {'text': 'but', 'start': 819.06, 'duration': 2.6},\n",
-       " {'text': \"I felt completely betrayed.  It's\",\n",
-       "  'start': 822.019,\n",
-       "  'duration': 3.991},\n",
-       " {'text': 'just', 'start': 824.16, 'duration': 4.91},\n",
-       " {'text': '[Music] [', 'start': 826.01, 'duration': 3.06},\n",
-       " {'text': 'Music] If', 'start': 831.75, 'duration': 3.09},\n",
-       " {'text': 'you', 'start': 839.88, 'duration': 5.04},\n",
-       " {'text': \"feel sexually shameful, that's\", 'start': 842.04, 'duration': 5.7},\n",
-       " {'text': \"sexual harassment. In Korea, that's the\",\n",
-       "  'start': 847.74,\n",
-       "  'duration': 5.06},\n",
-       " {'text': \"case.  It's\", 'start': 853.399, 'duration': 6.641},\n",
-       " {'text': 'scary to lose everything and become something really different only from the genitals',\n",
-       "  'start': 864.36,\n",
-       "  'duration': 5.59},\n",
-       " {'text': '[Music]', 'start': 864.95, 'duration': 5.0},\n",
-       " {'text': '[Music]', 'start': 888.91, 'duration': 3.09}]"
-      ]
-     },
-     "execution_count": 42,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "raw"
+    "import nltk\n",
+    "import re\n",
+    "import os"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Convert the list of dictionaries to a pandas dataframe\n",
-    "df = pd.DataFrame(raw)\n",
+    "def set_openai_key(key):\n",
+    "    if key == \"env\":\n",
+    "        key = os.environ.get(\"OPENAI_API_KEY\")\n",
+    "    openai.api_key = key\n",
     "\n",
-    "# Add end column\n",
-    "df['end'] = df['start'] + df['duration']\n",
+    "def get_youtube_data(url):\n",
     "\n",
-    "# Add a new column to the dataframe called 'total_words' that contains the total number of words so far in the transcript\n",
-    "df['total_words'] = df['text'].apply(lambda x: len(x.split())).cumsum()\n",
+    "    video_id = url.split(\"=\")[1]\n",
     "\n",
-    "# Add \"\\n\\n\" at the end of df[\"text\"]\n",
-    "df[\"text\"] = df[\"text\"] + \"\\n\\n\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Merge the text column into a single string and save to a transcript variable\n",
+    "    try:\n",
+    "        raw = YouTubeTranscriptApi.get_transcript(video_id)\n",
+    "    except:\n",
+    "        try:\n",
+    "            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)\n",
+    "            for transcript in transcript_list:\n",
+    "                raw = transcript.translate('en').fetch()\n",
+    "                break\n",
+    "        except:\n",
+    "            print(f\"No transcript found for {url}\") # Usually because the video itself disabled captions\n",
+    "            return False\n",
     "\n",
-    "transcript = df['text'].str.cat(sep=' ')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 45,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "\"[Music]\\n\\n [Music]\\n\\n [Applause]\\n\\n [Music]\\n\\n [Laughter] [Music] [Music] [\\n\\n Applause]\\n\\n [Music]\\n\\n Recently, the\\n\\n issue of sexual assault by celebrities has not stopped.  It's\\n\\n true that it happened to me. It's\\n\\n reallyembarrassing\\n\\n [Music] It's disastrous\\n\\n [Music]\\n\\n There's also a part where feminists are obsessed with men's genitals. I think their germinating\\n\\n power is a really scary part.  I\\n\\n think this castration will happen more often as they get castrated.\\n\\n In fact, feminism was popular at the time,\\n\\n but thanks to its popularity,\\n\\n a lot of\\n\\n things happened, such as scolding, ridicule, insults, and\\n\\n insults against\\n\\n men.  I\\n\\n just couldn't stay there. Well, the\\n\\n pepper is 3 cm.\\n\\n Besides, all men are potential rape\\n\\n criminals. Men are useless. Men stopped\\n\\n trusting women.  If you\\n\\n reach there, you may be hit by the #MeToo movement, so I\\n\\n think there are a lot of them right now. I think\\n\\n there may be a little more than in other countries.\\n\\n [Applause]\\n\\n [Music]\\n\\n Personally, I\\n\\n would say it's content that only has these emotions. As an\\n\\n example, I said that we need to strongly pass the anti-discrimination law, but\\n\\n this is actually an expression\\n\\n dictatorship class.  Guys\\n\\n [Applause]\\n\\n [Music] I think you\\n\\n 're talking a lot, but I want to hear it. Oh, you're not going to listen. You're so mean.\\n\\n For example, I went to the bathroom, and I\\n\\n saw that it was unisex, but I want it to be safe with the door open.  I\\n\\n think I was able to see well how men and women cut off harmony at the source. My\\n\\n girlfriend is\\n\\n not satisfied with something like that.\\n\\n Could anyone who really thought about gender equality talk about peeing and taking a shower?  I think\\n\\n [Music]\\n\\n In fact, many companies are\\n\\n paying for it,\\n\\n or in the labor market, there\\n\\n is employment discrimination without any reason for gender inequality.\\n\\n Evil men must disappear\\n\\n Because of these millitons, there are many such people\\n\\n in Korean society, and Ha Tae-kyung is a\\n\\n representative abolitionist.  It\\n\\n means that I oppose giving privileges to Han Gender.\\n\\n Korean feminists are now\\n\\n understood as\\n\\n discriminatory against men.  In such a society,\\n\\n I\\n\\n think we should pay more attention to the serious discrimination against women.\\n\\n Recently,\\n\\n when I look at the political situation in Korea, I feel that it is now retreating.\\n\\n 19% of female lawmakers in the National Assembly\\n\\n are now 19%.\\n\\n Why do people who send me messages like this send me messages like this\\n\\n when they come every day?  It\\n\\n 's necessary, but it seems that there\\n\\n are many cases where the target is directed at women,\\n\\n but if there's a motto that I\\n\\n personally take while leading this group, let's\\n\\n create a world where feminists don't have to choose feminism.  I\\n\\n choose\\n\\n feminism because I think you're watching.\\n\\n As a person, I live to protect the woman I love.\\n\\n I think I'm about the level of a director who creates a hero.\\n\\n Well,\\n\\n one day,\\n\\n [Music] We were\\n\\n humiliated like\\n\\n this.  I\\n\\n think there are so many messages in this very short video. First of all, I think there's\\n\\n enough room for it to be interpreted as\\n\\n sexual\\n\\n harassment.\\n\\n But I really\\n\\n had no intention of interfering with the event.\\n\\n It was a chance to share, but I\\n\\n think I couldn't give you a good answer after hearing that conversation.\\n\\n [Music] I was\\n\\n very surprised. I\\n\\n went with my faith,\\n\\n but\\n\\n I felt completely betrayed.  It's\\n\\n just\\n\\n [Music] [\\n\\n Music] If\\n\\n you\\n\\n feel sexually shameful, that's\\n\\n sexual harassment. In Korea, that's the\\n\\n case.  It's\\n\\n scary to lose everything and become something really different only from the genitals\\n\\n [Music]\\n\\n [Music]\\n\\n\""
-      ]
-     },
-     "execution_count": 45,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "transcript"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tt = TextTilingTokenizer()\n",
+    "    response = requests.get(f\"https://noembed.com/embed?dataType=json&url={url}\")\n",
+    "    data = json.loads(response.content)\n",
     "\n",
-    "# Tokenize the transcript into segments using the TextTilingTokenizer\n",
-    "segments = tt.tokenize(transcript)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# # Remove \\n\\n from each segment\n",
-    "segments = [segment.replace('\\n\\n','').strip() for segment in segments]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Calculate a list of word count for each segment\n",
-    "segments_wc = [len(segment.split()) for segment in segments]\n",
+    "    title, author = data[\"title\"], data[\"author_name\"]\n",
+    "\n",
+    "    # Remove any \"'\" from title\n",
+    "    title = title.replace(\"'\", \"\")\n",
+    "    author = author.replace(\"'\", \"\")\n",
+    "\n",
+    "    df = pd.DataFrame(raw)\n",
+    "\n",
+    "    df['end'] = df['start'] + df['duration']\n",
+    "    df['total_words'] = df['text'].apply(lambda x: len(x.split())).cumsum()\n",
+    "    df[\"text\"] = df[\"text\"] + \"\\n\\n\"\n",
+    "\n",
+    "    return df, title, author\n",
     "\n",
-    "# Make it cumulative\n",
-    "segments_wc = np.cumsum(segments_wc)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [],
-   "source": [
     "def to_timestamp(seconds):\n",
     "    seconds = int(seconds)\n",
     "\n",
@@ -358,175 +76,321 @@
     "    if seconds >= 3600:\n",
     "        return f\"{hours:02d}:{minutes:02d}:{seconds_remaining:02d}\"\n",
     "    else:\n",
-    "        return f\"{minutes:02d}:{seconds_remaining:02d}\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'01:40'"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "to_timestamp(100)"
+    "        return f\"{minutes:02d}:{seconds_remaining:02d}\"\n",
+    "\n",
+    "def to_seconds(timestamp):\n",
+    "    time_list = timestamp.split(':')\n",
+    "    total_seconds = 0\n",
+    "    if len(time_list) == 2:  # Minutes:Seconds format\n",
+    "        total_seconds = int(time_list[0]) * 60 + int(time_list[1])\n",
+    "    elif len(time_list) == 3:  # Hours:Minutes:Seconds format\n",
+    "        total_seconds = int(time_list[0]) * 3600 + int(time_list[1]) * 60 + int(time_list[2])\n",
+    "    else:\n",
+    "        raise ValueError(\"Invalid timestamp format\")\n",
+    "    return total_seconds\n",
+    "\n",
+    "def get_segments(df, title, author, split_by_topic, segment_length = 200):\n",
+    "\n",
+    "    transcript = df['text'].str.cat(sep=' ')\n",
+    "\n",
+    "    if not split_by_topic:\n",
+    "        words = transcript.split()\n",
+    "        segments = [' '.join(words[i:i+segment_length]) for i in range(0, len(words), segment_length)]\n",
+    "    else:\n",
+    "        try:\n",
+    "            segments = tt.tokenize(transcript)\n",
+    "        except:\n",
+    "            return \"\"\n",
+    "\n",
+    "    segments = [segment.replace('\\n','').strip() for segment in segments]\n",
+    "\n",
+    "    segments_wc = [len(segment.split()) for segment in segments]\n",
+    "    segments_wc = np.cumsum(segments_wc)\n",
+    "\n",
+    "    idx = [np.argmin(np.abs(df['total_words'] - total_words)) for total_words in segments_wc]\n",
+    "\n",
+    "    segments_end_times = df['end'].iloc[idx].values\n",
+    "    segments_end_times = np.insert(segments_end_times, 0, 0.0)\n",
+    "\n",
+    "    segments_times = [f\"({to_timestamp(segments_end_times[i-1])}, {to_timestamp(segments_end_times[i])})\" for i in range(1,len(segments_end_times))]\n",
+    "\n",
+    "    segments_text = [f\"Segment from '{title}' by {author}\\nTimestamp: {segment_time}\\n\\n{segment}\\n\" for segment, segment_time in zip(segments, segments_times)]\n",
+    "\n",
+    "    return segments_text\n",
+    "\n",
+    "def fit_searcher(segments, n_neighbours):\n",
+    "    global searcher\n",
+    "    searcher.fit(segments, n_neighbors=n_neighbours)\n",
+    "    return True\n",
+    "\n",
+    "def num_tokens(text, model):\n",
+    "    encoding = tiktoken.encoding_for_model(model)\n",
+    "    return len(encoding.encode(text))\n",
+    "\n",
+    "def refencify(text):\n",
+    "    title_pattern = r\"Segment from '(.+)'\"\n",
+    "    timestamp_pattern = r\"Timestamp: \\((.+)\\)\"\n",
+    "\n",
+    "    print(text)\n",
+    "\n",
+    "    title = re.search(title_pattern, text).group(1)\n",
+    "    timestamp = re.search(timestamp_pattern, text).group(1).split(\",\")\n",
+    "    start_timestamp, end_timestamp = timestamp\n",
+    "\n",
+    "    url = titles_to_urls[title]\n",
+    "    start_seconds = to_seconds(start_timestamp)\n",
+    "    end_seconds = to_seconds(end_timestamp)\n",
+    "\n",
+    "    video_iframe = f'''<iframe\n",
+    "    width=\"400\"\n",
+    "    height=\"240\"\n",
+    "    src=\"{url.replace(\"watch?v=\", \"embed/\")}?start={start_seconds}&end={end_seconds}&controls=0\"\n",
+    "    frameborder=\"0\"\n",
+    "    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\n",
+    "    allowfullscreen\n",
+    "    >\n",
+    "    </iframe>'''\n",
+    "\n",
+    "    return start_timestamp, end_timestamp, f\"{video_iframe}\\n\\n\"\n",
+    "\n",
+    "def form_query(question, model, token_budget):\n",
+    "\n",
+    "    results = searcher(question)\n",
+    "\n",
+    "    introduction = 'Use the below segments from multiple youtube videos to answer the subsequent question. If the answer cannot be found in the articles, write \"I could not find an answer.\" Cite each sentence using the [title, author, timestamp] notation. Every sentence MUST have a citation!'\n",
+    "\n",
+    "    message = introduction\n",
+    "\n",
+    "    question = f\"\\n\\nQuestion: {question}\"\n",
+    "\n",
+    "    references = \"\"\n",
+    "\n",
+    "    for i, result in enumerate(results):\n",
+    "        result = result + \"\\n\\n\"\n",
+    "        if (\n",
+    "            num_tokens(message + result + question, model=model)\n",
+    "            > token_budget\n",
+    "        ):\n",
+    "            break\n",
+    "        else:\n",
+    "            message += result\n",
+    "            start_timestamp, end_timestamp, iframe = refencify(result)\n",
+    "            references += f\"### Segment {i+1} ({start_timestamp} - {end_timestamp}):\\n\" + iframe\n",
+    "\n",
+    "    # Remove the last extra two newlines\n",
+    "    message = message[:-2]\n",
+    "\n",
+    "    references = \"Segments that might have been used to answer your question: (If you specified more segments than shown here, consider increasing your token budget)\\n\\n\" + references\n",
+    "\n",
+    "    return message + question, references\n",
+    "\n",
+    "def generate_answer(question, model, token_budget, temperature):\n",
+    "    \n",
+    "    message, references = form_query(question, model, token_budget)\n",
+    "\n",
+    "    messages = [\n",
+    "        {\"role\": \"system\", \"content\": \"You answer questions about YouTube videos.\"},\n",
+    "        {\"role\": \"user\", \"content\": message},\n",
+    "    ]\n",
+    "\n",
+    "    response = openai.ChatCompletion.create(\n",
+    "        model=model,\n",
+    "        messages=messages,\n",
+    "        temperature=temperature\n",
+    "    )\n",
+    "    \n",
+    "    response_message = response[\"choices\"][0][\"message\"][\"content\"]\n",
+    "\n",
+    "    return response_message, references\n",
+    "\n",
+    "def add_to_dict(title, url):\n",
+    "    global title_counter\n",
+    "\n",
+    "    if title not in titles_to_urls:\n",
+    "        # This is the first occurrence of this title\n",
+    "        titles_to_urls[title] = url\n",
+    "        return title\n",
+    "    else:\n",
+    "        # This title has already been seen, so we need to add a number suffix to it\n",
+    "        # First, check if we've already seen this title before\n",
+    "        if title in title_counter:\n",
+    "            # If we have, increment the counter\n",
+    "            title_counter[title] += 1\n",
+    "        else:\n",
+    "            # If we haven't, start the counter at 1\n",
+    "            title_counter[title] = 1\n",
+    "        \n",
+    "        # Add the suffix to the title\n",
+    "        new_title = f\"{title} ({title_counter[title]})\"\n",
+    "        \n",
+    "        # Add the new title to the dictionary\n",
+    "        titles_to_urls[new_title] = url\n",
+    "        return new_title\n",
+    "\n",
+    "def search_youtube(question, n_videos):\n",
+    "    videosSearch = VideosSearch(question, limit = n_videos)\n",
+    "    urls = [\"https://www.youtube.com/watch?v=\" + video[\"id\"] for video in videosSearch.result()[\"result\"]]\n",
+    "    print(urls)\n",
+    "    return urls"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# For each value in segments_wc, get the index of the closest value in df['total_words']\n",
-    "# This will be the index of the row in df that is closest to the end of each segment\n",
-    "idx = [np.argmin(np.abs(df['total_words'] - total_words)) for total_words in segments_wc]\n",
+    "def main(openAI_key, question, n_videos, urls_text, split_by_topic, segment_length, n_neighbours, model, token_budget, temperature):\n",
     "\n",
-    "# Get segment end times from idx\n",
-    "segment_end_times = df['end'].iloc[idx].values\n",
+    "    print(question)\n",
+    "    print(urls_text)\n",
+    "\n",
+    "    set_openai_key(openAI_key)\n",
+    "\n",
+    "    if urls_text == \"\":\n",
+    "        urls = search_youtube(question, n_videos)\n",
+    "    else:\n",
+    "        urls = list(set(urls_text.split(\"\\n\")))\n",
     "\n",
-    "# Add 0.0 to the beginning of segment_end_times\n",
-    "segment_end_times = np.insert(segment_end_times, 0, 0.0)\n",
+    "    global titles_to_urls\n",
+    "    titles_to_urls = {}\n",
     "\n",
-    "# segment_times is a list of tuples containing the start and end times of each segment\n",
-    "segment_times = [(to_timestamp(segment_end_times[i-1]), to_timestamp(segment_end_times[i])) for i in range(1,len(segment_end_times))]"
+    "    segments = []\n",
+    "\n",
+    "    for url in urls:\n",
+    "\n",
+    "        if \"youtu.be\" in url:\n",
+    "            url = url.replace(\"youtu.be/\", \"youtube.com/watch?v=\")\n",
+    "\n",
+    "        res = get_youtube_data(url)\n",
+    "\n",
+    "        if not res:\n",
+    "            continue\n",
+    "\n",
+    "        df, title, author = res\n",
+    "        \n",
+    "        title = add_to_dict(title, url)\n",
+    "\n",
+    "        video_segments = get_segments(df, title, author, split_by_topic, segment_length)\n",
+    "\n",
+    "        segments.extend(video_segments)\n",
+    "    \n",
+    "    if segments == []:\n",
+    "        return \"Something wrong happened! Try specifying the YouTube videos or changing the query.\", \"\"\n",
+    "\n",
+    "    print(\"Segments generated successfully!\")\n",
+    "\n",
+    "    if fit_searcher(segments, n_neighbours):\n",
+    "        print(\"Searcher fit successfully!\")\n",
+    "        answer, references = generate_answer(question, model, token_budget, temperature)\n",
+    "\n",
+    "    print(answer)\n",
+    "\n",
+    "    return answer, references"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 20,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package stopwords to\n",
+      "[nltk_data]     C:\\Users\\andrew\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package stopwords is already up-to-date!\n"
+     ]
+    }
+   ],
    "source": [
-    "# At the beginning of each segment, add the title, author, and segment times\n",
-    "segments_times = [f\"({to_timestamp(segment_end_times[i-1])}, {to_timestamp(segment_end_times[i])})\" for i in range(1,len(segment_end_times))]"
+    "nltk.download('stopwords')\n",
+    "tt = TextTilingTokenizer()\n",
+    "searcher = SemanticSearch()\n",
+    "\n",
+    "# Initialize a counter for duplicate titles\n",
+    "title_counter = {}\n",
+    "\n",
+    "# One to one mapping from titles to urls\n",
+    "titles_to_urls = {}"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "How much money do youtubers make?\n",
+      "\n",
+      "['https://www.youtube.com/watch?v=G4qOiwIE_o0', 'https://www.youtube.com/watch?v=ABG-1iy07NM', 'https://www.youtube.com/watch?v=-cz4in1WY-o', 'https://www.youtube.com/watch?v=KA5FmevAzTc', 'https://www.youtube.com/watch?v=JZ7GHzkBvzs']\n",
+      "Segments generated successfully!\n",
+      "Fitting with n=5...\n",
+      "Searcher fit successfully!\n",
+      "Segment from 'I Asked YouTube Millionaires How Much They Make' by Jordan Welch\n",
+      "Timestamp: (00:00, 01:04)\n",
+      "\n",
+      "what's the most you've generated in a month from your YouTube channel oh man oh the best month uh I'll just say our biggest one in earnings I've always wondered how much the biggest YouTubers make so today I'm here at the most exclusive YouTube conference in the world to ask them myself we're here with my brother phidius how long did it take you to make your first dollar from YouTube Oh two years what is the most amount of money that you spent on a single video oh I spent 50 000 on a video and I made back like 25. I'm very bad in business so how much money I make I make around seventy thousand dollars per month per month around 15K that we get and 15 chaos and sold thirty thousand dollars per month congratulations bro I wish you the best when you say it I have to say that this guy that's the best AdSense in the world so he makes more money than most of the people that he asks us questions let me share how much I make in the video yeah yeah tell us maybe and then dude I love you\n",
+      "\n",
+      "\n",
+      "\n",
+      "Segment from 'HOW MUCH MONEY DO YOUTUBERS MAKE?' by MrBeast\n",
+      "Timestamp: (02:48, 03:22)\n",
+      "\n",
+      "if you see a video with 1 million views they probably made around two thousand dollars if it had you know sponsorship or a brand deal or something they probably made a couple hundred to a couple thousand depending on their size and yeah that's the video make sure you like make sure you subscribe this is an iPad I just dropped the iPad why is the outro not playing Oh Oh\n",
+      "\n",
+      "\n",
+      "\n",
+      "Segment from 'How Much YouTube Pays You For 1,000 Views In 2023' by Make Money Matt\n",
+      "Timestamp: (00:00, 00:55)\n",
+      "\n",
+      "YouTube shows ads on the videos that you upload at the beginning sometimes in the middle and at the end of your videos once you get 4 000 hours of watch time and a thousand subscribers then you can join What's called the YouTube Partner program then YouTube will pay you a cut of the revenue and that's what we're going to be talking about in this video how much you actually make now I personally run a ton of different YouTube channels and I did some research into both my own channels and other people who've uploaded videos talking about how much money they make per thousand views and the answer is that channels make between one dollar and thirty dollars per thousand views so the average Channel based on my research makes 15.50 per thousand views but with that being said most channels from my personal experience only get around five to ten dollars per thousand views that you make now in order to make more money per thousand views you can do a couple things first of all you can create a channel in what's called a high RPM Niche now RPM is how much you actually\n",
+      "\n",
+      "\n",
+      "\n",
+      "Segment from 'HOW MUCH MONEY DO YOUTUBERS MAKE?' by MrBeast\n",
+      "Timestamp: (00:00, 00:57)\n",
+      "\n",
+      "what's up guys today we're gonna be talking about what's up guys today we're gonna be talking about how much money youtubers make I see comments about YouTube money all the time and most people have no idea what they're talking about so the question I see asked the most is how much do youtubers make per view well instead of just making up a random number we're actually going to take a look at my analytics I'm going to pick one day we're gonna see how many views I got and how much money I made off those views that way you know the numbers I'm telling you are actually real so I picked the day from last month and on that day I got 240,000 views and made around six hundred and sixty-nine dollars basically if you do the math that means every thousand views I got around two dollars and seventy cents so I want you to remember that number that every thousand views is two dollars and seventy cents because that's not just a number I made up that straight out of my analytics and we're going to use that to see how much these big\n",
+      "\n",
+      "\n",
+      "\n",
+      "Segment from 'HOW MUCH MONEY DO YOUTUBERS MAKE?' by MrBeast\n",
+      "Timestamp: (00:57, 02:02)\n",
+      "\n",
+      "youtubers are making so if every youtuber made what I made per thousand views that means in the last 30 days nine youtubers have made over a million dollars in just the last 30 days PewDiePie has 13 billion billion with the BEA views and if he made two dollars and seventy cents per thousand views that means over the lifespan of his channel he's made 35 million dollars even better a channel called Ryan toy review has over 6 billion views which means he's potentially made over 16 million dollars off of reviewing toys what's up guys beast toy reviews here and today we're reviewing this toy Karambit as you'll see it's uh can I just get the 16 million dollars and guys it keeps getting better most big youtubers make an additional six figures offer brand deals and sponsorships and to prove that that's true here's the deal that I was offered an app which I'm not going to name offered me $400 to do a video on it and then an additional 50 cents for every one of you that downloaded the app basically if I had done a video on that app I would have got\n",
+      "\n",
+      "\n",
+      "\n",
+      "Youtubers can make between one dollar and thirty dollars per thousand views, with the average channel making around 15.50 dollars per thousand views. However, most channels only get around five to ten dollars per thousand views. Some big youtubers make over a million dollars in just 30 days, and PewDiePie has made 35 million dollars over the lifespan of his channel. Additionally, most big youtubers make an additional six figures from brand deals and sponsorships. [Make Money Matt, 00:00-00:55; MrBeast, 00:57-02:02]\n"
+     ]
+    },
     {
      "data": {
       "text/plain": [
-       "['(00:00, 00:48)',\n",
-       " '(00:48, 01:10)',\n",
-       " '(01:10, 01:46)',\n",
-       " '(01:46, 02:26)',\n",
-       " '(02:26, 02:57)',\n",
-       " '(02:57, 03:25)',\n",
-       " '(03:25, 04:11)',\n",
-       " '(04:11, 04:41)',\n",
-       " '(04:41, 05:26)',\n",
-       " '(05:26, 05:45)',\n",
-       " '(05:45, 06:13)',\n",
-       " '(06:13, 06:40)',\n",
-       " '(06:40, 07:02)',\n",
-       " '(07:02, 07:54)',\n",
-       " '(07:54, 08:17)',\n",
-       " '(08:17, 09:24)',\n",
-       " '(09:24, 10:10)',\n",
-       " '(10:10, 11:02)',\n",
-       " '(11:02, 11:47)',\n",
-       " '(11:47, 12:09)',\n",
-       " '(12:09, 12:52)',\n",
-       " '(12:52, 13:50)',\n",
-       " '(13:50, 14:15)',\n",
-       " '(14:15, 14:38)',\n",
-       " '(14:38, 16:14)',\n",
-       " '(16:14, 17:16)',\n",
-       " '(17:16, 17:47)',\n",
-       " '(17:47, 18:17)',\n",
-       " '(18:17, 18:56)',\n",
-       " '(18:56, 19:31)',\n",
-       " '(19:31, 19:52)',\n",
-       " '(19:52, 21:03)',\n",
-       " '(21:03, 21:39)',\n",
-       " '(21:39, 22:08)',\n",
-       " '(22:08, 22:42)',\n",
-       " '(22:42, 23:35)',\n",
-       " '(23:35, 24:51)',\n",
-       " '(24:51, 26:01)',\n",
-       " '(26:01, 26:28)',\n",
-       " '(26:28, 26:57)',\n",
-       " '(26:57, 28:37)',\n",
-       " '(28:37, 29:00)',\n",
-       " '(29:00, 29:50)',\n",
-       " '(29:50, 30:12)',\n",
-       " '(30:12, 30:55)',\n",
-       " '(30:55, 31:47)',\n",
-       " '(31:47, 32:54)',\n",
-       " '(32:54, 33:33)',\n",
-       " '(33:33, 33:50)',\n",
-       " '(33:50, 34:20)',\n",
-       " '(34:20, 34:48)',\n",
-       " '(34:48, 35:22)',\n",
-       " '(35:22, 36:14)',\n",
-       " '(36:14, 37:15)']"
+       "('Youtubers can make between one dollar and thirty dollars per thousand views, with the average channel making around 15.50 dollars per thousand views. However, most channels only get around five to ten dollars per thousand views. Some big youtubers make over a million dollars in just 30 days, and PewDiePie has made 35 million dollars over the lifespan of his channel. Additionally, most big youtubers make an additional six figures from brand deals and sponsorships. [Make Money Matt, 00:00-00:55; MrBeast, 00:57-02:02]',\n",
+       " 'Segments that might have been used to answer your question: (If you specified more segments than shown here, consider increasing your token budget)\\n\\n### Segment 1 (00:00 -  01:04):\\n<iframe\\n    width=\"400\"\\n    height=\"240\"\\n    src=\"https://www.youtube.com/embed/JZ7GHzkBvzs?start=0&end=64&controls=0\"\\n    frameborder=\"0\"\\n    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\\n    allowfullscreen\\n    >\\n    </iframe>\\n\\n### Segment 2 (02:48 -  03:22):\\n<iframe\\n    width=\"400\"\\n    height=\"240\"\\n    src=\"https://www.youtube.com/embed/G4qOiwIE_o0?start=168&end=202&controls=0\"\\n    frameborder=\"0\"\\n    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\\n    allowfullscreen\\n    >\\n    </iframe>\\n\\n### Segment 3 (00:00 -  00:55):\\n<iframe\\n    width=\"400\"\\n    height=\"240\"\\n    src=\"https://www.youtube.com/embed/KA5FmevAzTc?start=0&end=55&controls=0\"\\n    frameborder=\"0\"\\n    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\\n    allowfullscreen\\n    >\\n    </iframe>\\n\\n### Segment 4 (00:00 -  00:57):\\n<iframe\\n    width=\"400\"\\n    height=\"240\"\\n    src=\"https://www.youtube.com/embed/G4qOiwIE_o0?start=0&end=57&controls=0\"\\n    frameborder=\"0\"\\n    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\\n    allowfullscreen\\n    >\\n    </iframe>\\n\\n### Segment 5 (00:57 -  02:02):\\n<iframe\\n    width=\"400\"\\n    height=\"240\"\\n    src=\"https://www.youtube.com/embed/G4qOiwIE_o0?start=57&end=122&controls=0\"\\n    frameborder=\"0\"\\n    allow=\"accelerometer; autoplay; modestbranding; encrypted-media; gyroscope; picture-in-picture\"\\n    allowfullscreen\\n    >\\n    </iframe>\\n\\n')"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "segments_times"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 72,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "text = '''\n",
-    "Segment from 'Feminism Is 'Dividing This'' Country' by VICE News\n",
-    "Timestamp: (10:51, 12:24)\n",
-    "---\n",
-    "personally take while leading this group, let's create a world where feminists don't have to choose feminism.  I choose feminism because I think you're watching. As a person, I live to protect the woman I love. I think I'm about the level of a director who creates a hero. Well, one day, [Music] We were humiliated like this.  I think there are so many messages in this very short video. First of all, I think there's\n",
-    "---\n",
-    "'''\n",
-    "\n",
-    "# Get the title and timestamp from the text\n",
-    "import re\n",
-    "\n",
-    "# define regular expression patterns\n",
-    "title_pattern = r\"Segment from '(.+)'\"\n",
-    "timestamp_pattern = r\"Timestamp: \\((.+)\\)\"\n",
-    "\n",
-    "# search for title, source, and timestamp using regular expressions\n",
-    "title = re.search(title_pattern, text).group(1)\n",
-    "start_timestamp = re.search(timestamp_pattern, text).group(1).split(\",\")[0]\n",
-    "\n",
-    "url = f\"URL: https://www.youtube.com/watch?v={video_id}&t={start_timestamp}\"\n",
-    "\n",
-    "# Add url in text before first \"---\"\n",
-    "text = re.sub(r\"---\", f\"{url}\\n---\", text, count=1)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from youtubesearchpython import VideosSearch\n",
-    "\n",
-    "videosSearch = VideosSearch('NoCopyrightSounds', limit = 2)\n",
-    "\n",
-    "# Get URLs\n",
-    "urls = [video[\"id\"] for video in videosSearch.result()[\"result\"]]"
+    "main(openAI_key=\"env\", \n",
+    "    question=\"How much money do youtubers make?\", n_videos=5, \n",
+    "    urls_text=\"\", \n",
+    "    split_by_topic=False, \n",
+    "    segment_length=200, \n",
+    "    n_neighbours=5, \n",
+    "    model=\"gpt-3.5-turbo\", \n",
+    "    token_budget=2000, \n",
+    "    temperature=0)"
    ]
   }
  ],