{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gradio in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 1)) (5.29.0)\n",
      "Requirement already satisfied: transformers in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 2)) (4.51.3)\n",
      "Requirement already satisfied: sentence-transformers in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 3)) (4.1.0)\n",
      "Requirement already satisfied: faiss-cpu in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 4)) (1.11.0)\n",
      "Requirement already satisfied: datasets in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 5)) (3.5.1)\n",
      "Requirement already satisfied: huggingface_hub in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from -r requirements.txt (line 6)) (0.30.2)\n",
      "Requirement already satisfied: aiofiles<25.0,>=22.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (24.1.0)\n",
      "Requirement already satisfied: anyio<5.0,>=3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (4.8.0)\n",
      "Requirement already satisfied: audioop-lts<1.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.2.1)\n",
      "Requirement already satisfied: fastapi<1.0,>=0.115.2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.115.12)\n",
      "Requirement already satisfied: ffmpy in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.5.0)\n",
      "Requirement already satisfied: gradio-client==1.10.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (1.10.0)\n",
      "Requirement already satisfied: groovy~=0.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.1.2)\n",
      "Requirement already satisfied: httpx>=0.24.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.28.1)\n",
      "Requirement already satisfied: jinja2<4.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (3.1.5)\n",
      "Requirement already satisfied: markupsafe<4.0,>=2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (3.0.2)\n",
      "Requirement already satisfied: numpy<3.0,>=1.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (2.2.3)\n",
      "Requirement already satisfied: orjson~=3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (3.10.18)\n",
      "Requirement already satisfied: packaging in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (24.2)\n",
      "Requirement already satisfied: pandas<3.0,>=1.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (2.2.3)\n",
      "Requirement already satisfied: pillow<12.0,>=8.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (11.1.0)\n",
      "Requirement already satisfied: pydantic<2.12,>=2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (2.11.4)\n",
      "Requirement already satisfied: pydub in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.25.1)\n",
      "Requirement already satisfied: python-multipart>=0.0.18 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.0.20)\n",
      "Requirement already satisfied: pyyaml<7.0,>=5.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (6.0.2)\n",
      "Requirement already satisfied: ruff>=0.9.3 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.11.8)\n",
      "Requirement already satisfied: safehttpx<0.2.0,>=0.1.6 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.1.6)\n",
      "Requirement already satisfied: semantic-version~=2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (2.10.0)\n",
      "Requirement already satisfied: starlette<1.0,>=0.40.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.46.2)\n",
      "Requirement already satisfied: tomlkit<0.14.0,>=0.12.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.13.2)\n",
      "Requirement already satisfied: typer<1.0,>=0.12 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.15.3)\n",
      "Requirement already satisfied: typing-extensions~=4.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (4.12.2)\n",
      "Requirement already satisfied: uvicorn>=0.14.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio->-r requirements.txt (line 1)) (0.34.2)\n",
      "Requirement already satisfied: fsspec in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio-client==1.10.0->gradio->-r requirements.txt (line 1)) (2025.3.0)\n",
      "Requirement already satisfied: websockets<16.0,>=10.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from gradio-client==1.10.0->gradio->-r requirements.txt (line 1)) (15.0.1)\n",
      "Requirement already satisfied: idna>=2.8 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 1)) (3.10)\n",
      "Requirement already satisfied: sniffio>=1.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 1)) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 1)) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 1)) (2025.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 1)) (2025.1)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<2.12,>=2.0->gradio->-r requirements.txt (line 1)) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.33.2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<2.12,>=2.0->gradio->-r requirements.txt (line 1)) (2.33.2)\n",
      "Requirement already satisfied: typing-inspection>=0.4.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<2.12,>=2.0->gradio->-r requirements.txt (line 1)) (0.4.0)\n",
      "Requirement already satisfied: click>=8.0.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (8.1.8)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (14.0.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (3.18.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (2024.11.6)\n",
      "Requirement already satisfied: requests in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (0.21.1)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (0.5.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from transformers->-r requirements.txt (line 2)) (4.67.1)\n",
      "Requirement already satisfied: torch>=1.11.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from sentence-transformers->-r requirements.txt (line 3)) (2.7.0)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from sentence-transformers->-r requirements.txt (line 3)) (1.6.1)\n",
      "Requirement already satisfied: scipy in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from sentence-transformers->-r requirements.txt (line 3)) (1.15.2)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from datasets->-r requirements.txt (line 5)) (20.0.0)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from datasets->-r requirements.txt (line 5)) (0.3.8)\n",
      "Requirement already satisfied: xxhash in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from datasets->-r requirements.txt (line 5)) (3.5.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from datasets->-r requirements.txt (line 5)) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from datasets->-r requirements.txt (line 5)) (3.11.18)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (2.6.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.3.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (25.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.6.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (6.4.3)\n",
      "Requirement already satisfied: propcache>=0.2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (0.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.20.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from click>=8.0.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (0.4.6)\n",
      "Requirement already satisfied: certifi in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 1)) (2025.1.31)\n",
      "Requirement already satisfied: httpcore==1.* in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 1)) (1.0.7)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpcore==1.*->httpx>=0.24.1->gradio->-r requirements.txt (line 1)) (0.14.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from python-dateutil>=2.8.2->pandas<3.0,>=1.0->gradio->-r requirements.txt (line 1)) (1.17.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->transformers->-r requirements.txt (line 2)) (3.4.1)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->transformers->-r requirements.txt (line 2)) (2.3.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (2.19.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 1)) (0.1.2)\n",
      "Requirement already satisfied: sympy>=1.13.3 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from torch>=1.11.0->sentence-transformers->-r requirements.txt (line 3)) (1.14.0)\n",
      "Requirement already satisfied: networkx in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from torch>=1.11.0->sentence-transformers->-r requirements.txt (line 3)) (3.4.2)\n",
      "Requirement already satisfied: setuptools in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from torch>=1.11.0->sentence-transformers->-r requirements.txt (line 3)) (75.8.2)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from sympy>=1.13.3->torch>=1.11.0->sentence-transformers->-r requirements.txt (line 3)) (1.3.0)\n",
      "Requirement already satisfied: joblib>=1.2.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from scikit-learn->sentence-transformers->-r requirements.txt (line 3)) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from scikit-learn->sentence-transformers->-r requirements.txt (line 3)) (3.6.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': '56be85543aeaaa14008c9063', 'title': 'Beyoncé', 'context': 'Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ bee-YON-say) (born September 4, 1981) is an American singer, songwriter, record producer and actress. Born and raised in Houston, Texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny\\'s Child. Managed by her father, Mathew Knowles, the group became one of the world\\'s best-selling girl groups of all time. Their hiatus saw the release of Beyoncé\\'s debut album, Dangerously in Love (2003), which established her as a solo artist worldwide, earned five Grammy Awards and featured the Billboard Hot 100 number-one singles \"Crazy in Love\" and \"Baby Boy\".', 'question': 'When did Beyonce start becoming popular?', 'answers': {'text': ['in the late 1990s'], 'answer_start': [269]}}\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Load SQuAD v2 train split\n",
    "squad = load_dataset(\"rajpurkar/squad_v2\", split=\"train\")\n",
    "print(squad[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['text-corpus', 'question-answer']\n"
     ]
    }
   ],
   "source": [
    "from datasets import get_dataset_config_names\n",
    "\n",
    "print(get_dataset_config_names(\"rag-datasets/rag-mini-wikipedia\"))\n",
    "# → ['text-corpus', 'question-answer']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 3200 passages.\n",
      "['Uruguay (official full name in  ; pron.  , Eastern Republic of  Uruguay) is a country located in the southeastern part of South America.  It is home to 3.3 million people, of which 1.7 million live in the capital Montevideo and its metropolitan area.', 'It is bordered by Brazil to the north, by Argentina across the bank of both the Uruguay River to the west and the estuary of RÃ\\xado de la Plata to the southwest, and the South Atlantic Ocean to the southeast. It is the second smallest independent country in South America, larger only than Suriname and the French overseas department of French Guiana.']\n"
     ]
    }
   ],
   "source": [
    "# After you’ve loaded:\n",
    "wiki = load_dataset(\n",
    "    \"rag-datasets/rag-mini-wikipedia\",\n",
    "    \"text-corpus\",\n",
    "    split=\"passages\"\n",
    ")\n",
    "\n",
    "# ← Notice split=\"passages\" above, and now:\n",
    "passages = wiki[\"passage\"]   # <-- use \"passage\", not \"text\"\n",
    "print(f\"Loaded {len(passages)} passages.\")\n",
    "print(passages[:2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a3f694b820f4b8f84646a8ab2f79970",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d019bd66d9547e0adcd5454c2ae248a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa2bce666e8140ab84d82cd34cd2d960",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0/26 [00:00<?, ?files/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05e3239406214ab5b911623bb23e4ca8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00026.parquet:  77%|#######7  | 231M/298M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\victo\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\huggingface_hub\\file_download.py:144: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\victo\\.cache\\huggingface\\hub\\datasets--mandarjoshi--trivia_qa. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b563007c5002441d85da63d3ac058bab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00026.parquet:   0%|          | 0.00/290M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9240545f3981467eaecb61c675e5576b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00026.parquet:   0%|          | 0.00/444M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "520175ce1b144a2e80a117d8796809f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00004-of-00026.parquet:   0%|          | 0.00/461M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c89505462bde4fc8b8f2cc30adaa1ace",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00005-of-00026.parquet:   0%|          | 0.00/474M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80c4d01825074ab6b9d732c1a999d883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00006-of-00026.parquet:   0%|          | 0.00/404M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "982196edb22949769d93de117077b0aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00007-of-00026.parquet:   0%|          | 0.00/324M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21dd300150f04169bcaed35c005355c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00008-of-00026.parquet:   0%|          | 0.00/329M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13802139a2e14a7f85de99be0f3f8adf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00009-of-00026.parquet:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4980b7601d894899b8afd68cf9820468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00010-of-00026.parquet:   0%|          | 0.00/400M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66427936e2514530b8c0c87cfe42edb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00011-of-00026.parquet:   0%|          | 0.00/370M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ea7415c9d164cfda25b361f30eb451a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00012-of-00026.parquet:   0%|          | 0.00/341M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc28d3c2fdb545e0935f2216817768fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00013-of-00026.parquet:   0%|          | 0.00/327M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "544359bd2cca4b8dba095b89e4101252",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00014-of-00026.parquet:   0%|          | 0.00/310M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6d2d3c209af4e039da71ffd7db271ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00015-of-00026.parquet:   0%|          | 0.00/157M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9830a8a9edd84737ae1e70ad2cc9c1c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00016-of-00026.parquet:   0%|          | 0.00/136M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6100a04c5bc54a6c8992eccd2768991b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00017-of-00026.parquet:   0%|          | 0.00/159M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9d487eb0ecf49a180b73395826aa850",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00018-of-00026.parquet:   0%|          | 0.00/200M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2911d2a55c414003b41c7f1e67800b5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00019-of-00026.parquet:   0%|          | 0.00/180M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "327a41e96c3949a6b2179c701ef47797",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00020-of-00026.parquet:   0%|          | 0.00/150M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d0d01d4b973430e90c072710bd31244",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00021-of-00026.parquet:   0%|          | 0.00/153M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0defa90b28684a159873edff4e736feb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00022-of-00026.parquet:   0%|          | 0.00/147M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9d77f763db744168b330490bc69d2f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00023-of-00026.parquet:   0%|          | 0.00/157M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "254a9cffc17641089d2e89865062f95d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00024-of-00026.parquet:   0%|          | 0.00/154M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e094b5c26eae4fc9b322cc8a2f426f09",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00025-of-00026.parquet:   0%|          | 0.00/158M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b594f4942c2145688b6187650b635932",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "validation-00000-of-00004.parquet:   0%|          | 0.00/327M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "834c2d1a34bb4beca57ab588c3aa8916",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "validation-00001-of-00004.parquet:   0%|          | 0.00/296M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad55d48669044e0a8da4064ff0e2ba9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "validation-00002-of-00004.parquet:   0%|          | 0.00/184M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1985723fa1fe481db76fedf15ba44c3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "validation-00003-of-00004.parquet:   0%|          | 0.00/129M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47aff557e9494b0babf955d257da9707",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "test-00000-of-00004.parquet:   0%|          | 0.00/307M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6950c967f2764410a4e9a58e6f75eb9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "test-00001-of-00004.parquet:   0%|          | 0.00/288M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1289a1af756a44519b933cdb04e331b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "test-00002-of-00004.parquet:   0%|          | 0.00/171M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8373939a6ccd4aae8335bda2ced8e8f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "test-00003-of-00004.parquet:   0%|          | 0.00/128M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e982782830e74fbc94bf1c62359e756d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/138384 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "721b7ae4ac194096972ab4dfcf481771",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/17944 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "317e5566726c420ab680f2dc25558c1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/17210 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    }
   ],
   "source": [
    "# Load the first 1,000 examples of the validation split\n",
    "trivia_small = load_dataset(\n",
    "    \"mandarjoshi/trivia_qa\",\n",
    "    \"rc\",\n",
    "    split=\"validation[:1000]\"\n",
    ")\n",
    "print(len(trivia_small))  # 1000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "082fae0dc7c846648262320f497ead9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\victo\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\huggingface_hub\\file_download.py:144: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\victo\\.cache\\huggingface\\hub\\models--sentence-transformers--all-MiniLM-L6-v2. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "657091b30bc247bf9a0be3c9fdcfafe1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config_sentence_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d02d9fcea144f7396c8b2f8b8a30b50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/10.5k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b23b9ee4be80470a96bfe5a7293d706c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91766368d2c44fe181aedc2debbfd2c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/612 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. Falling back to regular HTTP download. For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f0e8193141a487680b8bd8f1c5d00d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c63240c2991419999ab0aa1a2852731",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/350 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "353cd71ff2c244b2b3f0ecd4f9d1a0b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1ce19c9a9b745198ffcfac143a351f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a34f08cf92234b0a9f827a0bd43bd63f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb7c74f55e134aec920d424da729bd63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bfe1a75a9824eb192b2bf2c6c9859e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated embeddings shape: (3200, 384)\n"
     ]
    }
   ],
   "source": [
    "# 1.1) Import the embedder\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "# 1.2) Instantiate your Sentence-Transformer model\n",
    "#     “all-MiniLM-L6-v2” is small & fast, good for demos\n",
    "embedder = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "# 1.3) Encode your passages list\n",
    "#     This may take a moment—use show_progress_bar=True\n",
    "embeddings = embedder.encode(\n",
    "    passages, \n",
    "    show_progress_bar=True,\n",
    "    convert_to_numpy=True\n",
    ")\n",
    "\n",
    "print(f\"Generated embeddings shape: {embeddings.shape}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the FAISS index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FAISS index contains 3200 vectors of dimension 384.\n"
     ]
    }
   ],
   "source": [
    "import faiss\n",
    "\n",
    "# 2.1) Get the embedding dimension\n",
    "dim = embeddings.shape[1]\n",
    "\n",
    "# 2.2) Create a simple L2 index\n",
    "index = faiss.IndexFlatL2(dim)\n",
    "\n",
    "# 2.3) Add all passage embeddings into the index\n",
    "index.add(embeddings)\n",
    "\n",
    "print(f\"FAISS index contains {index.ntotal} vectors of dimension {dim}.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage 1 (dist=1.38):\n",
      "Modern Finnish popular music includes a renowned heavy metal scene, in common with other Nordic countries, as well as a number of prominent rock bands, jazz musicians, hip hop performers, and dance mu …\n",
      "\n",
      "Passage 2 (dist=1.40):\n",
      "* McDonough, James Lee, Shiloh: In Hell before Night (1977).  …\n",
      "\n",
      "Passage 3 (dist=1.41):\n",
      "Sami music …\n",
      "\n",
      "Passage 4 (dist=1.42):\n",
      "The people of northern Finland, Sweden and Norway, the Sami, are known primarily for highly spiritual songs called Joik. The same word sometimes refers to lavlu or vuelie songs, though this is technic …\n",
      "\n",
      "Passage 5 (dist=1.44):\n",
      "John Adams, portrait by John Trumbull. …\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 3.1) Pick a sample query\n",
    "query = \"Who wrote the song Halo?\"\n",
    "\n",
    "# 3.2) Embed the query\n",
    "q_emb = embedder.encode([query], convert_to_numpy=True)\n",
    "\n",
    "# 3.3) Search top 5 nearest passages\n",
    "distances, indices = index.search(q_emb, k=5)\n",
    "\n",
    "# 3.4) Print out the top-5 texts\n",
    "for rank, idx in enumerate(indices[0], start=1):\n",
    "    print(f\"Passage {rank} (dist={distances[0][rank-1]:.2f}):\")\n",
    "    print(passages[idx][:200], \"…\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and add the LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "^C\n"
     ]
    }
   ],
   "source": [
    "! huggingface-cli login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb405c91a771473e80578648166b0d5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\victo\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\huggingface_hub\\file_download.py:144: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\victo\\.cache\\huggingface\\hub\\models--google--flan-t5-base. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n",
      "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. Falling back to regular HTTP download. For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97617186c019485b83d0cb38f677266e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfb09d567c6d4b77bb26b3af9e75a77d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8210d38a78634a7f9b0d76970a145ce5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e98d397ffacb4561b45821efc96a6ece",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.40k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. Falling back to regular HTTP download. For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1966d468bfa549b3938e52deae90dc39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/990M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c901ab19c3d743f7b01a32a760cdb6ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Device set to use cpu\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a slang term for a group of people who are a part of a group of people\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline\n",
    "\n",
    "# 1) Point to the Google repo, not Facebook’s\n",
    "MODEL_ID = \"google/flan-t5-base\"\n",
    "\n",
    "# 2) Load tokenizer & model (your token is used under the hood)\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n",
    "model     = AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID)\n",
    "\n",
    "# 3) Create the text2text pipeline\n",
    "qa_pipe = pipeline(\n",
    "    \"text2text-generation\",\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    device=-1   # or 0 if you have a GPU\n",
    ")\n",
    "\n",
    "# 4) Sanity check\n",
    "out = qa_pipe(\"Question: What is RAG? Answer:\", max_length=50)\n",
    "print(out[0][\"generated_text\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: huggingface_hub[hf_xet] in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (0.30.2)\n",
      "Requirement already satisfied: filelock in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (3.18.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (2025.3.0)\n",
      "Requirement already satisfied: packaging>=20.9 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (24.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (6.0.2)\n",
      "Requirement already satisfied: requests in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (2.32.3)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (4.67.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from huggingface_hub[hf_xet]) (4.12.2)\n",
      "Collecting hf-xet>=0.1.4 (from huggingface_hub[hf_xet])\n",
      "  Downloading hf_xet-1.1.0-cp37-abi3-win_amd64.whl.metadata (498 bytes)\n",
      "Requirement already satisfied: colorama in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from tqdm>=4.42.1->huggingface_hub[hf_xet]) (0.4.6)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->huggingface_hub[hf_xet]) (3.4.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->huggingface_hub[hf_xet]) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->huggingface_hub[hf_xet]) (2.3.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\victo\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from requests->huggingface_hub[hf_xet]) (2025.1.31)\n",
      "Downloading hf_xet-1.1.0-cp37-abi3-win_amd64.whl (4.2 MB)\n",
      "   ---------------------------------------- 0.0/4.2 MB ? eta -:--:--\n",
      "   -- ------------------------------------- 0.3/4.2 MB ? eta -:--:--\n",
      "   ---- ----------------------------------- 0.5/4.2 MB 1.7 MB/s eta 0:00:03\n",
      "   ------- -------------------------------- 0.8/4.2 MB 1.5 MB/s eta 0:00:03\n",
      "   ------- -------------------------------- 0.8/4.2 MB 1.5 MB/s eta 0:00:03\n",
      "   --------- ------------------------------ 1.0/4.2 MB 1.0 MB/s eta 0:00:04\n",
      "   -------------- ------------------------- 1.6/4.2 MB 1.2 MB/s eta 0:00:03\n",
      "   -------------- ------------------------- 1.6/4.2 MB 1.2 MB/s eta 0:00:03\n",
      "   ----------------- ---------------------- 1.8/4.2 MB 1.2 MB/s eta 0:00:02\n",
      "   ---------------------- ----------------- 2.4/4.2 MB 1.2 MB/s eta 0:00:02\n",
      "   ------------------------ --------------- 2.6/4.2 MB 1.3 MB/s eta 0:00:02\n",
      "   --------------------------- ------------ 2.9/4.2 MB 1.3 MB/s eta 0:00:02\n",
      "   ----------------------------- ---------- 3.1/4.2 MB 1.3 MB/s eta 0:00:01\n",
      "   -------------------------------- ------- 3.4/4.2 MB 1.3 MB/s eta 0:00:01\n",
      "   ---------------------------------- ----- 3.7/4.2 MB 1.3 MB/s eta 0:00:01\n",
      "   ------------------------------------- -- 3.9/4.2 MB 1.3 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 4.2/4.2 MB 1.3 MB/s eta 0:00:00\n",
      "Installing collected packages: hf-xet\n",
      "Successfully installed hf-xet-1.1.0\n"
     ]
    }
   ],
   "source": [
    "! pip install huggingface_hub[hf_xet]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}