added backend functionality

Dockerfile

@@ -1,9 +1,18 @@
 FROM python:3.9-slim
 WORKDIR /app
+
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 RUN echo "Installed required Python packages."
 
-COPY frontend/ ./frontend
+COPY buffalo_rag/ /app/buffalo_rag
+COPY main.py /app/main.py
+COPY data/ /app/data
+COPY start.sh /app/start.sh
+
+RUN chmod +x /app/start.sh
+
 EXPOSE 7860
-CMD ["python", "frontend/flask_app.py"]
+EXPOSE 8000
+
+CMD ["/bin/bash", "/app/start.sh"]

buffalo_rag/api/main.py

@@ -0,0 +1,161 @@
+import os
+import json
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Query
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+
+from buffalo_rag.scraper.scraper import BuffaloScraper
+from buffalo_rag.embeddings.chunker import DocumentChunker
+from buffalo_rag.vector_store.db import VectorStore
+from buffalo_rag.model.rag import BuffaloRAG
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="BuffaloRAG API",
+    description="API for BuffaloRAG - AI Assistant for International Students at University at Buffalo",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Initialize components
+vector_store = VectorStore()
+rag = BuffaloRAG(vector_store=vector_store)
+
+# Pydantic models
+class QueryRequest(BaseModel):
+    query: str
+    k: int = 5
+    categories: Optional[List[str]] = None
+
+class QueryResponse(BaseModel):
+    query: str
+    response: str
+    sources: List[Dict[str, Any]]
+    timestamp: str
+
+class ScrapeRequest(BaseModel):
+    seed_url: str = "https://www.buffalo.edu/international-student-services.html"
+    max_pages: int = 100
+
+class ScrapeResponse(BaseModel):
+    status: str
+    message: str
+
+# Background tasks
+def run_scraper(seed_url: str, max_pages: int):
+    """Run the web scraper in the background."""
+    scraper = BuffaloScraper(seed_url=seed_url)
+    scraper.scrape(max_pages=max_pages)
+    
+    # After scraping, update the embeddings and index
+    chunker = DocumentChunker()
+    chunks = chunker.create_chunks()
+    chunker.create_embeddings(chunks)
+    
+    # Reload the vector store
+    global vector_store
+    vector_store = VectorStore()
+    
+    # Update the RAG model
+    global rag
+    rag = BuffaloRAG(vector_store=vector_store)
+
+def refresh_index():
+    """Refresh the vector index in the background."""
+    chunker = DocumentChunker()
+    chunks = chunker.create_chunks()
+    chunker.create_embeddings(chunks)
+    
+    # Reload the vector store
+    global vector_store
+    vector_store = VectorStore()
+    
+    # Update the RAG model
+    global rag
+    rag = BuffaloRAG(vector_store=vector_store)
+
+# Setup static files directory
+static_dir = os.path.join(os.path.dirname(__file__), "static")
+os.makedirs(static_dir, exist_ok=True)
+
+# Add this after creating the FastAPI app
+app.mount("/static", StaticFiles(directory=static_dir), name="static")
+
+# API endpoints
+@app.post("/api/ask", response_model=QueryResponse)
+async def ask(request: QueryRequest):
+    """Ask a question to the RAG system."""
+    try:
+        response = rag.answer(
+            query=request.query,
+            k=request.k,
+            filter_categories=request.categories
+        )
+        
+        # Add timestamp
+        response['timestamp'] = datetime.now().isoformat()
+        
+        # Log the query for analytics
+        with open("data/query_log.jsonl", "a") as f:
+            f.write(json.dumps(response) + "\n")
+        
+        return response
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/scrape", response_model=ScrapeResponse)
+async def scrape(request: ScrapeRequest, background_tasks: BackgroundTasks):
+    """Trigger web scraping."""
+    try:
+        background_tasks.add_task(run_scraper, request.seed_url, request.max_pages)
+        return {
+            "status": "success",
+            "message": f"Started scraping from {request.seed_url} (max {request.max_pages} pages)"
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/refresh-index", response_model=ScrapeResponse)
+async def refresh(background_tasks: BackgroundTasks):
+    """Refresh the vector index."""
+    try:
+        background_tasks.add_task(refresh_index)
+        return {
+            "status": "success",
+            "message": "Started refreshing the vector index"
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Add a route to serve the React app
+@app.get("/", include_in_schema=False)
+async def serve_frontend():
+    return FileResponse(os.path.join(static_dir, "index.html"))
+
+@app.get("/{path:path}", include_in_schema=False)
+async def serve_frontend_paths(path: str):
+    # First check if the file exists in static directory
+    file_path = os.path.join(static_dir, path)
+    if os.path.isfile(file_path):
+        return FileResponse(file_path)
+        
+    # Otherwise, return index.html for client-side routing
+    return FileResponse(os.path.join(static_dir, "index.html"))
+
+# Run the API server
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("buffalo_rag.api.main:app", host="0.0.0.0", port=8000, reload=True) 

buffalo_rag/api/static/asset-manifest.json

@@ -0,0 +1,10 @@
+{
+  "files": {
+    "main.js": "/static/js/main.55953463.js",
+    "index.html": "/index.html",
+    "main.55953463.js.map": "/static/js/main.55953463.js.map"
+  },
+  "entrypoints": [
+    "static/js/main.55953463.js"
+  ]
+}

buffalo_rag/api/static/index.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>BuffaloRAG - UB International Student Assistant</title><link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap"><link rel="icon" href="/favicon.ico"/><script defer="defer" src="/static/js/main.55953463.js"></script></head><body><div id="root"></div></body></html>

buffalo_rag/api/static/static/js/main.55953463.js.LICENSE.txt

@@ -0,0 +1,59 @@
+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+
+/** @license React v0.20.2
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.13.1
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

buffalo_rag/embeddings/chunker.py

@@ -0,0 +1,303 @@
+import os
+import json
+import re
+from typing import List, Dict, Any, Optional
+import pickle
+from tqdm import tqdm
+
+from sentence_transformers import SentenceTransformer
+
+class DocumentChunker:
+    def __init__(self, input_dir: str = "data/raw", 
+                 output_dir: str = "data/processed",
+                 embedding_dir: str = "data/embeddings",
+                 model_name: str = "BAAI/bge-small-en-v1.5"):
+        self.input_dir = input_dir
+        self.output_dir = output_dir
+        self.embedding_dir = embedding_dir
+        
+        # Create output directories
+        os.makedirs(output_dir, exist_ok=True)
+        os.makedirs(embedding_dir, exist_ok=True)
+        
+        # Load embedding model
+        self.model = SentenceTransformer(model_name)
+    
+    def load_documents(self) -> List[Dict[str, Any]]:
+        """Load all documents from the input directory."""
+        documents = []
+        
+        for filename in os.listdir(self.input_dir):
+            if filename.endswith('.json'):
+                filepath = os.path.join(self.input_dir, filename)
+                with open(filepath, 'r') as f:
+                    document = json.load(f)
+                    documents.append(document)
+        
+        return documents
+    
+    def chunk_by_headings(self, document: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Split document into chunks based on headings."""
+        chunks = []
+        
+        # If no headings, just create a single chunk
+        if not document.get('headings'):
+            chunk = {
+                'title': document['title'],
+                'content': document['content'],
+                'url': document['url'],
+                'categories': document.get('categories', []),
+                'scraped_at': document['scraped_at'],
+                'document_type': document.get('document_type', 'webpage')
+            }
+            chunks.append(chunk)
+            return chunks
+            
+        # Process document based on headings
+        headings = sorted(document['headings'], key=lambda h: h.get('level', 6))
+        content = document['content']
+        
+        # Use headings to split content
+        current_title = document['title']
+        current_content = ""
+        content_lines = content.split('\n')
+        line_index = 0
+        
+        for heading in headings:
+            heading_text = heading['text']
+            
+            # Find the heading in the content
+            heading_found = False
+            for i in range(line_index, len(content_lines)):
+                if heading_text in content_lines[i]:
+                    # Save the previous chunk
+                    if current_content.strip():
+                        chunk = {
+                            'title': current_title,
+                            'content': current_content.strip(),
+                            'url': document['url'],
+                            'categories': document.get('categories', []),
+                            'scraped_at': document['scraped_at'],
+                            'document_type': document.get('document_type', 'webpage')
+                        }
+                        chunks.append(chunk)
+                    
+                    # Start new chunk
+                    current_title = heading_text
+                    current_content = ""
+                    line_index = i + 1
+                    heading_found = True
+                    break
+            
+            if not heading_found:
+                current_content += heading_text + "\n"
+                
+            # Add content until the next heading
+            if line_index < len(content_lines):
+                for i in range(line_index, len(content_lines)):
+                    # Check if line contains any of the upcoming headings
+                    if any(h['text'] in content_lines[i] for h in headings if h['text'] != heading_text):
+                        break
+                    current_content += content_lines[i] + "\n"
+                    line_index = i + 1
+        
+        # Add the last chunk
+        if current_content.strip():
+            chunk = {
+                'title': current_title,
+                'content': current_content.strip(),
+                'url': document['url'],
+                'categories': document.get('categories', []),
+                'scraped_at': document['scraped_at'],
+                'document_type': document.get('document_type', 'webpage')
+            }
+            chunks.append(chunk)
+        
+        return chunks
+    
+    def chunk_faqs(self, document: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Extract FAQs as individual chunks."""
+        chunks = []
+        
+        if not document.get('faqs'):
+            return chunks
+        
+        for faq in document['faqs']:
+            chunk = {
+                'title': faq['question'],
+                'content': faq['answer'],
+                'url': document['url'],
+                'categories': document.get('categories', []),
+                'scraped_at': document['scraped_at'],
+                'document_type': 'faq',
+                'question': faq['question']
+            }
+            chunks.append(chunk)
+        
+        return chunks
+    
+    def chunk_semantically(self, document: Dict[str, Any], 
+                          max_chunk_size: int = 1000,
+                          overlap: int = 100) -> List[Dict[str, Any]]:
+        """Split document into fixed-size chunks with overlap."""
+        chunks = []
+        content = document['content']
+        
+        # Skip empty content
+        if not content.strip():
+            return chunks
+        
+        # Split content by paragraphs
+        paragraphs = re.split(r'\n\s*\n', content)
+        
+        current_chunk = ""
+        current_length = 0
+        
+        for para in paragraphs:
+            para = para.strip()
+            if not para:
+                continue
+                
+            para_length = len(para)
+            
+            # If paragraph alone exceeds max size, split by sentences
+            if para_length > max_chunk_size:
+                sentences = re.split(r'(?<=[.!?])\s+', para)
+                for sentence in sentences:
+                    sentence = sentence.strip()
+                    sentence_length = len(sentence)
+                    
+                    if current_length + sentence_length <= max_chunk_size:
+                        current_chunk += sentence + " "
+                        current_length += sentence_length + 1
+                    else:
+                        # Save current chunk
+                        if current_chunk:
+                            chunk = {
+                                'title': document['title'],
+                                'content': current_chunk.strip(),
+                                'url': document['url'],
+                                'categories': document.get('categories', []),
+                                'scraped_at': document['scraped_at'],
+                                'document_type': document.get('document_type', 'webpage')
+                            }
+                            chunks.append(chunk)
+                        
+                        # Start new chunk
+                        current_chunk = sentence + " "
+                        current_length = sentence_length + 1
+            
+            # Paragraph fits within limit
+            elif current_length + para_length <= max_chunk_size:
+                current_chunk += para + "\n\n"
+                current_length += para_length + 2
+            
+            # Paragraph doesn't fit, create a new chunk
+            else:
+                # Save current chunk
+                if current_chunk:
+                    chunk = {
+                        'title': document['title'],
+                        'content': current_chunk.strip(),
+                        'url': document['url'],
+                        'categories': document.get('categories', []),
+                        'scraped_at': document['scraped_at'],
+                        'document_type': document.get('document_type', 'webpage')
+                    }
+                    chunks.append(chunk)
+                
+                # Start new chunk
+                current_chunk = para + "\n\n"
+                current_length = para_length + 2
+        
+        # Add the last chunk
+        if current_chunk:
+            chunk = {
+                'title': document['title'],
+                'content': current_chunk.strip(),
+                'url': document['url'],
+                'categories': document.get('categories', []),
+                'scraped_at': document['scraped_at'],
+                'document_type': document.get('document_type', 'webpage')
+            }
+            chunks.append(chunk)
+        
+        return chunks
+    
+    def create_chunks(self) -> List[Dict[str, Any]]:
+        """Process all documents and create chunks."""
+        all_chunks = []
+        
+        # Load documents
+        documents = self.load_documents()
+        print(f"Loaded {len(documents)} documents")
+        
+        # Process each document
+        for document in tqdm(documents, desc="Chunking documents"):
+            # FAQ chunks
+            faq_chunks = self.chunk_faqs(document)
+            all_chunks.extend(faq_chunks)
+            
+            # Heading-based chunks
+            heading_chunks = self.chunk_by_headings(document)
+            all_chunks.extend(heading_chunks)
+            
+            # Semantic chunks as fallback
+            if not heading_chunks:
+                semantic_chunks = self.chunk_semantically(document)
+                all_chunks.extend(semantic_chunks)
+        
+        # Save chunks to output directory
+        with open(os.path.join(self.output_dir, 'chunks.json'), 'w') as f:
+            json.dump(all_chunks, f, indent=2)
+        
+        print(f"Created {len(all_chunks)} chunks")
+        return all_chunks
+    
+    def create_embeddings(self, chunks: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
+        """Create embeddings for all chunks."""
+        if chunks is None:
+            # Load chunks if not provided
+            chunks_path = os.path.join(self.output_dir, 'chunks.json')
+            if os.path.exists(chunks_path):
+                with open(chunks_path, 'r') as f:
+                    chunks = json.load(f)
+            else:
+                chunks = self.create_chunks()
+        
+        # Prepare texts for embedding
+        texts = []
+        for chunk in chunks:
+            # For FAQs, combine question and answer
+            if chunk.get('document_type') == 'faq':
+                text = f"{chunk['title']} {chunk['content']}"
+            else:
+                # For regular chunks, use title and content
+                text = f"{chunk['title']} {chunk['content']}"
+            texts.append(text)
+        
+        # Create embeddings
+        print("Creating embeddings...")
+        embeddings = self.model.encode(texts, show_progress_bar=True)
+        
+        # Create mapping of chunk ID to embedding
+        embedding_map = {}
+        for i, chunk in enumerate(chunks):
+            chunk_id = f"chunk_{i}"
+            embedding_map[chunk_id] = {
+                'embedding': embeddings[i],
+                'chunk': chunk
+            }
+        
+        # Save embeddings
+        with open(os.path.join(self.embedding_dir, 'embeddings.pkl'), 'wb') as f:
+            pickle.dump(embedding_map, f)
+        
+        print(f"Created embeddings for {len(chunks)} chunks")
+        return embedding_map
+
+# Example usage
+if __name__ == "__main__":
+    chunker = DocumentChunker()
+    chunks = chunker.create_chunks()
+    embedding_map = chunker.create_embeddings(chunks) 

{frontend → buffalo_rag/frontend}/flask_app.py

@@ -35,4 +35,4 @@ def refresh_index():
     return jsonify(response.json())
 
 if __name__ == '__main__':
-    app.run(host="0.0.0.0", port=7860) 
+    app.run(host="0.0.0.0", port=7860) 

buffalo_rag/model/rag.py

@@ -0,0 +1,139 @@
+import os
+import json
+from typing import List, Dict, Any, Optional, Tuple
+
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+
+from buffalo_rag.vector_store.db import VectorStore
+
+class BuffaloRAG:
+    def __init__(self, 
+                 model_name: str = "Qwen/Qwen1.5-1.8B-Chat",
+                 vector_store: Optional[VectorStore] = None):
+        self.vector_store = vector_store or VectorStore()
+        
+        try:
+            # Load model and tokenizer
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+            self.model = AutoModelForCausalLM.from_pretrained(
+                model_name, 
+                torch_dtype=torch.float16,
+                device_map="auto",
+                trust_remote_code=True,
+                low_cpu_mem_usage=True
+            )
+            
+            # More conservative generation parameters for stability
+            self.pipe = pipeline(
+                "text-generation",
+                model=self.model,
+                tokenizer=self.tokenizer,
+                max_new_tokens=256,  # Shorter outputs for stability
+                do_sample=False,     # Use greedy decoding instead of sampling
+                pad_token_id=self.tokenizer.eos_token_id
+            )
+        except Exception as e:
+            print(f"Error loading main model: {str(e)}")
+            print("Falling back to smaller model...")
+            # Fallback to a smaller, more stable model
+            self.tokenizer = AutoTokenizer.from_pretrained("distilgpt2")
+            self.model = AutoModelForCausalLM.from_pretrained("distilgpt2")
+            self.pipe = pipeline(
+                "text-generation",
+                model=self.model,
+                tokenizer=self.tokenizer,
+                max_new_tokens=256
+            )
+    
+    def retrieve(self, 
+                query: str, 
+                k: int = 5,
+                filter_categories: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """Retrieve relevant chunks for a query."""
+        return self.vector_store.hybrid_search(query, k=k, filter_categories=filter_categories)
+    
+    def format_context(self, results: List[Dict[str, Any]]) -> str:
+        """Format retrieved results into context."""
+        context = ""
+        
+        for i, result in enumerate(results):
+            chunk = result['chunk']
+            context += f"Source {i+1}: {chunk['title']}\n"
+            context += f"URL: {chunk['url']}\n"
+            context += f"Content: {chunk['content'][:500]}...\n\n"
+        
+        return context
+    
+    def generate_response(self, query: str, context: str) -> str:
+        """Generate response using the language model with error handling."""
+        prompt = f"""You are a friendly and professional counselor for international students at the University at Buffalo. Respond to the student's query in a supportive, detailed, and well-structured manner.
+
+For your responses:
+1. Address the student respectfully and empathetically
+2. Provide clear, accurate information with specific details and steps when applicable
+3. Organize your answer with appropriate headings, bullet points, or numbered lists when helpful
+4. If the student's question is unclear or lacks essential details, ask 1-2 specific clarifying questions to better understand their situation
+5. Include relevant deadlines, contacts, or resources when appropriate
+6. Conclude with a brief encouraging statement
+7. Only answer related to international students at UB, if it's not related to international students at UB, just say "I'm sorry, I don't have information about that."
+8. Do not entertain any questions that are not related to students at UB.
+
+Question: {query}
+
+Relevant Information:
+{context}
+
+Answer:"""
+        
+        try:
+            # Generate response
+            response = self.pipe(prompt)[0]['generated_text']
+            
+            # Extract only the generated part (after the prompt)
+            generated = response[len(prompt):].strip()
+            
+            return generated
+        except Exception as e:
+            print(f"Error during generation: {str(e)}")
+            # Fallback response
+            return "I'm sorry, I encountered an issue generating a response. Please try asking your question in a different way or contact UB International Student Services directly for assistance."
+    
+    def answer(self, 
+              query: str, 
+              k: int = 5,
+              filter_categories: Optional[List[str]] = None) -> Dict[str, Any]:
+        """End-to-end RAG pipeline."""
+        # Retrieve relevant chunks
+        results = self.retrieve(query, k=k, filter_categories=filter_categories)
+        
+        # Format context
+        context = self.format_context(results)
+        
+        # Generate response
+        response = self.generate_response(query, context)
+        
+        # Return response and sources
+        return {
+            'query': query,
+            'response': response,
+            'sources': [
+                {
+                    'title': result['chunk']['title'],
+                    'url': result['chunk']['url'],
+                    'score': result.get('rerank_score', result['score'])
+                }
+                for result in results
+            ]
+        }
+
+# Example usage
+if __name__ == "__main__":
+    rag = BuffaloRAG(model_name="Qwen/Qwen1.5-1.8B-Chat")
+    response = rag.answer("How do I apply for OPT?")
+    
+    print(f"Query: {response['query']}")
+    print(f"Response: {response['response']}")
+    print("\nSources:")
+    for source in response['sources']:
+        print(f"- {source['title']} (Score: {source['score']:.4f})") 

buffalo_rag/scraper/scraper.py

@@ -0,0 +1,380 @@
+import os
+import json
+import datetime
+from typing import List, Dict, Any, Set
+from urllib.parse import urlparse, urljoin
+import re
+import time
+
+from bs4 import BeautifulSoup
+from playwright.sync_api import sync_playwright
+import trafilatura
+import pymupdf
+
+class BuffaloScraper:
+    def __init__(self, seed_url: str = "https://www.buffalo.edu/international-student-services.html",
+                 output_dir: str = "data/raw"):
+        self.seed_url = seed_url
+        self.output_dir = output_dir
+        self.visited_urls: Set[str] = set()
+        self.queue: List[str] = [seed_url]
+        self.domain = urlparse(seed_url).netloc
+        
+        # Create output directory if it doesn't exist
+        os.makedirs(output_dir, exist_ok=True)
+        
+        # Keywords to filter useless content
+        self.useless_keywords = [
+            "privacy policy", "terms of use", "cookie", "last updated",
+            "©", "copyright", "follow us", "social media",
+            "related links", "site map", "skip to content", "all rights reserved"
+        ]
+    
+    def is_valid_url(self, url: str) -> bool:
+        """Check if URL should be scraped."""
+        parsed = urlparse(url)
+        
+        # Only process buffalo.edu URLs
+        if not parsed.netloc.endswith('buffalo.edu'):
+            return False
+        
+        # Skip certain file types
+        if parsed.path.endswith(('.jpg', '.jpeg', '.png', '.gif', '.css', '.js')):
+            return False
+            
+        # Skip already visited URLs
+        if url in self.visited_urls:
+            return False
+        
+        # Skip certain patterns that are likely not content pages
+        skip_patterns = [
+            '/search', '/login', '/user', '/admin', '/cart', '/account',
+            'javascript:', 'mailto:', 'tel:', '#', 'facebook.com', 'twitter.com', 
+            'instagram.com', 'youtube.com', 'linkedin.com'
+        ]
+        if any(pattern in url.lower() for pattern in skip_patterns):
+            return False
+            
+        return True
+    
+    def is_useless_line(self, line: str) -> bool:
+        """Check if a line of text is likely useless footer/header content."""
+        line = line.lower()
+        return any(kw in line for kw in self.useless_keywords)
+    
+    def is_valid_line(self, line: str) -> bool:
+        """Check if a line is valid content."""
+        if not line or len(line.strip().split()) < 3:
+            return False
+        if self.is_useless_line(line):
+            return False
+        return True
+    
+    def is_heading_like(self, line: str) -> bool:
+        """Check if a line is likely a heading."""
+        line = line.strip()
+        word_count = len(line.split())
+        return (
+            line.isupper() and word_count <= 10 or
+            (len(line) < 100 and word_count <= 15 and line.endswith((':', '?')))
+        )
+    
+    def extract_clean_content(self, html: str) -> str:
+        """Extract clean content with smart filtering."""
+        soup = BeautifulSoup(html, "html.parser")
+        
+        # Remove unwanted tags
+        for tag in soup(["script", "style", "header", "footer", "nav", "aside"]):
+            tag.decompose()
+        
+        # Try trafilatura first as it's often better at extracting main content
+        trafilatura_content = trafilatura.extract(html, include_tables=True, 
+                                                  include_images=False,
+                                                  include_links=True, 
+                                                  output_format='txt')
+        
+        # If trafilatura fails or returns little content, use our own extraction
+        if not trafilatura_content or len(trafilatura_content) < 200:
+            raw_text = soup.get_text(separator="\n")
+            lines = raw_text.split("\n")
+            clean_lines = []
+            
+            for line in lines:
+                line = line.strip()
+                if not self.is_valid_line(line):
+                    continue
+                clean_lines.append(line)
+            
+            # Merge lines smartly
+            formatted_text = ""
+            buffer = ""
+            
+            for line in clean_lines:
+                # Treat as Heading or List Item
+                if self.is_heading_like(line) or line.startswith(("-", "*", "•")):
+                    if buffer:
+                        formatted_text += buffer.strip() + "\n\n"
+                        buffer = ""
+                    formatted_text += line.strip() + "\n"
+                else:
+                    buffer += line + " "
+            
+            if buffer:
+                formatted_text += buffer.strip() + "\n"
+            
+            return formatted_text.strip()
+        
+        return trafilatura_content
+    
+    def extract_content(self, html: str, url: str) -> Dict[str, Any]:
+        """Extract structured content from HTML."""
+        soup = BeautifulSoup(html, 'html.parser')
+        
+        # Extract title
+        title = soup.title.text.strip() if soup.title else ""
+        
+        # Get cleaned content
+        content = self.extract_clean_content(html)
+        
+        # Extract headings
+        headings = []
+        for h in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
+            headings.append({
+                'level': int(h.name[1]),
+                'text': h.get_text(strip=True)
+            })
+        
+        # Extract FAQs (common patterns in UB sites)
+        faqs = []
+        # Look for accordion elements, common FAQ containers
+        faq_containers = soup.select('.accordion, .faq, .collapse, .panel-group, .question-answer, details')
+        for container in faq_containers:
+            # Look for question/answer pairs in various formats
+            question_selectors = ['.accordion-header', '.faq-question', '.card-header', 
+                                'summary', '.question', 'dt', 'h3', 'h4', '.panel-title']
+            answer_selectors = ['.accordion-body', '.faq-answer', '.card-body', 
+                              '.answer', 'dd', '.panel-body', 'p']
+            
+            # Try to select using CSS selectors
+            questions = container.select(', '.join(question_selectors))
+            answers = container.select(', '.join(answer_selectors))
+            
+            # Match questions with answers
+            for i, q in enumerate(questions):
+                if i < len(answers):
+                    faqs.append({
+                        'question': q.get_text(strip=True),
+                        'answer': answers[i].get_text(strip=True)
+                    })
+        
+        # Also try to detect Q&A patterns in paragraphs
+        p_texts = [p.get_text(strip=True) for p in soup.find_all('p')]
+        for i, text in enumerate(p_texts):
+            if i < len(p_texts) - 1 and text.strip().endswith('?'):
+                faqs.append({
+                    'question': text,
+                    'answer': p_texts[i+1]
+                })
+        
+        # Extract important links
+        important_links = []
+        for a in soup.find_all('a', href=True):
+            link_text = a.get_text(strip=True)
+            href = a['href']
+            if link_text and any(keyword in link_text.lower() for keyword in 
+                               ['form', 'document', 'application', 'guide', 'i-20', 'opt', 'cpt']):
+                important_links.append({
+                    'text': link_text,
+                    'url': href
+                })
+        
+        # Metadata extraction from URL
+        parsed = urlparse(url)
+        path_parts = [p for p in parsed.path.strip("/").split("/") if p]
+        
+        # Try to categorize the content
+        categories = []
+        if re.search(r'\b(visa|i-20|i20|sevis|immigration)\b', content, re.I):
+            categories.append('immigration')
+        if re.search(r'\b(opt|cpt|employment|work|job|internship)\b', content, re.I):
+            categories.append('employment')
+        if re.search(r'\b(tuition|fee|payment|cost|financial)\b', content, re.I):
+            categories.append('fees')
+        if re.search(r'\b(housing|accommodation|apartment|dorm|living)\b', content, re.I):
+            categories.append('housing')
+        
+        # Build structured document
+        document = {
+            'url': url,
+            'title': title,
+            'content': content,
+            'headings': headings,
+            'faqs': faqs,
+            'important_links': important_links,
+            'categories': categories,
+            'scraped_at': datetime.datetime.now().isoformat(),
+            'path_hierarchy': path_parts,
+            'domain': parsed.netloc
+        }
+        
+        return document
+    
+    def extract_links(self, html: str, base_url: str) -> List[str]:
+        """Extract all links from the page."""
+        soup = BeautifulSoup(html, 'html.parser')
+        links = []
+        
+        for a in soup.find_all('a', href=True):
+            href = a['href']
+            # Handle relative URLs
+            full_url = urljoin(base_url, href)
+            # Normalize URL
+            full_url = full_url.split('#')[0]  # Remove fragment
+            full_url = full_url.rstrip('/')    # Remove trailing slash
+            
+            if self.is_valid_url(full_url):
+                links.append(full_url)
+                
+        return links
+    
+    def process_pdf(self, url: str) -> Dict[str, Any]:
+        """Download and extract text from PDF."""
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            page = browser.new_page()
+            
+            try:
+                page.goto(url, timeout=60000)  # 60 second timeout
+                
+                # Get the PDF as bytes
+                pdf_data = page.pdf(path=None)
+                browser.close()
+            except Exception as e:
+                browser.close()
+                print(f"Error downloading PDF {url}: {str(e)}")
+                return None
+        
+        # Create a temporary file to use with PyMuPDF
+        temp_path = os.path.join(self.output_dir, "temp.pdf")
+        with open(temp_path, "wb") as f:
+            f.write(pdf_data)
+        
+        # Extract text from PDF
+        doc = pymupdf.open(temp_path)
+        text = ""
+        for page_num in range(doc.page_count):
+            page = doc[page_num]
+            text += page.get_text()
+        doc.close()
+        
+        # Remove temporary file
+        os.remove(temp_path)
+        
+        # Extract metadata from URL
+        parsed = urlparse(url)
+        path_parts = [p for p in parsed.path.strip("/").split("/") if p]
+        filename = os.path.basename(url)
+        
+        # Categorize PDF content
+        categories = []
+        if re.search(r'\b(visa|i-20|i20|sevis|immigration)\b', text, re.I):
+            categories.append('immigration')
+        if re.search(r'\b(opt|cpt|employment|work|job|internship)\b', text, re.I):
+            categories.append('employment')
+        if re.search(r'\b(tuition|fee|payment|cost|financial)\b', text, re.I):
+            categories.append('fees')
+        if re.search(r'\b(housing|accommodation|apartment|dorm|living)\b', text, re.I):
+            categories.append('housing')
+        
+        # Build structured document
+        document = {
+            'url': url,
+            'title': filename or os.path.basename(url),
+            'content': text,
+            'document_type': 'pdf',
+            'categories': categories,
+            'scraped_at': datetime.datetime.now().isoformat(),
+            'path_hierarchy': path_parts,
+            'domain': parsed.netloc
+        }
+        
+        return document
+    
+    def scrape(self, max_pages: int = 100, max_depth: int = 4) -> None:
+        """Main scraping function."""
+        pages_scraped = 0
+        depth_map = {self.seed_url: 0}  # Track depth of each URL
+        
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            page = browser.new_page()
+            
+            while self.queue and pages_scraped < max_pages:
+                url = self.queue.pop(0)
+                current_depth = depth_map.get(url, 0)
+                
+                if current_depth > max_depth:
+                    continue
+                
+                if url in self.visited_urls:
+                    continue
+                    
+                try:
+                    print(f"Scraping: {url} (depth: {current_depth})")
+                    self.visited_urls.add(url)
+                    
+                    # Handle PDFs separately
+                    if url.lower().endswith('.pdf'):
+                        document = self.process_pdf(url)
+                        if document:
+                            # Save the document
+                            filename = f"{pages_scraped:04d}_{urlparse(url).netloc.replace('.', '_')}.json"
+                            filepath = os.path.join(self.output_dir, filename)
+                            with open(filepath, 'w') as f:
+                                json.dump(document, f, indent=2)
+                            
+                            pages_scraped += 1
+                    else:
+                        # Regular webpage
+                        try:
+                            page.goto(url, timeout=30000)  # 30 second timeout
+                            page.wait_for_load_state('networkidle', timeout=10000)  # Wait for page to load
+                            html = page.content()
+                            
+                            # Extract content
+                            document = self.extract_content(html, url)
+                            
+                            # Save the document
+                            filename = f"{pages_scraped:04d}_{urlparse(url).netloc.replace('.', '_')}.json"
+                            filepath = os.path.join(self.output_dir, filename)
+                            with open(filepath, 'w') as f:
+                                json.dump(document, f, indent=2)
+                            
+                            pages_scraped += 1
+                            
+                            # Extract links for further scraping if we haven't reached max depth
+                            if current_depth < max_depth:
+                                links = self.extract_links(html, url)
+                                for link in links:
+                                    if link not in self.visited_urls and link not in self.queue:
+                                        self.queue.append(link)
+                                        depth_map[link] = current_depth + 1
+                        except Exception as e:
+                            print(f"Error processing page {url}: {str(e)}")
+                            continue
+                    
+                except Exception as e:
+                    print(f"Error scraping {url}: {str(e)}")
+                
+                # Add a small delay to be nice to the server
+                time.sleep(1)
+            
+            browser.close()
+        
+        print(f"Scraping completed. Scraped {pages_scraped} pages.")
+
+# Example usage
+if __name__ == "__main__":
+    scraper = BuffaloScraper()
+    scraper.scrape(max_pages=100, max_depth=4)
+            

buffalo_rag/vector_store/db.py

@@ -0,0 +1,213 @@
+import os
+import pickle
+import json
+import numpy as np
+from typing import List, Dict, Any, Optional, Tuple
+import faiss
+from tqdm import tqdm
+from sentence_transformers import SentenceTransformer, CrossEncoder
+
+class VectorStore:
+    def __init__(self, 
+                 embedding_dir: str = "data/embeddings",
+                 model_name: str = "BAAI/bge-small-en-v1.5",
+                 reranker_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"):
+        self.embedding_dir = embedding_dir
+        self.index = None
+        self.chunk_ids = []
+        self.chunks = {}
+        
+        # Load embedding model
+        self.model = SentenceTransformer(model_name)
+        
+        # Load reranker model
+        self.reranker = CrossEncoder(reranker_name)
+        
+        # Load or create index
+        self.load_or_create_index()
+    
+    def load_or_create_index(self) -> None:
+        """Load existing index or create a new one."""
+        index_path = os.path.join(self.embedding_dir, 'faiss_index.pkl')
+        
+        if os.path.exists(index_path):
+            # Load existing index
+            with open(index_path, 'rb') as f:
+                data = pickle.load(f)
+                self.index = data['index']
+                self.chunk_ids = data['chunk_ids']
+                self.chunks = data['chunks']
+            print(f"Loaded existing index with {len(self.chunk_ids)} chunks")
+        else:
+            # Create new index
+            embeddings_path = os.path.join(self.embedding_dir, 'embeddings.pkl')
+            if os.path.exists(embeddings_path):
+                self.create_index()
+            else:
+                print("No embeddings found. Please run the chunker first.")
+    
+    def create_index(self) -> None:
+        """Create FAISS index from embeddings."""
+        embeddings_path = os.path.join(self.embedding_dir, 'embeddings.pkl')
+        
+        with open(embeddings_path, 'rb') as f:
+            embedding_map = pickle.load(f)
+        
+        # Extract embeddings and chunk IDs
+        chunk_ids = list(embedding_map.keys())
+        embeddings = np.array([embedding_map[chunk_id]['embedding'] for chunk_id in chunk_ids])
+        chunks = {chunk_id: embedding_map[chunk_id]['chunk'] for chunk_id in chunk_ids}
+        
+        # Create FAISS index
+        dimension = embeddings.shape[1]
+        index = faiss.IndexFlatL2(dimension)
+        index.add(embeddings.astype(np.float32))
+        
+        # Save index and metadata
+        self.index = index
+        self.chunk_ids = chunk_ids
+        self.chunks = chunks
+        
+        # Save to disk
+        with open(os.path.join(self.embedding_dir, 'faiss_index.pkl'), 'wb') as f:
+            pickle.dump({
+                'index': index,
+                'chunk_ids': chunk_ids,
+                'chunks': chunks
+            }, f)
+        
+        print(f"Created index with {len(chunk_ids)} chunks")
+    
+    def search(self, 
+              query: str, 
+              k: int = 5, 
+              filter_categories: Optional[List[str]] = None,
+              rerank: bool = True) -> List[Dict[str, Any]]:
+        """Search for relevant chunks."""
+        if self.index is None:
+            print("No index available. Please create an index first.")
+            return []
+        
+        # Create query embedding
+        query_embedding = self.model.encode([query])[0]
+        
+        # Search index
+        D, I = self.index.search(np.array([query_embedding]).astype(np.float32), min(k * 2, len(self.chunk_ids)))
+        
+        # Get results
+        results = []
+        for i, idx in enumerate(I[0]):
+            chunk_id = self.chunk_ids[idx]
+            chunk = self.chunks[chunk_id]
+            
+            # Apply category filter if specified
+            if filter_categories and not any(cat in chunk.get('categories', []) for cat in filter_categories):
+                continue
+            
+            result = {
+                'chunk_id': chunk_id,
+                'score': float(D[0][i]),
+                'chunk': chunk
+            }
+            results.append(result)
+        
+        # Rerank results if requested
+        if rerank and results:
+            # Prepare pairs for reranking
+            pairs = [(query, result['chunk']['content']) for result in results]
+            
+            # Get reranking scores
+            rerank_scores = self.reranker.predict(pairs)
+            
+            # Update scores and sort
+            for i, score in enumerate(rerank_scores):
+                results[i]['rerank_score'] = float(score)
+            
+            # Sort by rerank score
+            results = sorted(results, key=lambda x: x['rerank_score'], reverse=True)
+            
+            # Limit to k results
+            results = results[:k]
+        
+        return results
+    
+    def hybrid_search(self, 
+                     query: str, 
+                     k: int = 5,
+                     filter_categories: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """Combine dense vector search with BM25-style keyword matching."""
+        # Get vector search results
+        vector_results = self.search(query, k=k, filter_categories=filter_categories, rerank=False)
+        
+        # Simple keyword matching (simulating BM25)
+        keywords = query.lower().split()
+        
+        # Score all chunks by keyword presence
+        keyword_scores = {}
+        for chunk_id, chunk_data in self.chunks.items():
+            chunk = chunk_data
+            content = (chunk['title'] + " " + chunk['content']).lower()
+            
+            # Count keyword matches
+            score = sum(content.count(keyword) for keyword in keywords)
+            
+            # Apply category filter if specified
+            if filter_categories and not any(cat in chunk.get('categories', []) for cat in filter_categories):
+                continue
+                
+            keyword_scores[chunk_id] = score
+        
+        # Get top keyword matches
+        keyword_results = sorted(
+            [{'chunk_id': chunk_id, 'score': score, 'chunk': self.chunks[chunk_id]} 
+             for chunk_id, score in keyword_scores.items() if score > 0],
+            key=lambda x: x['score'], 
+            reverse=True
+        )[:k]
+        
+        # Combine results (remove duplicates)
+        seen_ids = set()
+        combined_results = []
+        
+        # Add vector results first
+        for result in vector_results:
+            combined_results.append(result)
+            seen_ids.add(result['chunk_id'])
+        
+        # Add keyword results if not already added
+        for result in keyword_results:
+            if result['chunk_id'] not in seen_ids:
+                combined_results.append(result)
+                seen_ids.add(result['chunk_id'])
+        
+        # Limit to k results
+        combined_results = combined_results[:k]
+        
+        # Rerank final results
+        if combined_results:
+            # Prepare pairs for reranking
+            pairs = [(query, result['chunk']['content']) for result in combined_results]
+            
+            # Get reranking scores
+            rerank_scores = self.reranker.predict(pairs)
+            
+            # Update scores and sort
+            for i, score in enumerate(rerank_scores):
+                combined_results[i]['rerank_score'] = float(score)
+            
+            # Sort by rerank score
+            combined_results = sorted(combined_results, key=lambda x: x['rerank_score'], reverse=True)
+        
+        return combined_results
+
+# Example usage
+if __name__ == "__main__":
+    vector_store = VectorStore()
+    results = vector_store.hybrid_search("How do I apply for OPT?")
+    
+    print(f"Found {len(results)} results")
+    for i, result in enumerate(results[:3]):
+        print(f"Result {i+1}: {result['chunk']['title']}")
+        print(f"Score: {result.get('rerank_score', result['score'])}")
+        print(f"Content: {result['chunk']['content'][:100]}...")
+        print() 

data/query_log.jsonl

@@ -0,0 +1,18 @@
+{"query": "hey", "response": "Hello! How can I assist you today?", "sources": [{"title": "E", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -7.515041351318359}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events.html", "score": -8.835952758789062}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -8.835952758789062}, {"title": "Activities", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -8.88674259185791}, {"title": "Site Map", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -10.0355224609375}], "timestamp": "2025-04-25T02:44:00.868275"}
+{"query": "hey give me details", "response": "Hello! Here are some detailed information about the International Student Services (ISS) at the University at Buffalo:\n\n1. Activities:\nThe ISS offers various activities and events to help new international students settle into their academic and social lives on campus. Some of these include:\n\na. Welcome Orientation: This event is held in the fall semester and provides an overview of the university's policies, procedures, and resources for international students. It includes information on housing, immigration, cultural differences, and academic support services.\nb. Cultural Events: The ISS hosts several cultural events throughout the year, such as food festivals, language exchange programs, and cultural workshops. These events provide opportunities for international students to learn about American culture, traditions, and customs.\nc. Social Activities: The ISS organizes social events like club meetings, study groups, and sports teams to foster a sense of community among international students. These events offer a chance to meet other students from different countries and make friends.\nd. Academic Support: The ISS provides academic support services to help international students navigate the challenges of studying in the United States. These services include tutoring, writing assistance, and access to academic resources such as databases and online libraries.\ne. Career Development: The ISS offers career development workshops and resources to help international students explore potential", "sources": [{"title": "Activities", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -9.338510513305664}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events.html", "score": -9.429910659790039}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -9.429910659790039}, {"title": "E", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -9.850290298461914}, {"title": "Monday", "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html", "score": -11.325536727905273}], "timestamp": "2025-04-25T02:47:32.093997"}
+{"query": "please give me more details about what i need to get for visa interview", "response": "To prepare for a visa interview at a U.S. Consulate, you will need to gather the following documents:\n\n1. Original documents:\n\t* Passport: Make sure it is valid for at least six months beyond your intended stay in the United States. You will need both an original and a photocopy of your passport.\n\t* I-94 card (if applicable): This is a digital copy of your current entry stamp in your passport. It serves as proof of your status in the United States and allows you to apply for a visa.\n\t* Original I-20 form (F-1 or J-1): This is a document issued by your educational institution indicating your acceptance into a degree program or research fellowship. It includes your student's name, date of admission, course schedule, tuition fees, and other relevant information.\n\t* Proof of financial support: If you are a F-1 student, you will need to provide proof of sufficient funds to cover your living expenses during your stay in the United States. This could include bank statements, pay stubs, or a letter from your employer stating your expected salary and benefits.\n\t* Proof of English proficiency: If your field of study requires a certain level of English proficiency, you will need to provide evidence", "sources": [{"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": 2.1697254180908203}, {"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -2.3090591430664062}, {"title": "Your I-20 & Next Steps - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/your-i-20---next-steps.html", "score": -2.3359806537628174}, {"title": "Visa Information - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/immigration-and-visas.html", "score": -3.3146982192993164}, {"title": "2. Immigration Documents (Original Documents + One Set of Photocopies)", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -8.720430374145508}], "timestamp": "2025-04-25T02:52:04.124952"}
+{"query": "Hey what document do I need for visa internview?", "response": "The relevant document for visa internship at the University at Buffalo is an F-1 or J-1 visa. This type of visa allows international students to work in the United States on a temporary basis under the auspices of a university or educational institution. To obtain an F-1 visa, you will need to provide certain documentation, including:\n\n1. Original passport: A valid passport that has been issued within the past six years.\n2. Valid I-94 card (if issued): An electronic copy of your current I-94 card, which shows your status as an F-1 student.\n3. Proof of financial support: A letter from your university or educational institution indicating that you have sufficient funds to cover your tuition, living expenses, and any other necessary costs for the duration of your internship.\n4. Proof of English proficiency: A TOEFL score of at least 80 or aIELTS score of 6.0 (with no writing component).\n5. Academic transcripts: A copy of your academic transcript showing your enrollment status and the courses you completed during your time in the United States.\n6. Proof of work authorization: A letter from your employer confirming that they have authorized you to work in the United States during your internship.\n7. Medical certificate", "sources": [{"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -3.452972888946533}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -3.5396194458007812}, {"title": "Visa Information - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/immigration-and-visas.html", "score": -3.560807228088379}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -3.8072965145111084}, {"title": "2. Immigration Documents (Original Documents + One Set of Photocopies)", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -9.123757362365723}], "timestamp": "2025-04-25T03:13:39.929645"}
+{"query": "Aslo, i'm a graduate student, and I have C in one subject will I fail in subject? or will my status be taken from me? give me some suggesting if i'm in safe zone, I'm an internatinal student", "response": "In general, if you are a graduate student with a C in one subject and are currently in F-1 status, you should expect to pass your subject exam and maintain your status throughout your studies. However, there are certain circumstances where you may face challenges in passing the subject exam:\n\n1. Changing degree level: If you have changed your degree level at UB or transferred from another U.S. college/university, you may need to complete additional coursework or take additional exams to demonstrate proficiency in the new subject. This could impact your ability to receive CPT authorization.\n2. Graduation requirements: If you are completing your final semester of studies and are planning to graduate within the next academic year, you may need to pass your subject exam before you can officially graduate. Failure to pass the exam could result in a delay in graduation or a reduction in your degree status.\n3. Course requirements: Some departments may require specific courses or projects to fulfill their internship, practicum, Co-op, or course requirements. If your department does not offer an internship, practicum, Co-op, or course meeting these requirements, you may not be eligible for CPT. This could limit your opportunities for practical training experiences and potentially impact your ability to receive CPT authorization.\n4. Academic integrity concerns", "sources": [{"title": "Eligibility Requirements", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html", "score": -5.543703556060791}, {"title": "F-1 Student - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student.html", "score": -8.566695213317871}, {"title": "Important Considerations:", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/estimated-expenses.html", "score": -8.704761505126953}, {"title": "Important Reminders", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html", "score": -8.846111297607422}, {"title": "Enrollment Requirements", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html", "score": -9.886970520019531}], "timestamp": "2025-04-25T03:15:30.018764"}
+{"query": "How do I apply for OPT?", "response": "To apply for Optional Practical Training (OPT) at the University at Buffalo (UB), follow these steps:\n\n1. Review the requirements: Before starting the process, make sure you understand the eligibility criteria for OPT, including the length of time allowed for the program, the type of training, and the number of hours worked per week. The maximum duration of OPT varies depending on the individual's country of origin, but it typically ranges from 12 to 16 months.\n\n2. Obtain necessary documentation: You will need to provide several documents to support your request for OPT, including:\n\n   a. F-1 student visa: A valid visa, Form I-901, must be present when applying for OPT.\n   \n   b. Proof of employment: An employer letter stating that you have been hired for an occupation related to your field of study or research, along with a copy of your job offer letter.\n   \n   c. Proof of income: A pay stub or bank statement showing your current income, which supports your ability to meet the training costs.\n   \n   d. Proof of English proficiency: A TOEFL score (minimum 80 for non-native speakers) or IELTS score (minimum 7.0 for native speakers) can demonstrate your English", "sources": [{"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -0.36291322112083435}, {"title": "Students, Faculty and Staff", "url": "https://ubisss.buffalo.edu", "score": -2.3324174880981445}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -4.302582740783691}, {"title": "OPT STEM Extension", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events/workshops.html", "score": -5.674208164215088}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -7.419798851013184}], "timestamp": "2025-04-25T03:38:42.829977"}
+{"query": "hey, help mw out with my grades, i have a grade C in one subject, will i get fal or deported, what is the requirement for international students", "response": "Hello there! I'm happy to assist you with your grades and concerns regarding your F-1 visa status. Let's start by addressing your question:\n\nQ: Will I receive a failing grade or be deported if I have a grade C in one subject?\n\nA: The eligibility requirements for F-1 student visa holders vary depending on the country of origin and the specific program or course you are participating in at UB. However, generally speaking, a grade C in one subject does not necessarily mean that you will receive a failing grade or be deported.\n\nF-1 students are typically allowed to participate in certain programs or courses that require a certain level of proficiency in English, such as those offered by the University at Buffalo's School of Education or the College of Arts & Sciences. These courses often include language classes, cultural exchange programs, or research opportunities that require a certain level of understanding of the language and culture.\n\nIf you have a grade C in one subject, it could indicate that you have demonstrated some difficulty in understanding or communicating effectively in English, which may limit your ability to fully engage in the course material and complete assignments. This could potentially impact your overall performance and potential for academic success.\n\nHowever, it's important to note that the F-1 visa process is complex, and the", "sources": [{"title": "Eligibility Requirements", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html", "score": -9.177477836608887}, {"title": "A Message of Support - April 2025 - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel/travel-alerts/a-message-of-support-for-international-students---april-2025.html", "score": -9.72881031036377}, {"title": "F-1 Student - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student.html", "score": -9.94715404510498}, {"title": "Important Reminders", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html", "score": -10.463960647583008}, {"title": "Annual Report - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/about-us/annualreport.html", "score": -10.93886947631836}], "timestamp": "2025-04-25T03:50:11.064181"}
+{"query": "fuck you help me", "response": "Hello there! I'm here to assist you with any concerns or questions you may have regarding international students at the University at Buffalo (UB). Let's start by addressing your request for help with \"fuck you.\"\n\nFirstly, let's clarify what you're referring to. It seems like you might be experiencing frustration or anger towards someone who has provided assistance or support to you as an international student at UB. However, without more context or information, it's difficult to provide a specific answer.\n\nIf you could please provide more details about the situation, such as:\n\n1. Who is the person you're referring to?\n2. What specifically happened that led to your feelings of frustration or anger?\n3. Have you tried seeking help from other sources, such as counseling services, academic advisors, or campus resources? If so, how did they respond?\n4. Are there any specific incidents or events that stand out in your memory that contributed to your negative emotions?\n\nOnce we have a better understanding of your situation, we can explore potential solutions and strategies to address your feelings of frustration or anger. Here are some steps you can take:\n\n1. Take a break: Sometimes, taking a step back from the situation can help you process your emotions and gain perspective. Try engaging in activities that relax", "sources": [{"title": "E", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -8.984875679016113}, {"title": "J", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -9.218276977539062}, {"title": "Login Required", "url": "https://www.buffalo.edu/content/www/international-student-services/admin-pw.html", "score": -9.834419250488281}, {"title": "Site Map", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -10.994070053100586}, {"title": "Monday", "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html", "score": -11.279172897338867}], "timestamp": "2025-04-25T03:57:46.344246"}
+{"query": "find 4-2*8 value", "response": "To find the 4-2*8 value, we need to multiply the number of factors (4) by the product of the two numbers (2*8). Let's break down the calculation step-by-step:\n\n1. Multiply the number of factors (4):\n   - Factors = 4\n   - Value = 4 * 8 = 32\n\n2. Multiply the product of the two numbers (2*8):\n   - Product = 2 * 8 = 16\n\nNow, let's calculate the value:\nValue = Factors * Product\nValue = 4 * 16\nValue = 64\n\nSo, the 4-2*8 value is 64. \n\nIf you need more information or clarification regarding this calculation, please provide the sources you mentioned or clarify the specific requirements for finding the value. If you're unsure about accessing the specific e-form or picking up documents, you can contact the International Student Services (ISS) advisor through their website or phone number provided in the sources. They offer drop-in appointments for immigration-related inquiries and can assist with document pick-ups. The shortest path of travel from an accessible parking space to a barrier-free entrance is approximately 299 feet, and the options for", "sources": [{"title": "E", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -10.142992973327637}, {"title": "Login Required", "url": "https://www.buffalo.edu/content/www/international-student-services/admin-pw.html", "score": -11.353201866149902}, {"title": "Monday", "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html", "score": -11.437908172607422}, {"title": "Tuesday", "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html", "score": -11.488155364990234}, {"title": "Navigate Drop-ins", "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html", "score": -11.497251510620117}], "timestamp": "2025-04-25T03:59:58.703305"}
+{"query": "How do I apply for OPT?", "response": "To apply for Optional Practical Training (OPT) at the University at Buffalo (UB), follow these steps:\n\n1. Familiarize yourself with the requirements:\n   - Visit the Department of International Studies (DIS) website (<https://www.buffalo.edu/international-student-services/dis/index.html>) to find the current list of departments offering OPT courses.\n   - Review the course descriptions and prerequisites to ensure they align with your program of study and meet the minimum academic requirements for OPT.\n   - Familiarize yourself with the F-1 and J-1 categories of visas, which differ in terms of eligibility, duration, and job opportunities.\n   - Understand the difference between F-1 and J-1 students, including the differences in employment authorization, work hours, and job responsibilities.\n\n2. Obtain necessary documentation:\n   - As per Source 1, you need to bring the following documents to the International Student Services (ISS) office:\n     * F-1 visa application form (DS-160)\n     * Proof of enrollment in an eligible degree program (e.g., transcript, official class list, or certificate of completion)\n     * A completed F-1 Form 1A (Student Affidavit of Eligibility)\n     * A copy of", "sources": [{"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -0.36291491985321045}, {"title": "Students, Faculty and Staff", "url": "https://ubisss.buffalo.edu", "score": -2.3324153423309326}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -4.302586555480957}, {"title": "OPT STEM Extension", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events/workshops.html", "score": -5.6742072105407715}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -7.4197916984558105}], "timestamp": "2025-04-28T03:35:35.354370"}
+{"query": "What documents do I need for my I-20?", "response": "To obtain the necessary documents for your I-20, please follow these steps:\n\n1. **Financial Documentation**:\n   - Bank Statements/Letters: You will need to provide proof of your income, such as bank statements, letters from your employer, or a certified check. These documents should show your monthly income, including any bonuses, commissions, or other sources of income. Make sure to include the following items:\n     - A recent bank statement showing your salary or wages for the past three months.\n     - A letter from your employer confirming your employment status and stating your expected salary or wage amount.\n     - A copy of your tax return, if applicable, showing your income and deductions.\n     - A copy of your W-2 form, if applicable, showing your employer's withholding taxes.\n\n2. **Certificate of Deposit (CD) Statements**:\n   - CD Statements: If you have a CD with a maturity date within the next six months, you will need to provide a certificate of deposit (CD) statement to prove your ownership of the funds. This document should show the balance on your CD and the interest earned during the specified period. The following items are typically required:\n     - A recent CD statement showing the current balance on your CD.\n     - A", "sources": [{"title": "Financial Documentation - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/financial-documentation.html", "score": 5.1504974365234375}, {"title": "Your I-20 & Next Steps - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/your-i-20---next-steps.html", "score": 3.5453882217407227}, {"title": "2. Immigration Documents (Original Documents + One Set of Photocopies)", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -6.35145378112793}, {"title": "Social Security Card", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -8.054825782775879}, {"title": "Banking - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/life-in-buffalo/banking.html", "score": -8.616048812866211}], "timestamp": "2025-04-28T03:48:17.492692"}
+{"query": "Hello", "response": "Hello [Student Name],\n\nThank you for reaching out to us regarding your concerns as an international student at the University at Buffalo (UB). We understand that navigating the academic and personal aspects of studying abroad can be overwhelming, especially when dealing with unfamiliar cultures, language barriers, and academic expectations. To provide you with the most accurate and helpful information, we'll break down the process into several key areas:\n\n1. Academic Support:\na. English Language Learning: The University offers English language courses specifically designed for international students. These courses cover grammar, vocabulary, reading comprehension, writing, and speaking skills. You can find more information on our website under the \"Language & Cultural Services\" section.\nb. Academic Transcripts: Your official transcripts from your home country will serve as evidence of your academic achievements while studying at UB. You can request them through the International Student Office or by submitting a transcript request form online.\nc. Course Registration: Registering for classes is crucial to ensure that you meet the academic requirements for your program. Visit the UB Calendar or the course registration portal (<https://www.buffalo.edu/courses>) to view available courses and register for them.\nd. Academic Advising: An academic advisor can help you develop a study plan, identify potential challenges, and provide guidance", "sources": [{"title": "J", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -7.357999801635742}, {"title": "Activities", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -7.7490620613098145}, {"title": "E", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -7.804978847503662}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events.html", "score": -8.040448188781738}, {"title": "New Students", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -8.040448188781738}], "timestamp": "2025-04-28T03:49:54.504100"}
+{"query": "How do I apply for OPT?", "response": "To apply for Optional Practical Training (OPT) at the University at Buffalo (UB), follow these steps:\n\n1. Familiarize yourself with the requirements:\n   - Visit the Department of International Studies (DIS) website (<https://www.buffalo.edu/international-student-services/dis/index.html>) to find the current list of departments offering OPT courses.\n   - Review the course descriptions and prerequisites to ensure they align with your program of study and meet the minimum academic requirements for OPT.\n   - Familiarize yourself with the F-1 and J-1 categories of visas, which differ in terms of eligibility, duration, and job opportunities.\n   - Understand the difference between F-1 and J-1 students, including the differences in employment authorization, work hours, and job responsibilities.\n\n2. Obtain necessary documentation:\n   - As per Source 1, you need to bring the following documents to the International Student Services (ISS) office:\n     * F-1 visa application form (DS-160)\n     * Proof of enrollment in an eligible degree program (e.g., transcript, official class schedule, or certificate of completion)\n     * A completed F-1 Form 1040NR (Non-Resident Alien Income Tax Return)", "sources": [{"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -0.36291491985321045}, {"title": "Students, Faculty and Staff", "url": "https://ubisss.buffalo.edu", "score": -2.3324153423309326}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -4.302586555480957}, {"title": "OPT STEM Extension", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events/workshops.html", "score": -5.6742072105407715}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -7.4197916984558105}], "timestamp": "2025-04-28T04:07:46.961901"}
+{"query": "What documents do I need for my I-20?", "response": "To obtain the necessary documents for your I-20, please follow these steps:\n\n1. **Financial Documentation**:\n   - Bank Statements/Letters: You will need to provide proof of your income, such as bank statements, letters from your employer, or a certified check. These documents should show your monthly income, including any bonuses, commissions, or other sources of income. Make sure to include the following items:\n     - A recent bank statement showing your salary or wages for the past three months.\n     - A letter from your employer confirming your employment status and stating your expected salary or wage amount.\n     - A copy of your tax return, if applicable, showing your income and deductions.\n     - A copy of your W-2 form, if applicable, showing your employer's withholding taxes.\n\n2. **Certificate of Deposit (CD) Statements**:\n   - CD Statements: If you have a CD with a maturity date within the next six months, you will need to provide a certificate of deposit (CD) statement to prove your ownership of the funds. This document should show the balance on your CD and the interest earned during the specified period. The following items are typically required:\n     - A recent CD statement showing the current balance on your CD.\n     - A", "sources": [{"title": "Financial Documentation - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/financial-documentation.html", "score": 5.1504974365234375}, {"title": "Your I-20 & Next Steps - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/your-i-20---next-steps.html", "score": 3.5453882217407227}, {"title": "2. Immigration Documents (Original Documents + One Set of Photocopies)", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -6.35145378112793}, {"title": "Social Security Card", "url": "https://www.buffalo.edu/international-student-services/social-security-card---income-tax-filing/social-security-card.html", "score": -8.054825782775879}, {"title": "Banking - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/life-in-buffalo/banking.html", "score": -8.616048812866211}], "timestamp": "2025-04-28T04:09:25.417379"}
+{"query": "How do I apply for OPT?", "response": "To apply for Optional Practical Training (OPT) at the University at Buffalo (UB), follow these steps:\n\n1. Familiarize yourself with the requirements:\n   - Visit the Department of International Studies (DIS) website (<https://www.buffalo.edu/international-student-services/dis/index.html>) to find the current list of departments offering OPT courses.\n   - Review the course descriptions and prerequisites to ensure they align with your program of study and meet the minimum academic requirements for OPT.\n   - Familiarize yourself with the F-1 and J-1 categories of visas, which differ in terms of eligibility, duration, and job opportunities.\n   - Understand the difference between F-1 and J-1 students, including the differences in employment authorization, work hours, and job responsibilities.\n\n2. Obtain necessary documentation:\n   - As per Source 1, you need to bring the following documents to the International Student Services (ISS) office:\n     * F-1 visa application form (DS-160)\n     * Proof of enrollment in an eligible degree program (e.g., transcript, official transcripts, or CSS Profile)\n     * A completed F-1 Form 1040NR (Non-Resident Alien Income Tax Return)\n     * A", "sources": [{"title": "Travel - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html", "score": -0.36291491985321045}, {"title": "Students, Faculty and Staff", "url": "https://ubisss.buffalo.edu", "score": -2.3324153423309326}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -4.302586555480957}, {"title": "OPT STEM Extension", "url": "https://www.buffalo.edu/international-student-services/workshops-trips-events/workshops.html", "score": -5.6742072105407715}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -7.4197916984558105}], "timestamp": "2025-04-28T04:22:07.971683"}
+{"query": "How can I extend my student visa?", "response": "To extend your student visa, you will need to follow these steps:\n\n1. Obtain an F-1 visa: The first step is to apply for an F-1 visa, also known as a student visa. This visa allows you to enter the United States as a non-immigrant student and stay for up to four years. You can apply for an F-1 visa through the U.S. Department of State's (DHS) Electronic System for Travel Authorization (ESTA) website. Follow the instructions provided by the university's International Student Services office to complete the online application process.\n\n2. Complete the F-1 visa application: After submitting your ESTA application, you will need to gather additional documents and complete the F-1 visa application form. These include:\n\n   a. Proof of financial support: A bank statement showing proof of sufficient funds to cover your tuition, living expenses, and other educational costs for the duration of your stay.\n   b. Proof of English proficiency: A valid English language test score (such as TOEFL, IELTS, or Cambridge English Test) indicating that you meet the minimum requirements for academic study in the United States.\n   c. Academic transcripts: Official transcripts from your previous school(s) demonstrating your academic achievements and ability to", "sources": [{"title": "Visa Information - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/immigration-and-visas.html", "score": -1.6663498878479004}, {"title": "Immigration & Visa", "url": "https://www.buffalo.edu/international-student-services/about-us/site-map.html", "score": -2.531528949737549}, {"title": "Your I-20 & Next Steps - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/for-new-students/your-i-20---next-steps.html", "score": -2.5841100215911865}, {"title": "U.S. Visa", "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html", "score": -4.821455955505371}, {"title": "Immigration & Visa - International Student Services - University at Buffalo", "url": "https://www.buffalo.edu/international-student-services/immigration-visa.html", "score": -5.183196544647217}], "timestamp": "2025-04-28T23:45:56.984886"}
+{"query": "What are the different libraries in UB?", "response": "The University at Buffalo (UB) has several libraries that serve various purposes for students, faculty, and researchers. Here are some of the main libraries:\n\n1. UB Libraries - This is the central hub for all academic, research, and administrative services offered by UB. It houses the following libraries:\n\n   a. UB Main Library - Located on the UB campus, this library offers a wide range of collections, including print and electronic resources, rare books, manuscripts, and archives. It also serves as a hub for research and study spaces, hosting workshops, seminars, and events.\n   \n   b. UB East Campus Library - Situated on the east side of UB, this library provides access to UB's extensive collection of books, journals, and periodicals, as well as a variety of digital resources such as online databases, e-books, and streaming media.\n   \n   c. UB South Campus Library - Located on the south side of UB, this library focuses on providing access to UB's collection of books, journals, and periodicals, as well as a range of digital resources, including e-books, audiobooks, and streaming media.\n   \n   d. UB North Campus Library - Located on the north side of UB, this library serves as a hub for research and study spaces", "sources": [{"title": "Subscribe", "url": "https://calendar.buffalo.edu/event/boost-your-internship--job-search", "score": 2.134333610534668}, {"title": "Research Guides", "url": "https://library.buffalo.edu", "score": 0.4708395004272461}, {"title": "Research Guides", "url": "https://library.buffalo.edu", "score": 0.4708395004272461}, {"title": "University at Buffalo Libraries", "url": "https://library.buffalo.edu", "score": -4.499747276306152}, {"title": "University at Buffalo Libraries", "url": "https://library.buffalo.edu", "score": -4.499747276306152}], "timestamp": "2025-04-28T23:46:22.234302"}
+{"query": "name the restaurants in UB north campus", "response": "Hello [Student Name],\nThank you for reaching out to us regarding dining options on the North Campus of UB. Here are some popular restaurants located within the university:\n\n1. The Dining Commons: This iconic dining hall offers a wide range of cuisines, including American, Italian, Mexican, and Asian dishes. It features multiple dining locations, including the Main Dining Hall, the Student Union Dining Hall, and the East Side Dining Hall. You can find delicious food and drinks at The Dining Commons throughout the day, including breakfast, lunch, and dinner. To access the dining hall, simply enter the main entrance of the North Campus and follow the signs to the dining area.\n\n2. The UB Kitchen: Located in the UB Tower, The UB Kitchen serves as a hub for student cooking classes, workshops, and events. The kitchen offers a variety of menu items, including salads, sandwiches, pizzas, and desserts. Students can also take advantage of the kitchen's catering services for events and meetings. To learn more about The UB Kitchen, visit the UB Tower website or contact the culinary department directly.\n\n3. The UB Food Court: The UB Food Court is a convenient spot for grab-and-go meals, snacks, and beverages. It features a variety of fast-food chains, such as McDonald's", "sources": [{"title": "Life at UB", "url": "https://www.buffalo.edu/home/life.html", "score": 3.1820428371429443}, {"title": "Visiting UB - University at Buffalo", "url": "https://www.buffalo.edu/home/visiting-ub.html", "score": -2.2049336433410645}, {"title": "N", "url": "https://www.buffalo.edu/home/academics/degree_programs.html", "score": -6.767813205718994}, {"title": "UB Pride & Service Day", "url": "https://calendar.buffalo.edu/event/ub-pride--service-day-2", "score": -9.982414245605469}, {"title": "UB Pride & Service Day", "url": "https://calendar.buffalo.edu/event/ub-pride--service-day-2", "score": -9.982414245605469}], "timestamp": "2025-04-28T23:54:32.082853"}

data/raw/0000_www_buffalo_edu.json

@@ -0,0 +1,127 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services.html",
+  "title": "International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nISS is committed to supporting UB's international students during times of uncertainty. READ our [Recent Updates on Immigration Actions](https://www.buffalo.edu/international-student-services/immigration-visa/travel/travel-alerts/updates-on-recent-immigration-actions.html) to learn about resources you can access, how we are responding, and our advice in these circumstances.\nHave a question? ISS offers virtual drop-in advising throughout the week. No appointment needed!\nClick the button below or visit our [Contact Us page](/international-student-services/about-us/contact-us.html) for details.\nInternational students can work on campus but it's important to understand what rules apply. [Learn about on-campus employment](/international-student-services/immigration-visa/f-1-student/on-campus-employment.html).",
+  "headings": [
+    {
+      "level": 1,
+      "text": "International Student Services"
+    },
+    {
+      "level": 3,
+      "text": "F & J Travel"
+    },
+    {
+      "level": 3,
+      "text": "Working on Campus"
+    },
+    {
+      "level": 3,
+      "text": "Working off Campus"
+    },
+    {
+      "level": 3,
+      "text": "New to UB"
+    },
+    {
+      "level": 3,
+      "text": "Information Sessions"
+    },
+    {
+      "level": 2,
+      "text": "Drop In Advising"
+    },
+    {
+      "level": 2,
+      "text": "Student Life Events"
+    },
+    {
+      "level": 2,
+      "text": "Follow Us on Social Media"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:27:25.411366",
+  "path_hierarchy": [
+    "international-student-services.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0001_www_buffalo_edu.json

@@ -0,0 +1,66 @@
+{
+  "url": "https://www.buffalo.edu",
+  "title": "Welcome to the University at Buffalo - University at Buffalo",
+  "content": "We have your major\nWhatever your passion, you\u2019ll find your path at New York\u2019s flagship university. UB is No. 1 in the Northeast for its range of 500+ degree programs. And we\u2019re always creating more.\nSee more [stories](/home/how.html) of UB\u2019s impact\n#36\npublic university in U.S.\n1st\nSUNY invited to join AAU\n#1\npublic university\nin NYS for\nresearch activity\n#2\nin U.S. for climate action\n12:1\nstudent/faculty ratio",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    },
+    {
+      "level": 2,
+      "text": "We have your major"
+    },
+    {
+      "level": 2,
+      "text": "See morestoriesof UB\u2019s impact"
+    },
+    {
+      "level": 3,
+      "text": "Come see what it means to be True Blue"
+    },
+    {
+      "level": 2,
+      "text": "Schools and Colleges"
+    },
+    {
+      "level": 2,
+      "text": "Find People and Departments"
+    },
+    {
+      "level": 2,
+      "text": "Join Our UB Family"
+    },
+    {
+      "level": 2,
+      "text": "For Media"
+    },
+    {
+      "level": 2,
+      "text": "UB Social Media"
+    },
+    {
+      "level": 2,
+      "text": "Maps and Directions"
+    },
+    {
+      "level": 2,
+      "text": "UB Mobile App"
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "2/24/25Find it in the Student Guide",
+      "url": "/studentlife.html"
+    },
+    {
+      "text": "2/24/25Student Guide",
+      "url": "/studentlife.html"
+    }
+  ],
+  "categories": [],
+  "scraped_at": "2025-04-26T01:27:30.545768",
+  "path_hierarchy": [],
+  "domain": "www.buffalo.edu"
+}

data/raw/0002_www_buffalo_edu.json

@@ -0,0 +1,59 @@
+{
+  "url": "https://www.buffalo.edu/internationaleducation.html",
+  "title": "International Education - University at Buffalo",
+  "content": "As a diverse and inclusive scholarly community committed to academic freedom, UB has established the UB Scholars at Risk Fellowship to afford highly qualified academics around the world who are threatened in their own countries the opportunity to continue their research and teaching in a safe and supportive environment at UB.\nUB offers a variety of undergraduate programs in the Singapore Institute of Management (SIM), which prepares highly qualified graduates for exciting careers in Singapore\u2014as well as advanced degree programs in Singapore and abroad.\nThe Office of International Education (OIE) offers a variety of funding opportunities to support global and international research collaborations by faculty and graduate students.\nThe latest census by the Institute of International Education ranked UB No. 24 among the nation\u2019s colleges and universities hosting international students.\nThe goal of an International Faculty Launch Program at the University at Buffalo is to provide advice, information, and encouragement for new international faculty, within a confidential and positive atmosphere.\nUB schools and departments wishing to develop new international affiliations that require formal agreements should review this information and follow these guidelines.\nThe Office of International Education collaborates with participating schools in the development, approval and delivery of UB degree programs in other countries.\nThe University at Buffalo maintains institutional partnerships with more than 80 leading universities around the globe. These affiliations significantly enhance UB\u2019s international engagement and global reach, and help advance the university\u2019s research, education and service missions.\nAs a diverse and inclusive scholarly community committed to academic freedom, UB has established the UB Scholars at Risk Fellowship to afford highly qualified academics around the world who are threatened in their own countries the opportunity to continue their research and teaching in a safe and supportive environment at UB.\nUB offers a variety of undergraduate programs in the Singapore Institute of Management (SIM), which prepares highly qualified graduates for exciting careers in Singapore\u2014as well as advanced degree programs in Singapore and abroad.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "International Education"
+    },
+    {
+      "level": 2,
+      "text": "Events"
+    },
+    {
+      "level": 2,
+      "text": "Upcoming Events"
+    },
+    {
+      "level": 2,
+      "text": "Campus-Wide International News"
+    },
+    {
+      "level": 2,
+      "text": "What We Do"
+    },
+    {
+      "level": 2,
+      "text": "You may also be looking for"
+    },
+    {
+      "level": 2,
+      "text": "About"
+    },
+    {
+      "level": 2,
+      "text": "Services & Programs"
+    },
+    {
+      "level": 2,
+      "text": "News & Events"
+    },
+    {
+      "level": 2,
+      "text": "Grants & Fellowships"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [],
+  "categories": [],
+  "scraped_at": "2025-04-26T01:27:33.574783",
+  "path_hierarchy": [
+    "internationaleducation.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0003_www_buffalo_edu.json

@@ -0,0 +1,201 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/about-us/contact-us.html",
+  "title": "Contact Us - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nBefore you contact us, answers to most frequently asked questions are found on:\nDue to the high advising volume, advisors will not be meeting with those asking about processing times or those requesting an expedite.\nFor current processing times, please review [our website.](/international-student-services/immigration-visa/e-form-processing-times.html) If your request is outside of processing times, you can email us. Our team will review this information and follow-up with you to provide an updated timeline for the request or guidance on next steps.\nStill have a question? This is how you can reach us:\nHave questions? No appointment needed! Click the tab that fits your situation to see your advising options.\nNavigate Drop-ins\nIn-person Drop-ins\nNavigate Drop-ins\nNavigate Drop-ins\nDrop-ins are only active in Navigate during hours listed above. If you try to access the queue outside of advising hours you will not see ISS as an option.\nIt is not possible to book a drop-in appointment. Advising during drop-in times is first-come, first-served.\nIf you graduated more than 1 year ago, please visit us during OPT advising. Navigate may not work for you.\nTry accessing Navigate directly: [buffalo.edu/ubsuccess](https://buffalo.edu/ubsuccess/)\nMonday - Friday 9AM - 5PM\n[[email protected]](mailto:[email protected])\nPlease allow 3-4 business days for a response.\nThe 1st business day is the first full day immediately following ISS receiving your email. Business days are Monday, Tuesday, Wednesday, Thursday, Friday. University holidays are not considered business days.\nOur phone line is monitored by our student staff. Student staff can help connect you with resources and answer questions about using Navigate Drop-ins. Student staff do not answer immigration questions.\nIf you have immigration questions, connect with an ISS advisor using our advising services listed above.\nISS offers pick-up for select document requests. If pick-up is available for your request you will see the option in the specific e-form on [UB Global.](/international-student-services/about-us/ub-global.html)\nRequests that do not require a physical print-out are sent by email.\nThe shortest path of travel from an accessible parking space to a barrier-free entrance is 299 feet:\n| Feature | Available |\n|---|---|\n| Automatic door | Yes |\n| Ramp | Not applicable |\n| Bicycle rack | Yes |\nTo report a problem or provide feedback about these directions, please contact Equity, Diversity and Inclusion (EDI) at [[email protected]](mailto:[email protected]).",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Contact Us"
+    },
+    {
+      "level": 2,
+      "text": "On this page"
+    },
+    {
+      "level": 2,
+      "text": "Advising Schedule"
+    },
+    {
+      "level": 3,
+      "text": "Monday"
+    },
+    {
+      "level": 3,
+      "text": "Tuesday"
+    },
+    {
+      "level": 3,
+      "text": "Thursday"
+    },
+    {
+      "level": 3,
+      "text": "Friday"
+    },
+    {
+      "level": 4,
+      "text": "Navigate Drop-ins"
+    },
+    {
+      "level": 2,
+      "text": "How to Use Navigate Drop-in Advising"
+    },
+    {
+      "level": 2,
+      "text": "Need Help Using Navigate?"
+    },
+    {
+      "level": 3,
+      "text": "Issues Accessing Navigate?"
+    },
+    {
+      "level": 2,
+      "text": "Email Assistance"
+    },
+    {
+      "level": 4,
+      "text": "Counting Business Days"
+    },
+    {
+      "level": 2,
+      "text": "Phone Assistance"
+    },
+    {
+      "level": 2,
+      "text": "Document Pick-Up"
+    },
+    {
+      "level": 3,
+      "text": "Document Pick-up Hours"
+    },
+    {
+      "level": 2,
+      "text": "Accessible Directions and Map"
+    },
+    {
+      "level": 3,
+      "text": "Accessible Routes for Visiting Us"
+    },
+    {
+      "level": 2,
+      "text": "Talbert Hall"
+    },
+    {
+      "level": 3,
+      "text": "Accessible parking"
+    },
+    {
+      "level": 3,
+      "text": "Outside approach"
+    },
+    {
+      "level": 3,
+      "text": "210 Talbert Hall"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Document Pick-Up",
+      "url": "#title_1184985257"
+    },
+    {
+      "text": "4/5/25Curricular Practical Training (CPT)",
+      "url": "/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html"
+    },
+    {
+      "text": "2/22/24Optional Practical Training (OPT)",
+      "url": "/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-.html"
+    },
+    {
+      "text": "OPT STEM Extension",
+      "url": "http://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-/24-month-stem-extension.html"
+    },
+    {
+      "text": "Post-Completion OPT",
+      "url": "/international-student-services/about-us/contact-us.post-completion-opt.html"
+    },
+    {
+      "text": "STEM OPT",
+      "url": "/international-student-services/about-us/contact-us.stem-opt.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:19:28.867635",
+  "path_hierarchy": [
+    "international-student-services",
+    "about-us",
+    "contact-us.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0004_ubisss_buffalo_edu.json

@@ -0,0 +1,32 @@
+{
+  "url": "https://ubisss.buffalo.edu",
+  "title": "UB Global",
+  "content": "UB Global\nUB Global provides a variety of online services for international students, scholars and employees. Below are examples of the services available for each type of user. Please log in for full access to your international services account.\nStudents, Faculty and Staff\n- New Students: ISS Check-in, Reporting Visa delays, etc.\n- Current Students: Reduced Course Load, Practical Training, Transfer Request, etc.\n- Students on OPT: Employment reporting, U.S. address changes, change of status, etc.\n- Faculty and Staff: Process e-form requests for students in your department.\n- ISS, IA, IS, ELI: Administrative Services\nStudents on OPT: change your UBITName password. Starting in 2022, UB students are required to change their UBITName password. If you do not change your password, you will be unable to access your UB account in the future. If you have not done so already, please log into ubidm.buffalo.edu and change your password as soon as possible. [Learn more](https://www.buffalo.edu/ubit/news/article.host.html/content/shared/www/ubit/news/2021/password-resets-select-accounts.detail.html)\nUB Global Issues\nWe are aware that some students are having trouble uploading documents in e-form requests. Our team is working closely with tech support to fix these issues. The system is working intermittenly, so please keep trying and we hope to have it fixed soon.",
+  "headings": [
+    {
+      "level": 3,
+      "text": "UB Global"
+    },
+    {
+      "level": 1,
+      "text": "UB Global"
+    },
+    {
+      "level": 2,
+      "text": "Students, Faculty and Staff"
+    },
+    {
+      "level": 4,
+      "text": "Secure Online Session"
+    }
+  ],
+  "faqs": [],
+  "important_links": [],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:19:31.570969",
+  "path_hierarchy": [],
+  "domain": "ubisss.buffalo.edu"
+}

data/raw/0005_www_buffalo_edu.json

@@ -0,0 +1,119 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students.html",
+  "title": "New Students - International Student Services - University at Buffalo",
+  "content": "Congratulations on your acceptance to the University at Buffalo!\nWe would like to welcome you to UB! UB has a longstanding tradition of welcoming students, faculty and researchers from around the world. Our international population enriches our community through its knowledge, talents, and diverse perspectives. Here we truly embrace the American principles of inclusion for all people regardless of their race, religion or nationality.\nThe information contained in this web site is provided as a service to the international students, faculty, staff, employees and administrators of the University at Buffalo, and does not constitute legal advice on any immigration, tax or other matter. We try to provide useful information, but we make no claims, promises or guarantees about the accuracy, completeness or adequacy of the information contained in or linked to this website or any associated site. As legal advice must be tailored to the specific circumstances of each case, and laws are constantly changing, nothing provided herein should be used as a substitute for the advice of competent counsel. Neither the University at Buffalo nor the Office of International Student Services is responsible for any errors or omissions contained in this website, or for the results obtained from the use of this information.",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Admissions FAQ"
+    },
+    {
+      "level": 1,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Welcome to UB"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps3/31/25",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information4/18/25",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:19:34.094576",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0006_www_buffalo_edu.json

@@ -0,0 +1,137 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/your-i-20---next-steps.html",
+  "title": "Your I-20 & Next Steps - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nWelcome to UB! We're excited to have you join our community. Once you receive your I-20, there are a few important steps you'll need to follow to prepare for your journey to the U.S.\nVerify all information listed on your I-20 is accurate. If any discrepancies are found, notify us immediately, so the I-20 can be revised and reissued. Please review the [FAQ for F-1 Students](https://www.buffalo.edu/internationaladmissions/reference/frequently-asked-questions.html) to help determine if the details on your I-20 follows federal guidance and UB policy.\nStudent Attestation\nRegulation requires all prospective F-1 students to pay the I-901 Student and Exchange Visitor Information System (SEVIS) Fee before the Department of State issues you a visa. Additional information can be found on the [Study in the States](https://studyinthestates.dhs.gov/students/prepare/paying-the-i-901-sevis-fee) website.\nTo pay the I-901 SEVIS Fee [visit FMJfee.com](https://www.fmjfee.com/i901fee/index.html) to access the SEVIS Form I-901.\nAll prospective F-1 students must obtain a student visa to request entry to the U.S. in F-1 status.\nVisit the U.S. Department of State for instructions on [how to apply for an F-1 visa](https://travel.state.gov/content/travel/en/us-visas/study/student-visa.html).\n*Citizens of Canada are not required to obtain an F-1 visa. Canadian citizens must clearly present the I-20 and SEVIS Fee documentation at the port of entry to ensure they are correctly admitted in F-1 status.\nYou may enter the U.S. no more than 30 days prior to the start date listed on your I-20.\nLatest recommended arrival dates:",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Admissions FAQ"
+    },
+    {
+      "level": 1,
+      "text": "Your I-20 & Next Steps"
+    },
+    {
+      "level": 2,
+      "text": "Step 1: Check Your I-20 For Errors"
+    },
+    {
+      "level": 2,
+      "text": "Step 2: Sign Your I-20"
+    },
+    {
+      "level": 2,
+      "text": "Step 3: Begin ISS Check-in"
+    },
+    {
+      "level": 3,
+      "text": "When is Check-in Available?"
+    },
+    {
+      "level": 2,
+      "text": "Step 4: Pay the SEVIS I-901 Fee"
+    },
+    {
+      "level": 2,
+      "text": "Step 5: Apply for an F-1 Visa"
+    },
+    {
+      "level": 2,
+      "text": "Step 6: Review Pre-Arrival & Orientation Details"
+    },
+    {
+      "level": 2,
+      "text": "Step 7: Plan Your Travel"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "fees"
+  ],
+  "scraped_at": "2025-04-26T01:19:36.854009",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "your-i-20---next-steps.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0007_www_buffalo_edu.json

@@ -0,0 +1,198 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/immigration-and-visas.html",
+  "title": "Visa Information - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nWe can't wait to see you in Buffalo! Use these pages to help plan your arrival.\nYour visa category will impact many parts of your life in the U.S. Your studies, employment, vacation, and travel are a few examples. Your visa (sometimes referred to as visa stamp or visa foil) is the document placed on a page in your passport that permits you to apply for admission to the U.S. in accordance with your visa category.\nMost international students come to the U.S. on F-1 visas, while some come on J-1 visas. However, there are some other visa categories that permit studying (H-1B, H-4, L-2, E-2, etc.).\nTo come to the United States to study, you will need to apply for a visa at a U.S. Embassy or Consulate.\nYou cannot study for a degree on a tourist (B-1/2) visa or on the Visa Waiver Program (VWP). Therefore, you should not enter the U.S. as a B-1/2 or on the VWP if you plan to study.\nIf you are already in the U.S. on an F-1 or J-1 visa, you can study at UB on the same visa.\nVisas are entry permits that allow you to enter the U.S. from anywhere in the world, if your visa and number of entries have not expired, and if all of your other immigration documents are in order. Visas do not determine how long you can stay in the U.S., so, unless you travel abroad before joining UB, you can study at UB without renewing your visa even if it has expired.\nIf you are currently outside the U.S., you must apply for a U.S. visa stamp before you can arrive.\nIf you're not sure who to put as your contact? UB students can use ISS's information:\nISS cannot answer this question for you. We recommend you use your best estimate on the time you need to complete your program. You can contact your academic department for an estimate if you are not sure.\nYour answer to this question does not determine the [length of your visa](https://travel.state.gov/content/travel/en/us-visas/Visa-Reciprocity-and-Civil-Documents-by-Country.html).\nISS cannot provide detailed guidance on completing your DS-160. If you have questions we recommend contacting your local embassy or consulate for assistance.\nCanadian students do not need to apply for a U.S. visa.\nBefore you can transfer to UB from another U.S. school, be sure you understand how a transfer may affect your immigration status.\nIf you are already in the U.S. as an F-1 or J-1 student, it may be possible to transfer your SEVIS (Student and Exchange Visitor Information System) record from your current school to UB.\nAs a SEVIS transfer student, you will receive a \u201cTransfer Pending from (current school)\u201d I-20 instead of an \u201cInitial\u201d I-20.\nTo be eligible for a SEVIS transfer, you must:\nYour transfer release date is the day your current school releases your SEVIS record to your new school.\nThe only exception to the requirement to enroll during the next available session is when it is a summer vacation for which you are eligible. To be eligible for this exception, you must have been enrolled at a U.S. school for a full academic year AND you must enroll within five months of your transfer release date.\nIf you are not eligible for a SEVIS transfer, you must depart the U.S. and re-enter with an Initial I-20.\nThe rules for J-1 students\u2019 transfers are different than for F-1 students. If you are a J-1 student hoping to transfer, contact our office for advice.\nYou can transfer to another school during the 60-day grace period after your OPT, as long as the transfer release date occurs within the grace period.\nYou must begin classes within 5 months of your transfer release date or the date your OPT authorization ends, whichever is earlier.\nYour OPT work authorization will end on your transfer release date. Therefore, if you wish to complete your OPT, your transfer release date must be after your OPT ends.\nIf you meet all the conditions for a SEVIS transfer, you do not need to pay the SEVIS fee again.\nYes, but you must be careful to re-enter the U.S. using the I-20/DS-2019 issued by the school that currently has your SEVIS record.\nYou\u2019re ready for the next step in your UB education and we're glad to have you back!\nIf you have continually maintained your F-1 or J-1 status or were outside the U.S. for fewer than five months, you do not need to pay the SEVIS fee again. However, if you failed to maintain your F-1 or J-1 status or were outside the U.S. for more than five months not on authorized overseas study, you will have to pay the SEVIS fee again.\nYour OPT work authorization will automatically end when you start studying in your new degree program.\nYou can study part time while on OPT as long as you are not admitted to a new degree program at UB.\nIf you apply to a new degree program and change your mind after you receive admission to the program, inform International Student Services.",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Admissions FAQ"
+    },
+    {
+      "level": 1,
+      "text": "Visa Information for Accepted Students"
+    },
+    {
+      "level": 2,
+      "text": "What is a U.S. visa?"
+    },
+    {
+      "level": 3,
+      "text": "Can I use my current student visa if it is still valid?"
+    },
+    {
+      "level": 2,
+      "text": "New to the U.S."
+    },
+    {
+      "level": 3,
+      "text": "How do I apply for a student visa?"
+    },
+    {
+      "level": 2,
+      "text": "Tips for Your DS-160"
+    },
+    {
+      "level": 3,
+      "text": "U.S. Point of Contact"
+    },
+    {
+      "level": 3,
+      "text": "Intended Length of Stay"
+    },
+    {
+      "level": 3,
+      "text": "Important Points to Consider"
+    },
+    {
+      "level": 2,
+      "text": "Transferring from Other U.S. Schools"
+    },
+    {
+      "level": 3,
+      "text": "What if I am already in the U.S. as an F-1 or J-1 student?"
+    },
+    {
+      "level": 3,
+      "text": "Am I eligible for a SEVIS transfer?"
+    },
+    {
+      "level": 4,
+      "text": "Enrollment exception"
+    },
+    {
+      "level": 3,
+      "text": "J-1 transfers"
+    },
+    {
+      "level": 3,
+      "text": "What if I am on Optional Practical Training (OPT)?"
+    },
+    {
+      "level": 3,
+      "text": "Do I have to pay the SEVIS fee again?"
+    },
+    {
+      "level": 3,
+      "text": "If I am transferring, can I travel outside the U.S.?"
+    },
+    {
+      "level": 2,
+      "text": "Continuing UB Students"
+    },
+    {
+      "level": 3,
+      "text": "Do I have to pay the SEVIS fee again?"
+    },
+    {
+      "level": 3,
+      "text": "What if I am on Optional Practical Training (OPT)?"
+    },
+    {
+      "level": 3,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "application instructions",
+      "url": "https://travel.state.gov/content/travel/en/us-visas/visa-information-resources/forms/ds-160-online-nonimmigrant-visa-application.html"
+    },
+    {
+      "text": "Learn about F-1 visa and other F-1 immigration documents",
+      "url": "/international-student-services/immigration-visa/f-1-student/immigration-documents.html"
+    },
+    {
+      "text": "Learn about J-1 visa and other J-1 immigration documents",
+      "url": "/international-student-services/immigration-visa/j-1-student1/j-1-student/immigration-documents.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment",
+    "fees"
+  ],
+  "scraped_at": "2025-04-26T01:19:39.773146",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "immigration-and-visas.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0008_www_buffalo_edu.json

@@ -0,0 +1,180 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/new-international-student-check-in.html",
+  "title": "Check-In & Arrival - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nTo meet reporting requirements & be able to register for classes, complete your International Student Check-in early.\nAll F-1 and J-1 students starting a new program at UB, transferring to UB from another U.S. university or entering the U.S. with an initial I-20/DS-2019 are required to complete the International Student Check-in. There are several steps involved so we recommend you begin as soon as possible.\nYou cannot access your International Student Check-in until you have activated your UBIT account. If you're having trouble activating your account, visit [buffalo.edu/ubit/help](https://www.buffalo.edu/ubit/help) for assistance.\nISS prioritizes e-form requests based on timing of submission and order of processing priority for the office. Find a list of [e-form processing times](/international-student-services/immigration-visa/e-form-processing-times.html) for all ISS e-forms.\nWhen you arrive in the United States, it is important to have the right documents and know what to expect at the border.\nYou will need to carry the documents listed below with you. Do not pack them in your luggage because you will need to show them to the immigration inspector when you enter the U.S.\nFirst, after deplaning you will proceed to passport control.\nSecond, you will proceed to the baggage claim area and go through customs.\nNever make false statements to a border officer as the penalty for doing so is very severe.\nYou may enter the U.S. no more than 30 days before the start date listed on your I-20/DS-2019.\nClasses begin on Monday, August 25, 2025.\nWe know that things outside of your control, like travel problems and visa delays, can happen. If you will not be able to arrive in the U.S. by your program start date, please review the process for requesting a late arrival support letter.",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Want to learn more?"
+    },
+    {
+      "level": 1,
+      "text": "Check-in & Arriving at UB"
+    },
+    {
+      "level": 2,
+      "text": "Who Needs to do International Student Check-in?"
+    },
+    {
+      "level": 2,
+      "text": "Completing International Student Check-in"
+    },
+    {
+      "level": 3,
+      "text": "Before You Begin"
+    },
+    {
+      "level": 3,
+      "text": "How to Access International Student Check-in"
+    },
+    {
+      "level": 2,
+      "text": "After Starting Check-in"
+    },
+    {
+      "level": 3,
+      "text": "Undergraduate Students"
+    },
+    {
+      "level": 3,
+      "text": "Graduate Students"
+    },
+    {
+      "level": 2,
+      "text": "Arriving in the U.S."
+    },
+    {
+      "level": 2,
+      "text": "Preparing Immigration Documents"
+    },
+    {
+      "level": 2,
+      "text": "Traveling with Dependents"
+    },
+    {
+      "level": 2,
+      "text": "Going Through Customs"
+    },
+    {
+      "level": 2,
+      "text": "Arrival Deadlines"
+    },
+    {
+      "level": 3,
+      "text": "Know When to Arrive"
+    },
+    {
+      "level": 4,
+      "text": "Graduate, J-1 and UB/SIM Students"
+    },
+    {
+      "level": 4,
+      "text": "Undergraduate Students"
+    },
+    {
+      "level": 3,
+      "text": "Late Arrival"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "Pre-Arrival Information Sessions",
+      "url": "/international-student-services/for-new-students/graduate-students/pre-arrival-information-sessions.html"
+    },
+    {
+      "text": "e-form processing times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Read about F-1 immigration documents",
+      "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/immigration-documents.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:19:42.796366",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "new-international-student-check-in.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0009_www_buffalo_edu.json

@@ -0,0 +1,120 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/undergraduate-students.html",
+  "title": "Undergraduate Students - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nWelcome undergraduate students! Along with New to UB orientation programs, we're here to help you succeed at UB.\nAll first-year and transfer undergraduate F-1 students will participate in the New to UB orientation programs.\nStart with [Frequently Asked Questions (FAQ)](https://www.buffalo.edu/studentlife/new-to-ub/first-year-and-transfer-students/faqs-orientation.html) in the Student Guide. If you need additional information please contact Orientation, Transition and Parent Programs. They will respond during their regular business hours of Monday through Friday from 8:30 a.m. - 5:00 p.m. Eastern time.\nThe International Student Welcome Fee (listed as \"International Student ISS Orientation - Undergraduate\" on your student account) is a one-time $25 charge for undergraduate international students new to the UB campus that covers welcome and transition programming provided by ISS.\nThis fee will appear on your student account during the second or third billing cycle. The fee is charged to all new students, and is not charged based on your choice of participation in any or all of the welcome and transition programming offered by ISS.\nThis fee is separate from the orientation fee that covers your New to UB programming and communications facilitated by OTPP ([Orientation, Transition, and Parent Programs](https://www.buffalo.edu/studentlife/who-we-are/departments/orientation.html))",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Coming to UB from SIM?"
+    },
+    {
+      "level": 2,
+      "text": "Admissions FAQ"
+    },
+    {
+      "level": 1,
+      "text": "Undergraduate Students"
+    },
+    {
+      "level": 2,
+      "text": "New to UB Orientation Programs"
+    },
+    {
+      "level": 3,
+      "text": "New to UB Questions?"
+    },
+    {
+      "level": 3,
+      "text": "International Student Welcome Fee"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "fees"
+  ],
+  "scraped_at": "2025-04-26T01:19:45.503936",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "undergraduate-students.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0010_www_buffalo_edu.json

@@ -0,0 +1,126 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/graduate-students.html",
+  "title": "Graduate Students - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nWelcome graduate students! We're here to help guide you in getting started at UB.\nAttend our virtual [Pre-Arrival Information Live Events](/international-student-services/for-new-students/graduate-students/pre-arrival-information-sessions.html)!\nThis series of live events will help you successfully prepare to arrive, live and study at UB. Session recordings are posted on the series' page for your convenience.\nNew and transfer graduate international students should arrive by Sunday, August 17.\nWhat to do if [you can't arrive in Buffalo by the first day of class](/international-student-services/for-new-students/graduate-students/LateArrivalInformation.html).\nIf you arrive late, you still must complete your [International Student Check-in](/international-student-services/for-new-students/new-international-student-check-in.html). Don't forget to upload your I-94!",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Graduate Students"
+    },
+    {
+      "level": 2,
+      "text": "International Student Welcome"
+    },
+    {
+      "level": 3,
+      "text": "Are You Ready?"
+    },
+    {
+      "level": 4,
+      "text": "Pre-Arrival Live Events"
+    },
+    {
+      "level": 4,
+      "text": "Important Dates"
+    },
+    {
+      "level": 2,
+      "text": "When Should I Arrive?"
+    },
+    {
+      "level": 3,
+      "text": "ISS Check-in Required"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "Pre-Arrival Information Live Events",
+      "url": "/international-student-services/for-new-students/graduate-students/pre-arrival-information-sessions.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [],
+  "scraped_at": "2025-04-26T01:19:48.073254",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "graduate-students.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0011_www_buffalo_edu.json

@@ -0,0 +1,145 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/for-new-students/making-ub-your-home.html",
+  "title": "Making UB Your Home - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nDiscover everything that UB and Western New York have to offer.\nBefore arriving in Buffalo, you can experience UB and the beautiful Buffalo-Niagara region online.\nAfter you arrive in Buffalo, your best source of information will be the International Student Welcome Series. This program is designed to help you get the things you need to be a UB student, inform you about campus resources and services, get you off to a good start academically, introduce you to other new students, and provide valuable information about U.S. culture and life outside UB.\nBuffalo \u2014 dubbed the City of Good Neighbors \u2014 is the second-largest city in New York State, yet it\u2019s a big city with a hometown feel.\nBuffalo is a thriving city in a region of more than two million people that offers a world of opportunity. World-class art galleries and museums, a comprehensive city-wide system of parks and green space designed by renowned landscape architect Frederick Law Olmsted, and major and minor league sports teams are just a few of the wide array of cultural and recreational elements that make Buffalo such a great place to study, work, and live.\nBuffalo has four seasons, so you should be prepared for all types of weather (but remember, snow is fun and beautiful!). In winter, heavy jackets, warm hats, scarves, gloves and snow boots are necessary. During autumn and spring days, you will need a raincoat or lightweight coat or jacket. In the summer, shorts and T-shirts are adequate. Comfortable walking shoes are recommended for any season.\nUnless good quality winter clothing is available in your country, you should purchase it after you arrive in Buffalo. You will find a large selection of winter clothes available at different price levels.\nAt UB, it\u2019s easy to get around campus, between campuses, or anywhere you need to go.\nUB's international student clubs possess a wealth of information about academic programs, campus resources, social opportunities and life in Buffalo. While the clubs are all very different, they revolve around specific countries, cultures, languages and/or religions. It is easy to get involved with any club in which you have an interest\u2014regardless of your own affiliation. All you need is a desire to get involved and to learn. Be sure to contact your club or join a few!\nFor information about undergraduate international student clubs at UB, please visit the [UB Student Association Clubs web page.](https://www.sa.buffalo.edu/clubs)\nTo learn more about graduate international student clubs, please visit the UB [Graduate Student Association International Clubs web page](http://ubwp.buffalo.edu/gsa/clubs/international/).\nEducation in the United States relies on the expectation that students submit academic work that they have completed using only their own knowledge. Called \u201cacademic integrity,\u201d this expectation can be quite different from standards in other parts of the world.\nThe important thing to remember is that all students at UB are held accountable to upholding academic integrity rules, even if they do not know what those rules are. So please go to the [Office of Academic Integrity website](http://www.buffalo.edu/academic-integrity) to learn more about what is expected of you. Additionally, every instructor may set specific limits on how much students can work together, so always check your course syllabus and pay attention to assignment and exam instructions to ensure that you are following the proper standards.\nIf you ever have questions about what is allowed, ask your professor for clarification.\nF-1 international students are eligible to work on-campus without special authorization. J-1 international students are also eligible to work on-campus, but need a letter from International Student Services authorizing such employment.\nInternational students are not eligible to apply for \u201cWork Study\u201d positions. Work study is only available to domestic students who, as part of their financial aid package, are awarded funding from the U.S. federal government to work in paid campus positions while studying at the university.\nOnce you have activated your UBIT Account, you can start to search for on-campus jobs using UB Career Design Center's [Bullseye powered by Handshake](https://www.buffalo.edu/career/bullseye.html) job search engine.",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Admissions FAQ"
+    },
+    {
+      "level": 1,
+      "text": "Making UB Your Home"
+    },
+    {
+      "level": 2,
+      "text": "On this page:"
+    },
+    {
+      "level": 2,
+      "text": "Learn about UB and life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "Buffalo and Western New York"
+    },
+    {
+      "level": 3,
+      "text": "Buffalo's Four Seasons"
+    },
+    {
+      "level": 4,
+      "text": "What kind of clothes should I bring to Buffalo?"
+    },
+    {
+      "level": 2,
+      "text": "Public Transportation in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "International Student Clubs"
+    },
+    {
+      "level": 2,
+      "text": "Adapting to U.S. Academic Standards"
+    },
+    {
+      "level": 2,
+      "text": "On-Campus Employment"
+    },
+    {
+      "level": 3,
+      "text": "Finding an On-Campus Job"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "employment",
+    "fees"
+  ],
+  "scraped_at": "2025-04-26T01:19:50.965730",
+  "path_hierarchy": [
+    "international-student-services",
+    "for-new-students",
+    "making-ub-your-home.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0012_www_buffalo_edu.json

@@ -0,0 +1,95 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/life-in-buffalo.html",
+  "title": "Life In Buffalo - International Student Services - University at Buffalo",
+  "content": "The Buffalo/Niagara region can become your new home. UB has many resources available to help you while you're here.\nThe information contained in this web site is provided as a service to the international students, faculty, staff, employees and administrators of the University at Buffalo, and does not constitute legal advice on any immigration, tax or other matter. We try to provide useful information, but we make no claims, promises or guarantees about the accuracy, completeness or adequacy of the information contained in or linked to this web site or any associated site. As legal advice must be tailored to the specific circumstances of each case, and laws are constantly changing, nothing provided herein should be used as a substitute for the advice of competent counsel. Neither the University at Buffalo nor the Office of International Student Services is responsible for any errors or omissions contained in this website, or for the results obtained from the use of this information.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Life In Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:19:54.646783",
+  "path_hierarchy": [
+    "international-student-services",
+    "life-in-buffalo.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0013_www_buffalo_edu.json

@@ -0,0 +1,94 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/life-in-buffalo/health-insurance.html",
+  "title": "Health Insurance - International Student Services - University at Buffalo",
+  "content": "Want an easy app with lots of helpful UB resources? Looking to connect with classmates and find study buddies? Navigate life as a UB student with the Navigate app.\nThe U.S. Department of State and the State University of New York (SUNY) require all J-1 Exchange Visitors and their dependents in the U.S. to have specific insurance coverage for the duration of their Exchange Visitor status.\nQuestions?\nWe encourage you to read the Student Health Insurance Office's website about health insurance.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Health Insurance"
+    },
+    {
+      "level": 2,
+      "text": "Questions?"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [],
+  "scraped_at": "2025-04-26T01:19:57.377061",
+  "path_hierarchy": [
+    "international-student-services",
+    "life-in-buffalo",
+    "health-insurance.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0014_www_buffalo_edu.json

@@ -0,0 +1,40 @@
+{
+  "url": "http://www.buffalo.edu/studyabroad.html",
+  "title": "UB Education Abroad - University at Buffalo",
+  "content": "UB Education Abroad - University at Buffalo\n\nInterested in meeting with an advisor about studying abroad?\nStop by our office during our drop-in advising hours! Monday-Friday, 11am-3pm. View our Programs! Choosing a Program UB Program Search SUNY Program Search Costs & Scholarships",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Education Abroad"
+    },
+    {
+      "level": 2,
+      "text": "Get Started"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "5/21/24What Does an Application Look Like?",
+      "url": "/educationabroad/projects/application.html"
+    },
+    {
+      "text": "11/5/24Study Abroad Course Planning Form (UB programs only)",
+      "url": "/educationabroad/projects/study-abroad-course-planning-form.html"
+    },
+    {
+      "text": "5/21/24Information for Faculty and Staff",
+      "url": "/educationabroad/information-for-faculty-staff.html"
+    }
+  ],
+  "categories": [],
+  "scraped_at": "2025-04-26T01:20:00.701577",
+  "path_hierarchy": [
+    "studyabroad.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0015_www_buffalo_edu.json

@@ -0,0 +1,119 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa.html",
+  "title": "Immigration & Visa - International Student Services - University at Buffalo",
+  "content": "Here you will find out what you need to do to maintain your immigration status and how to obtain the immigration benefits for which you are eligible.\nThe information contained in this web site is provided as a service to the international students, faculty, staff, employees and administrators of the University at Buffalo, and does not constitute legal advice on any immigration, tax or other matter. We try to provide useful information, but we make no claims, promises or guarantees about the accuracy, completeness or adequacy of the information contained in or linked to this web site or any associated site. As legal advice must be tailored to the specific circumstances of each case, and laws are constantly changing, nothing provided herein should be used as a substitute for the advice of competent counsel. Neither the University at Buffalo nor the Office of International Student Services is responsible for any errors or omissions contained in this website, or for the results obtained from the use of this information.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "Drop In Advising"
+    },
+    {
+      "level": 2,
+      "text": "Upcoming Events"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "E-form Processing Times2/10/25",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation2/10/25",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:20:03.516633",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0016_www_buffalo_edu.json

@@ -0,0 +1,128 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/f-1-student.html",
+  "title": "F-1 Student - International Student Services - University at Buffalo",
+  "content": "Being in F-1 student status means that your primary purpose for being in the U.S. is to study.\nThe information contained in this web site is provided as a service to the international students, faculty, staff, employees and administrators of the University at Buffalo, and does not constitute legal advice on any immigration, tax or other matter. We try to provide useful information, but we make no claims, promises or guarantees about the accuracy, completeness or adequacy of the information contained in or linked to this web site or any associated site. As legal advice must be tailored to the specific circumstances of each case, and laws are constantly changing, nothing provided herein should be used as a substitute for the advice of competent counsel. Neither the University at Buffalo nor the Office of International Student Services is responsible for any errors or omissions contained in this website, or for the results obtained from the use of this information.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "F-1 Student"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Immigration Documents",
+      "url": "/international-student-services/immigration-visa/f-1-student/immigration-documents.html"
+    },
+    {
+      "text": "Curricular Practical Training (CPT)",
+      "url": "/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html"
+    },
+    {
+      "text": "Optional Practical Training (OPT)",
+      "url": "/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "Immigration Documents1/16/25",
+      "url": "/international-student-services/immigration-visa/f-1-student/immigration-documents.html"
+    },
+    {
+      "text": "Curricular Practical Training (CPT)4/5/25",
+      "url": "/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html"
+    },
+    {
+      "text": "Optional Practical Training (OPT)2/22/24",
+      "url": "/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:20:05.802261",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "f-1-student.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0017_www_buffalo_edu.json

@@ -0,0 +1,121 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/j-1-student1.html",
+  "title": "J-1 Student - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nThe J-1 Exchange Visitor Program enables foreign nationals to come to the U.S. to study, research, train, and more. The purpose of the program is to enhance skills, expand knowledge, and promote international exchange. Program length varies from a few weeks to several years.\nThe purpose of the J-1 Degree-Seeking Category is to complete a full-time academic degree-seeking program at the University at Buffalo.\nThe purpose of the J-1 Exchange Student Category is to complete a full-time course of study at the University at Buffalo as a part of a formal reciprocal agreement between the student's home university and the University at Buffalo.\nThe purpose of the J-1 Student Intern category is to participate in a U.S. internship that will fulfill the educational objectives for their current degree program at an academic institution outside the U.S. The internship must consist of a minimum of 32 hours of work per week.\nThe purpose of the J-1 Student Non-Degree category is to participate in prescribed course of study at the University at Buffalo. The prescribed course of study should be equivalent to a full-time workload.\n*Program pre-approval by ISS is required before students can be invited to participate in this category.\nIf you apply for a waiver of the Two-Year Home Country Physical Presence Requirement (INA 212e), and your waiver is recommended in writing by the U.S. Department of State (\u201cNo Objection Statement\u201d) or you receive Form I-612 from U.S. Citizenship and Immigration Services, you are no longer eligible for any J-1 benefits (DS-2019 Extension, Academic Training, SEVIS Transfer, etc.). Therefore, please consult with ISS before applying for the waiver.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "J-1 Student"
+    },
+    {
+      "level": 2,
+      "text": "J-1 Student Visa Options at UB"
+    },
+    {
+      "level": 2,
+      "text": "Information for all J-1 Student Categories"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "J-1 Student Visa Options at UB",
+      "url": "#title_2135517390"
+    },
+    {
+      "text": "Information for all J-1 Student Categories",
+      "url": "#title_2135517390_cop"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:20:08.299359",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "j-1-student1.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0018_www_buffalo_edu.json

@@ -0,0 +1,206 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/us-visa.html",
+  "title": "U.S. Visa - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nHere you will find information about applying for non-immigrant U.S. visas for international students.\nA non-immigrant U.S. visa is an entry permit for individuals coming to the U.S. temporarily for specific purposes such as study or work. It is issued by the U.S. Department of State and is stamped in your passport. It allows you to request permission to enter the U.S. at a U.S. Port of Entry.\nA visa alone does not guarantee entry into the U.S. You must also carry your other documents, including valid passport, and valid I-20, DS-2019 or I-797 Approval Notice.\nIf you have a valid F-1 visa stamp in your passport, you may be able to use that stamp to re-enter the United States. You are encouraged to contact the closest U.S. embassy or consulate in your home country to inquire as to whether or not you may use a visa stamp associated with your previous SEVIS record with a new document to enter the United States. Please note that Customs and Border Protection makes the final decision on your eligibility to enter the U.S. and can deny entry.\nAll applicants for F or J visas are required to demonstrate their intent to return to their home country following the completion of their studies or scholarly activities. There are several ways you can demonstrate your intent to return home:\nFor additional information about applying for an F-1 or J-1 visa, please visit the Department of State website.\nYou must obtain a U.S. visa at a U.S. Embassy or Consulate outside the U.S.\nWhile it is always recommended that you apply for a U.S. visa in your home country, some people apply in Canada. For information on applying for a U.S. visa in Canada, please refer to the U.S. Embassy and Consulates in Canada's website.\nIf you decide to apply for your U.S. Visa in Canada, please note that you may have to wait in Canada while your visa application is pending. In addition, if your application is denied, you will not be able to return to the U.S.\nJune, July, and August are the busiest months in most U.S. Consulates so interview appointments may be difficult to obtain during that period.\nDue to background checks, mandatory visa interviews and increased scrutiny of individuals studying or doing research in \u201csensitive fields,\u201d you should be prepared for delays in the processing of your visa application. If your field of study or research is in a sensitive field, you should obtain a letter from your professor explaining the nature of your studies or research. This letter should be written in simple language so it can be easily understood by non-specialists.\nAlso, if you violated your previous immigration status, have applied for Lawful Permanent Residence in the U.S. or have a criminal record, your application for a new visa will be subject to additional scrutiny or denial.\nThe U.S. Department of State website lists the forms, documents, etc. which are required for the U.S. visa application. It is important that you bring everything to your visa interview to avoid needless delays in the processing of your application.\nComplete the Online Nonimmigrant Visa Application (Form DS-160). When you upload your photo, be sure it conforms to the Photograph Requirements. Print the application confirmation page for your interview.\nIf you're not sure who to put as your contact? UB students can use ISS's information:\nISS cannot answer this question for you. We recommend you use your best estimate on the time you need to complete your program. You can contact your academic department for an estimate if you are not sure.\nYour answer to this question does not determine the [length of your visa](https://travel.state.gov/content/travel/en/us-visas/Visa-Reciprocity-and-Civil-Documents-by-Country.html).\nISS cannot provide detailed guidance on completing your DS-160. If you have questions we recommend contacting your local embassy or consulate for assistance.\nPay the nonrefundable Visa Application (MRV) Fee. Note: When your visa is issued, you may also be required to pay the Reciprocal Visa Issuance Fee. Check the Visa Reciprocity Table to see if you are required to pay this fee.\n(If required) Pay the SEVIS I-901 Fee online, by mail or by Western Union. If you already paid the SEVIS Fee and don't know if you need to pay it again, please read SEVIS Fee (for F-1 students), SEVIS Fee (for J-1 students and scholars) or SEVIS I-901 Fee. For additional information, read SEVIS I-901 Fee Frequently Asked Questions.\nPlease refer to the list of documents above for U.S. visa applications. In addition to these documents, you should bring your UB transcripts. We also recommend that you print out an Enrollment Verification Certificate from the HUB Student Center for additional verification of your full-time student status.\nIf you have been out of the U.S. for more than 5 months, you may need to pay the SEVIS fee again. Please review the SEVIS I-901 Fee and SEVIS I-901 Fee FAQ links below to determine whether or not you are required to pay the SEVIS fee.\nFor additional requirements, please contact the U.S. Consulate or Embassy where you will apply for your visa.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "U.S. Visa"
+    },
+    {
+      "level": 2,
+      "text": "Visa Links"
+    },
+    {
+      "level": 2,
+      "text": "What is a Non-Immigrant U.S. Visa?"
+    },
+    {
+      "level": 3,
+      "text": "Non-Immigrant Intent"
+    },
+    {
+      "level": 2,
+      "text": "Where to Get a U.S. Visa"
+    },
+    {
+      "level": 2,
+      "text": "Applying in Canada"
+    },
+    {
+      "level": 2,
+      "text": "Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "Application Process"
+    },
+    {
+      "level": 3,
+      "text": "Documents Needed for F-1 Student Visa or J-1 Student Visa Applications"
+    },
+    {
+      "level": 3,
+      "text": "How Do I Apply for an F-1 Student Visa or J-1 Student Visa?"
+    },
+    {
+      "level": 4,
+      "text": "Step 1."
+    },
+    {
+      "level": 2,
+      "text": "Tips for Your DS-160"
+    },
+    {
+      "level": 3,
+      "text": "U.S. Point of Contact"
+    },
+    {
+      "level": 3,
+      "text": "Intended Length of Stay"
+    },
+    {
+      "level": 4,
+      "text": "Step 2."
+    },
+    {
+      "level": 4,
+      "text": "Step 3."
+    },
+    {
+      "level": 2,
+      "text": "What Should I Bring to My Visa Interview?"
+    },
+    {
+      "level": 2,
+      "text": "Renewing a U.S. Student Visa"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "application instructions",
+      "url": "https://travel.state.gov/content/travel/en/us-visas/visa-information-resources/forms/ds-160-online-nonimmigrant-visa-application.html"
+    },
+    {
+      "text": "Application Process",
+      "url": "#title_1458022789_cop"
+    },
+    {
+      "text": "Online Nonimmigrant Visa Application",
+      "url": "https://ceac.state.gov/genniv/"
+    },
+    {
+      "text": "Learn more about the application",
+      "url": "https://travel.state.gov/content/travel/en/us-visas/visa-information-resources/forms/ds-160-online-nonimmigrant-visa-application.html"
+    },
+    {
+      "text": "Visa Application (MRV) Fee",
+      "url": "https://travel.state.gov/content/travel/en/us-visas/visa-information-resources/fees/fees-visa-services.html"
+    },
+    {
+      "text": "F-1 SEVIS Fee Information",
+      "url": "/international-student-services/immigration-visa/f-1-student/sevis-fee.html"
+    },
+    {
+      "text": "J-1 SEVIS Fee Information",
+      "url": "/international-student-services/immigration-visa/j-1-student1/sevis-fee.html"
+    },
+    {
+      "text": "SEVIS I-901 Fee Information",
+      "url": "http://studyinthestates.dhs.gov/students/sevis-i-901-fee"
+    },
+    {
+      "text": "SEVIS I-901 Fee Information",
+      "url": "http://studyinthestates.dhs.gov/students/sevis-i-901-fee"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment",
+    "fees"
+  ],
+  "scraped_at": "2025-04-26T01:20:15.761046",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "us-visa.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0019_www_buffalo_edu.json

@@ -0,0 +1,221 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/travel.html",
+  "title": "Travel - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nLearn about what documents to carry and other frequently asked questions for international student travel.\nWe strongly recommend that you review this additional information on [travel while on OPT](/international-student-services/immigration-visa/travel/on-opt.html) for more specific considerations such as travel with a pending OPT, applying for a visa, unemployment, Cap Gap situations and more.\nIf you are approved for post-completion Academic Training, in addition to the above required documents, we recommend that you carry a letter from your employer verifying your academic training employment.\nCanada has its own immigration rules and requirements. Many students will need a [Canadian Visa](https://www.canada.ca/en/immigration-refugees-citizenship/services/visit-canada/about-visitor-visa.html) before traveling to Canada. We encourage you to review our [travel to Canada](/international-student-services/immigration-visa/travel/outside-the-us.html) information before planning your trip.\nUpon arrival at a U.S. international airport, seaport, or land border crossing, all persons, including U.S. citizens, are subject to inspection by U.S. Customs and Border Protection (CBP). Anyone seeking to lawfully enter the U.S. must establish their admissibility to the satisfaction of the CBP officer.\nYou must present your passport, visa, and other supporting documents. The CBP officer will review your travel documents to confirm your identity and nationality, and ask questions about your visit to the U.S. Most nonimmigrants will have their digital finger prints and photograph taken. If you are determined admissible, the CBP officer may place an admission stamp in your passport, specifying the date of entry, the visa class, and the duration of admission and will issue an electronic I-94.\nFor most travelers, the screening process takes no more than a few minutes.\nIf the travel endorsement signature on your I-20 (page 2) or DS-2019 (bottom right, page 1) has expired please complete the appropriate Travel Endorsement/Travel Validation Request eform in [UB Global](https://ubisss.buffalo.edu/istart/controllers/start/StartEngine.cfm).\nIf you lost your I-20, please complete the I-20/DS-2019 Reprint Request eform found in [UB Global](https://ubisss.buffalo.edu/istart/controllers/start/StartEngine.cfm). Check our [e-form processing page](/international-student-services/immigration-visa/e-form-processing-times.html) for current processing times. Once approved we will send you a new I-20 with a new travel signature.\nIf there is not enough time before your flight or trip to Canada to receive the updated I-20 we recommend that you change your flight date/travel plans to allow time for ISS to get a new I-20 to you. If that is not possible, you can choose to attempt to enter the U.S. without a valid travel signature, however, this is risky and not recommended. U.S. Customs and Border Protection (CBP) officers may deny entry into the United States if:\nCBP officers can also decide to permit your entry to the U.S. by issuing an I-515A. Please read the information below about I-515As. Again, our strong recommendation is that you prepare ahead of time and receive a travel endorsement before your planned re-entry to the U.S.\nRather than deny your entry to the U.S., the CBP officer has the discretion to issue you a [Form I-515A](https://studyinthestates.dhs.gov/students/form-i-515a-overview). This document allows you to enter the U.S. and remain in the U.S. for 30 days. To remain in the U.S. beyond 30 days, and to maintain your F-1/J-1 status, you must submit required documents to SEVP within 30 days of arriving in the U.S. It is extremely important that you contact ISS for advise on completing the I-515. In many cases we must issue a new I-20/DS-2019 before you can submit the documents to SEVP.\nIf you fail to complete any required actions or send all of the required documents to SEVP within 30 days of entering the U.S., your SEVIS record and I-20 may be terminated. If your SEVIS record and I-20 are terminated, you will be required to leave the U.S. immediately or file for Reinstatement to F-1 status with USCIS, if eligible.\nIf your passport is lost or stolen, either in the U.S. or while traveling, you must first contact the embassy or consulate of your own country to get instructions on how to obtain a new passport. If you were outside the U.S. when your passport was stolen or lost, you must contact a U.S. embassy/consulate to make an appointment to apply for a new F-1 or J-1 visa before you can return to the U.S. If your passport was stolen or lost inside the U.S., a new visa is not immediately required. You will need a new visa the next time you plan to enter the U.S., so please plan accordingly.\nIf your Form I-20 or DS-2019 have also been stolen or lost, you can request new documents by submitting a I-20/DS-2019 Re-print Request in [UB Global](https://ubisss.buffalo.edu/).\nYou must have a valid F-1 or J-1 visa and a valid passport in order to reenter the U.S.\nMost UB students will clear U.S. immigration after your documents are reviewed by a CBP officer. However, a few may be referred to another officer to verify your student status or for additional questioning. This is referred to as \u201c[secondary inspection](https://studyinthestates.dhs.gov/what-is-secondary-inspection)\u201d.\nAlthough a referral to secondary inspection can be intimidating and make you feel anxious, there is usually little cause for alarm. Be aware that it can take a few hours to complete this inspection process.\nIf the CBP officer has questions that can be answered by ISS, ask the officer if they would like to call ISS for confirmation. During office hours, CBP can reach us at our general office number, (716) 645-2258.\nIf you are regularly sent to secondary inspection, please contact ISS for advice.\nPuerto Rico and the United States Virgin Islands are part of the United States. You should carry your valid passport and valid immigration documents when you travel there, as you should with any travel in the United States. However, if you go from Puerto Rico or the United States Virgin Islands to other islands, then you are leaving the United States and will need to formally re-enter and a valid visa will be required, unless you qualify for Automatic Revalidation.\nIf they are not eligible for the [visa waiver program](https://www.cbp.gov/travel/international-visitors/esta) (ESTA), they may apply for the [B-2 Visitor Visa](https://travel.state.gov/content/travel/en/us-visas/tourism-visit/visitor.html).\nFor a list of additional documents required for a B-2 visa application, please check the Consulate website where they will apply. Generally, it is recommended to provide your visitors with copies of your immigration documents (I-20, I-94, F-1 Visa Stamp) and a copy of your enrollment verification or transcript from the [Registrar office](https://www.buffalo.edu/registrar). It is also important that they show sufficient funds to cover any expenses during their trip to the U.S.",
+  "headings": [
+    {
+      "level": 2,
+      "text": "Drop In Advising"
+    },
+    {
+      "level": 1,
+      "text": "F-1 & J-1 Travel"
+    },
+    {
+      "level": 2,
+      "text": "Documents Needed For Travel"
+    },
+    {
+      "level": 3,
+      "text": "Click on the section below that applies to you to learn more"
+    },
+    {
+      "level": 2,
+      "text": "F-1 Enrolled Student"
+    },
+    {
+      "level": 2,
+      "text": "F-1 Student on OPT"
+    },
+    {
+      "level": 2,
+      "text": "F-2 Dependent"
+    },
+    {
+      "level": 2,
+      "text": "J-1 Enrolled Students & those on Academic Training"
+    },
+    {
+      "level": 2,
+      "text": "J-2 Dependent"
+    },
+    {
+      "level": 2,
+      "text": "Important Travel Considerations and Frequently Asked Questions"
+    },
+    {
+      "level": 2,
+      "text": "I want to travel to Canada, what do I need to know?"
+    },
+    {
+      "level": 2,
+      "text": "What should I expect at the Port of Entry?"
+    },
+    {
+      "level": 2,
+      "text": "I forgot to get a travel signature or I forgot/lost my I-20/DS-2019. What should I do?"
+    },
+    {
+      "level": 2,
+      "text": "I forgot to get a travel signature or I forgot/lost my I-20/DS-2019. The CBP officer gave me an I-515A. What should I do?"
+    },
+    {
+      "level": 2,
+      "text": "My passport was stolen or is lost, what should I do?"
+    },
+    {
+      "level": 2,
+      "text": "When I entered the U.S., the Border official sent me to secondary inspection. Why did that happen?"
+    },
+    {
+      "level": 2,
+      "text": "Do I need a valid visa to visit Puerto Rico, or the U.S. Virgin Islands or the Northern Mariana Islands?"
+    },
+    {
+      "level": 2,
+      "text": "I want my family to visit me in the U.S. What do they need?"
+    },
+    {
+      "level": 2,
+      "text": "Looking for more information?"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "Travel Endorsement Information",
+      "url": "/international-student-services/immigration-visa/travel/travel-endorsement-information.html"
+    },
+    {
+      "text": "While on Optional Practical Training (OPT)",
+      "url": "/international-student-services/immigration-visa/travel/on-opt.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "Documents Needed For Travel",
+      "url": "#title_1346824965"
+    },
+    {
+      "text": "Looking for more information?",
+      "url": "#title_1318488101"
+    },
+    {
+      "text": "Maintaining status on Post-OPT",
+      "url": "http://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-/maintaining-status-during-post-completion-opt.html"
+    },
+    {
+      "text": "Maintaining Status on STEM OPT",
+      "url": "http://www.buffalo.edu/international-student-services/immigration-visa/f-1-student/optional-practical-training--opt-/maintaining-status-while-on-opt.html"
+    },
+    {
+      "text": "travel while on OPT",
+      "url": "/international-student-services/immigration-visa/travel/on-opt.html"
+    },
+    {
+      "text": "e-form processing page",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Form I-515A",
+      "url": "https://studyinthestates.dhs.gov/students/form-i-515a-overview"
+    },
+    {
+      "text": "2/3/25Travel Endorsement Information",
+      "url": "/international-student-services/immigration-visa/travel/travel-endorsement-information.html"
+    },
+    {
+      "text": "8/12/24While on Optional Practical Training (OPT)",
+      "url": "/international-student-services/immigration-visa/travel/on-opt.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:20:19.426179",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "travel.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0020_www_buffalo_edu.json

@@ -0,0 +1,141 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/e-form-processing-times.html",
+  "title": "E-form Processing Times - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nISS provides important e-form services to thousands of F and J students through UB Global. ISS prioritizes e-form requests based on timing of submission and order of processing priority for the office.\nProcessing times are updated as needed based on the volume of requests and may vary. This means that two requests submitted on the same date may be processed on different dates.\nPlease find current estimated processing times measured in business days for ISS e-forms below. Please note the 1st business day is the first full day immediately following the submission of the e-form. Business days are Monday, Tuesday, Wednesday, Thursday, Friday. University holidays are not considered business days.\n| E-form Title | Estimated Processing in Business Days |\n|---|---|\n| Academic Reduced Course Load | 7-10 |\n| Adding an F-2 Dependent Step 1 | 3 |\n| Adding an F-2 Dependent Step 2 | 7-10 |\n| Curricular Practical Training | 12-15 |\n| CPT Extension Requests* | 10 |\n| F-1 CPT Pharm D IPPE/APPE Rotation | 12-15 |\n| Final Semester Reduced Course Load | 7-10 |\n| General Document Upload | 5-7 |\n| Graduate Leave of Absence | 5-7 |\n| I-20 Extension of Stay | 12-15 |\n| I-20 Reprint | 7-10 |\n| Initial Status I-20 Request | 12-15 |\n| Medical Reduced Course Load | 7-10 |\n| Request to End F-1 Status | 5-7 |\n| SEVIS Transfer Request | 5-7 |\n| Social Security DSO Letter Request | 5-7 |\n| Travel Endorsement Request | 7-10 |\n| Undergraduate Leave of Absence | 5-7 |\n*CPT extension requests are currently accepted as emails from the academic department confirming they support the extension of the current CPT through a specific date ([within acceptable dates as listed on our CPT page)](/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html) and explaining the academic necessity for the work authorization extension.\n| E-form Title | Estimated Processing in Business Days |\n|---|---|\n| Add New STEM Employment | 7-10 |\n| Change of Employer While STEM Extension is Pending | 7-10 |\n| Edit STEM Employment | 7-10 |\n| I-20 Reprint Request - OPT | 7-10 |\n| OPT Employment Reporting: SEVP Portal Issue | 7-10 |\n| OPT Travel Endorsement Request | 7-10 |\n| Post-Completion OPT I-20 Request | 12-15 |\n| Pre-Completion OPT I-20 Request | 12-15 |\n| Request for Cap Gap Extension I-20 | 12-15 |\n| SEVP Portal Unlock Request | 1-3 |\n| STEM Extension I-20 Request | 12-15 |\n| STEM OPT Reporting | 5-7 |\n| E-form Title | Estimated Processing in Business Days |\n|---|---|\n| Academic Reduced Course Load | 7-10 |\n| Academic Training Application | 7-10 |\n| Academic Training Evaluation | 7-10 |\n| Change of Status Notification | 7-10 |\n| Departure from UB | 7-10 |\n| DS-2019 Extension of Stay Step 1 | 1-3 |\n| DS-2019 Extension of Stay Step 2 | 12-15 |\n| DS-2019 Reprint Request | 7-10 |\n| Final Semester Reduced Course Load | 7-10 |\n| General Document Upload | 5-7 |\n| Medical Reduced Course Load | 7-10 |\n| On-Campus Work Authorization Letter Request | 5-7 |\n| Social Security ARO Letter Request | 5-7 |\n| Travel Validation Request | 7-10 |\n| E-form Title | Estimated Processing in Business Days |\n|---|---|\n| Change of Plans | 5-7 |\n| Form DS-2019 | 5-7 |\n| Form I-20 | 5-7 |\n| I-94 Record** | 5-7 |\n| Orientation Data Form | 5-7 |\n| Passport | 5-7 |\n| Visa Stamp | 5-7 |\n** Processing times are for ISS e-form processing. If your I-94 needs a correction please note corrections take 2-4 weeks due to CBP processing times.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "E-form Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "Determining Processing Times"
+    },
+    {
+      "level": 3,
+      "text": "Example: Counting 10 business days"
+    },
+    {
+      "level": 2,
+      "text": "F-1 General E-form Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "F-1 OPT E-form Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "J-1 E-form Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "ISS Check-in Processing Times"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "F-1 General E-form Processing Times",
+      "url": "#title_0_copy_copy_co"
+    },
+    {
+      "text": "F-1 OPT E-form Processing Times",
+      "url": "#title_0_copy_copy_co_1918805256"
+    },
+    {
+      "text": "J-1 E-form Processing Times",
+      "url": "#title_0_copy_copy_co_1027843613"
+    },
+    {
+      "text": "within acceptable dates as listed on our CPT page)",
+      "url": "/international-student-services/immigration-visa/f-1-student/curricular-practical-training--cpt-.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:20:22.114870",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "e-form-processing-times.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0021_www_buffalo_edu.json

@@ -0,0 +1,130 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/financial-documentation.html",
+  "title": "Financial Documentation - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nIssuing new, extended or dependent I-20\u2019s & DS-2019\u2019s requires current financial documentation that meets specific standards.\n| Documentation | Acceptable |\n|---|---|\n| Bank Statements/Letters | Yes* |\n| Certificate of Deposit (CD) Statements | Yes* |\n| Chartered Accountant Statement | No |\n| Credit Card Statements | No |\n| Employer Letter/Salary Statement | No |\n| Line of Credit/Loan Letters | Yes* |\n| Proof of Tuition Scholarship/Assistantship | Yes* |\n| Proof of Adjunct Teaching Appointment at UB | Yes |\n| National Letter of Intent (NLI) | Yes |\n| Stock Market Statement | No |\n| Valuation Statements (Jewelry, Gold, Property) | No |\nBelow is sample language to be used if the financial documentation submitted is not in your own name.\nThe letter must be signed by the sponsor and must include the actual signature (not typed) of the person providing support. The letter must be clear; scanned copies are acceptable. ISS does not need the original letter mailed to the office.\nDATE\nI, NAME OF SPONSOR, agree to financially support NAME OF STUDENT, for their tuition, fees and living expenses incurred while a student at the University at Buffalo. Attached please find my financial documents indicating the amount of funding available to NAME OF STUDENT.\nIf additional information is needed, please do not hesitate to contact me.\nSincerely,\n[INCLUDE SIGNATURE OF SPONSOR]\nTYPE NAME OF SPONSOR\nInclude sponsor\u2019s home address\nInclude sponsor\u2019s phone number",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Financial Documentation"
+    },
+    {
+      "level": 2,
+      "text": "Document Types"
+    },
+    {
+      "level": 3,
+      "text": "*Additional Information"
+    },
+    {
+      "level": 2,
+      "text": "Document Requirements"
+    },
+    {
+      "level": 2,
+      "text": "Sample Sponsor Letter"
+    },
+    {
+      "level": 3,
+      "text": "Sample Sponsor Letter"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "Document Types",
+      "url": "#title_804082812"
+    },
+    {
+      "text": "Document Requirements",
+      "url": "#title_504513248"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "fees",
+    "housing"
+  ],
+  "scraped_at": "2025-04-26T01:20:24.654657",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "financial-documentation.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0022_www_buffalo_edu.json

@@ -0,0 +1,271 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/estimated-expenses.html",
+  "title": "Estimated Expenses - International Student Services - University at Buffalo",
+  "content": "Requesting a new, extended, or dependent I-20 or DS-2019? International Student Services requires current, valid financial documentation. Find out how much financial support you will need to show.\nThese figures are estimates; the final amount on your I-20/DS-2019 may vary slightly based on your specific program. Actual costs will vary depending on degree, program and personal spending. Visit Student Accounts for more information.\nHow to Use This Page\nTo determine your estimate you must consider:\nWhat are you requesting?\nIf you are extending your I-20 or DS-2019 the length of the stay impacts how many semesters you need to finance.\nIf you are adding a dependent the default estimate is based on one year of expenses.\nDo you need enrollment?\nIf yes you must know how many credits you will take for each semester covered in the estimate.\nIf you will be below full-time enrollment you must consider how you will maintain status (reduced course load, full-time certification, etc.)\nIn many cases Summer enrollment is optional. If you will not enroll in Summer you do not need to include Summer tuition in your estimate. Summer estimates are provided for convenience for those who do enroll.\nIf Summer is your first or final semester enrollment is required to maintain status.\nUse the below estimates based on your academic degree level and planned credit loads to determine your estimated tuition costs.\nAdd living expenses for each semester.\nAdd dependent expenses for each semester based on the number and type of F-2/J-2 dependents currently on your SEVIS record.\nIf you have a dependent on your SEVIS record who is no longer in the US in F-2/J-2 status contact ISS so we can update your SEVIS record.\nUsing your estimated expenses gather financial documentation and submit your request to ISS on UB Global. For detailed information on acceptable documentation please read our Financial Documentation page.\nOn Post-Completion OPT, STEM OPT, or Post-Graduation Academic Training?\nIf you're in F/J status after graduation and no longer enrolled in courses you only need to determine estimated living expenses for yourself and any dependents on your SEVIS record. In most cases you will need estimated living expenses for 1 year.\nMaintaining F/J Status\nBelow are example estimates based on common credit loads for students based on full-time enrollment, enrollment with a Reduced Course Load, enrollment with an assistantship, or enrollment with full-time certification. When planning your enrollment for any semester be sure you understand how to maintain status and how any enrollment under 12 credits must be accompanied by additional approvals.\nEstimated Expenses 2024-2025*\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$33,800\n$16,900\n$16,900\n$8,500\nPart Time Enrollment - 6 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$17,000\n$8,500\n$8,500\n$4,200\nPart Time Enrollment- 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,000\n$1,500\n$1,500\n$1,500\nIf your full-time certification (FTC) or assistantship information does not yet appear in HUB for a future semester, we will request proof of full-time funding for that semester (12+ credits).\nGeneral Graduate Programs\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$30,600\n$15,300\n$15,300\n$7,700\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$24,000\n$12,000\n$12,000\n$3,800\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,000\n$1,500\n$1,500\n$1,500\nGraduate Management Programs\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$32,400\n$16,200\n$16,200\n$8,100\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$25,000\n$12,500\n$12,500\n$4,100\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,000\n$1,500\n$1,500\n$1,500\nGraduate Architecture Programs\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$32,800\n$16,400\n$16,400\n$8,200\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$25,000\n$12,500\n$12,500\n$4,100\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,000\n$1,500\n$1,500\n$1,500\nIf your full-time certification or assistantship information does not yet appear in HUB for a future semester, we will request proof of full-time funding for that semester (12+ credits).\nLaw Programs\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$37,600\n$18,800\n$18,800\n$9,400\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$29,000\n$14,500\n$14,500\n$4,700\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,200\n$1,600\n$1,600\n$1,500\nPharmacy Programs\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$44,600\n$22,300\n$22,300\n$11,100\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$34,000\n$17,000\n$17,000\n$5,600\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$4,000\n$2,000\n$2,000\n$2,000\nPhysical Therapy Doctorate\nFull Time Enrollment - 12 credits Fall & Spring, 6 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$39,000\n$19,500\n$19,500\n$9,600\nWith Assistantship - 9 credits Fall & Spring, 3 credits Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$29,000\n$14,500\n$14,500\n$4,800\nWith Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nTuition & Fees\n$3,200\n$1,600\n$1,600\n$1,600\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nLiving Expenses\n$17,00\n$8,500\n$8,500\n$5,500\nAdd the below expenses to your estimated expense costs based on the number of dependents included.\nDependents\nAcademic Year (9 months)\nFall (4 months)\nSpring (5 months)\nSummer (3 months)\nSpouse\n$7,000\n$3,500\n$3,500\n$2,100\nChild\n$5,000\n$2,500\n$2,500\n$1,600\nImportant Considerations:\nMaintaining Status\nIn order to maintain status you must study full-time and make normal progress toward your degree. For details on enrollment requirements to maintain your status please refer to:\nFees include the student activity fee, comprehensive fees, the international student fee, and international student health insurance.\nFood and housing (also known as \u201cboard\u201d) costs are based on average costs; this cost will vary slightly depending on your personal living arrangements and lifestyle.\nSome programs, such as architecture, art programs, and health-related programs may have significantly higher costs for supplies.\n*All above estimated expenses are reviewed annually and are subject to change. Increases can be expected.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Estimated Expenses"
+    },
+    {
+      "level": 2,
+      "text": "How to Use This Page"
+    },
+    {
+      "level": 3,
+      "text": "Step 1: Planning"
+    },
+    {
+      "level": 3,
+      "text": "Step 2: Adding Up Estimated Costs"
+    },
+    {
+      "level": 3,
+      "text": "Step 3: Submit Your Request to ISS"
+    },
+    {
+      "level": 3,
+      "text": "On Post-Completion OPT, STEM OPT, or Post-Graduation Academic Training?"
+    },
+    {
+      "level": 3,
+      "text": "Maintaining F/J Status"
+    },
+    {
+      "level": 2,
+      "text": "Estimated Expenses 2024-2025*"
+    },
+    {
+      "level": 2,
+      "text": "Undergraduate Tuition Estimates"
+    },
+    {
+      "level": 3,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 3,
+      "text": "Part Time Enrollment - 6 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 3,
+      "text": "Part Time Enrollment- 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 2,
+      "text": "Graduate Tuition Estimates"
+    },
+    {
+      "level": 3,
+      "text": "General Graduate Programs"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 3,
+      "text": "Graduate Management Programs"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 3,
+      "text": "Graduate Architecture Programs"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 2,
+      "text": "Professional Tuition Estimates (Law, Pharmacy, etc.)"
+    },
+    {
+      "level": 3,
+      "text": "Law Programs"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 3,
+      "text": "Pharmacy Programs"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 3,
+      "text": "Physical Therapy Doctorate"
+    },
+    {
+      "level": 4,
+      "text": "Full Time Enrollment - 12 credits Fall & Spring, 6 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Assistantship - 9 credits Fall & Spring, 3 credits Summer"
+    },
+    {
+      "level": 4,
+      "text": "With Full-Time Certification (FTC) in HUB - 1 credit Fall, Spring, & Summer"
+    },
+    {
+      "level": 2,
+      "text": "F-1/J-1 Living Expense Estimates"
+    },
+    {
+      "level": 2,
+      "text": "Dependent Expense Estimates"
+    },
+    {
+      "level": 3,
+      "text": "Add the below expenses to your estimated expense costs based on the number of dependents included."
+    },
+    {
+      "level": 2,
+      "text": "Important Considerations:"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "http://www.buffalo.edu/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment",
+    "fees",
+    "housing"
+  ],
+  "scraped_at": "2025-04-26T01:20:27.262154",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "estimated-expenses.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0023_www_buffalo_edu.json

@@ -0,0 +1,129 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/u-s--laws-and-your-rights.html",
+  "title": "U.S. Laws and Your Rights - International Student Services - University at Buffalo",
+  "content": "[University at Buffalo (UB)](//www.buffalo.edu/)\nIt\u2019s important to understand the laws based on where you live and study. Here, you'll find information about what you can and can't do, and how to stay safe. Some of the words might be new, but don't worry \u2013 if you're not sure about something, ask ISS or other campus offices for help understanding.\nAs someone visiting the United States from another country, you have rights to keep you safe. But there are also things you need to be careful about when you join protests or demonstrations.\nIf you get arrested, charged, and found guilty while participating in a protest, it could affect your ability to stay in the country or apply for visas later on. For example, if the police tell everyone to leave, and you don't go quickly enough, you could end up getting arrested. Being found guilty means you broke a state or federal law. Your immigration status may depend on avoiding legal trouble by following all laws.\nIf you are arrested, it will be very important to consult with a criminal attorney to advise and support you during any court proceedings. While a court will assign an individual an attorney if they cannot afford one, not all attorneys are familiar with the impact on immigration status of an arrest or acceptance of certain plea decisions.\nIt is also important that you continue to maintain your immigration status. International students need to remember that their purpose for being in the U.S. is to be a full-time student, and they should continue to attend all classes.\nComplying with [University rules and codes of conduct](https://www.buffalo.edu/studentlife/life-on-campus/community/rules.html) are a part of maintaining status.\nDid you know that UB has its own policy for assembly and protest? At UB, you have the right to protest and gather together, but you must always follow the rules from the university and law enforcement.\nProtest and free speech rights in the United States are outlined by the U.S. Constitution. The U.S. Constitution's First Amendment protects people's rights to gather for protests (freedom of assembly), speak out (freedom of speech), and ask the government for changes without reprisal (right to petition).\nThese rights aren't only for U.S. citizens. They also protect visitors from other countries who join lawful public demonstrations and protests. Know your rights as a noncitizen and learn [how to respond to immigration questions](https://www.aclu.org/know-your-rights/immigrants-rights#law-enforcement-asks-about-my-immigration-status) if you are stopped by police or other government officials.\nFederal laws apply to everyone in the United States. State and local laws apply to people who live or work in a particular state, county, or town. As an international student you are expected to comply with all laws and regulations that apply to your situation, including federal laws. Failure to do so can result in a violation of status.\nWhile New York and many U.S. states have recently changed laws related to marijuana, it is still considered a controlled substance by the federal government. International students who possess or use cannabis products may be subject to severe consequences (for example, visa refusal or revocation). In this case, where state and federal law conflict, international students are always subject to federal law. It may also be helpful to review [UB's policy on alcohol & drugs](https://www.buffalo.edu/administrative-services/policy-compliance-and-internal-controls/policy/ub-policy-lib/alcoholic-beverages-drugs-narcotics.html).",
+  "headings": [
+    {
+      "level": 1,
+      "text": "U.S. Laws and Your Rights"
+    },
+    {
+      "level": 2,
+      "text": "Protests and Your Rights"
+    },
+    {
+      "level": 3,
+      "text": "Potential Impact on Your Immigration Status"
+    },
+    {
+      "level": 3,
+      "text": "UB Picketing & Assembling Policy"
+    },
+    {
+      "level": 3,
+      "text": "Your Rights"
+    },
+    {
+      "level": 2,
+      "text": "State Law vs. Federal Law"
+    },
+    {
+      "level": 2,
+      "text": "Additional Resources"
+    },
+    {
+      "level": 2,
+      "text": "Disclaimer"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration",
+    "employment"
+  ],
+  "scraped_at": "2025-04-26T01:20:29.750621",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "u-s--laws-and-your-rights.html"
+  ],
+  "domain": "www.buffalo.edu"
+}

data/raw/0024_www_buffalo_edu.json

@@ -0,0 +1,108 @@
+{
+  "url": "https://www.buffalo.edu/international-student-services/immigration-visa/immigration---visa-links.html",
+  "title": "Immigration & Visa Links - International Student Services - University at Buffalo",
+  "content": "Completing your DS-160 and looking for assistance? Be sure to review the information on the Department of State's website FAQs and application instructions.\nImmigration & Visa Links\nUse these links to quickly and easily find the information you need.",
+  "headings": [
+    {
+      "level": 1,
+      "text": "Immigration & Visa Links"
+    },
+    {
+      "level": 2,
+      "text": "New Students"
+    },
+    {
+      "level": 2,
+      "text": "Activities"
+    },
+    {
+      "level": 2,
+      "text": "Immigration & Visa"
+    },
+    {
+      "level": 2,
+      "text": "Social Security Card & Income Tax"
+    },
+    {
+      "level": 2,
+      "text": "Life in Buffalo"
+    },
+    {
+      "level": 2,
+      "text": "About Us"
+    },
+    {
+      "level": 1,
+      "text": "Your Privacy is Important To Us."
+    }
+  ],
+  "faqs": [],
+  "important_links": [
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    },
+    {
+      "text": "application instructions",
+      "url": "https://travel.state.gov/content/travel/en/us-visas/visa-information-resources/forms/ds-160-online-nonimmigrant-visa-application.html"
+    },
+    {
+      "text": "U.S. Department of State Visa Information",
+      "url": "https://travel.state.gov/content/visas/en.html"
+    },
+    {
+      "text": "3/31/25Your I-20 & Next Steps",
+      "url": "/international-student-services/for-new-students/your-i-20---next-steps.html"
+    },
+    {
+      "text": "4/18/25Visa Information",
+      "url": "/international-student-services/for-new-students/immigration-and-visas.html"
+    },
+    {
+      "text": "2/10/25Information Sessions",
+      "url": "/international-student-services/workshops-trips-events/workshops.html"
+    },
+    {
+      "text": "2/10/25E-form Processing Times",
+      "url": "/international-student-services/immigration-visa/e-form-processing-times.html"
+    },
+    {
+      "text": "2/10/25Financial Documentation",
+      "url": "/international-student-services/immigration-visa/financial-documentation.html"
+    }
+  ],
+  "categories": [
+    "immigration"
+  ],
+  "scraped_at": "2025-04-26T01:20:32.544631",
+  "path_hierarchy": [
+    "international-student-services",
+    "immigration-visa",
+    "immigration---visa-links.html"
+  ],
+  "domain": "www.buffalo.edu"
+}