Refactor code structure for improved readability and maintainability

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,22 @@
+.env
+chroma_db
+src/__pycache__/__init__.cpython-313.pyc
+config/__pycache__/settings.cpython-313.pyc
+src/data_loader/__pycache__/loader.cpython-313.pyc
+src/data_loader/__pycache__/__init__.cpython-313.pyc
+src/document_processor/__pycache__/processor.cpython-313.pyc
+src/document_processor/__pycache__/__init__.cpython-313.pyc
+src/embedding_generator/__pycache__/__init__.cpython-313.pyc
+src/embedding_generator/__pycache__/embedder.cpython-313.pyc
+src/ingestion_orchestrator/__pycache__/__init__.cpython-313.pyc
+src/ingestion_orchestrator/__pycache__/orchestrator.cpython-313.pyc
+src/llm_integrator/__pycache__/__init__.cpython-313.pyc
+src/llm_integrator/__pycache__/llm.cpython-313.pyc
+src/query_service/__pycache__/__init__.cpython-313.pyc
+src/retrieval_handler/__pycache__/__init__.cpython-313.pyc
+src/retrieval_handler/__pycache__/retriever.cpython-313.pyc
+src/utils/__pycache__/__init__.cpython-313.pyc
+src/utils/__pycache__/logging.cpython-313.pyc
+src/query_service/__pycache__/api.cpython-313.pyc
+src/vector_store_manager/__pycache__/__init__.cpython-313.pyc
+src/vector_store_manager/__pycache__/chroma_manager.cpython-313.pyc

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+EXPOSE 7860
+
+COPY --chown=user . /app
+ENV PYTHONPATH=/app
+ENV HOST=0.0.0.0 PORT=7860
+CMD ["python", "scripts/run_query_api.py"]

PLANNING.md

@@ -0,0 +1,92 @@
+### General RAG Platform Plan
+
+#### Overview
+The goal is to transform the existing RAG system into a versatile, modular platform for building various RAG applications.
+
+#### Architecture
+
+```mermaid
+%%{init: {'theme': 'neutral', 'themeVariables': { 'primaryColor': '#e3f2fd', 'edgeLabelBackground':'#fffde7'}}}%%
+graph TD
+    A[General RAG Platform Architecture]
+    
+    subgraph DataIngestion
+        A1[Universal Data Loader\n<- Files\n<- Databases\n<- APIs\n<- Cloud Storage]
+        A2[Smart Document Processor\n<- Format detection\n<- Metadata extraction\n<- Content normalization]
+        A3[Chunking Strategies\n<- Semantic\n<- Structural\n<- Domain-specific]
+    end
+
+    subgraph CoreServices
+        B1[Embedding Service\n<- Multi-model support\n<- Batch processing\n<- Cache layer]
+        B2[VectorDB Orchestrator\n<- Chroma\n<- Pinecone\n<- Weaviate\n<- FAISS]
+        B3[LLM Gateway\n<- OpenAI\n<- Anthropic\n<- Mistral\n<- Custom models]
+    end
+
+    subgraph QueryEngine
+        C1[Query Analyzer\n<- Intent detection\n<- Query expansion\n<- Filter generation]
+        C2[Hybrid Retriever\n<- Vector search\n<- Keyword\n<- Hybrid ranking]
+        C3[Response Generator\n<- Citation\n<- Formatting\n<- Guardrails]
+    end
+
+    subgraph Management
+        D1[Config Manager\n<- Tenant isolation\n<- Model configs\n<- Access controls]
+        D2[Monitoring\n<- Metrics\n<- Logging\n<- Alerting]
+        D3[API Gateway\n<- REST\n<- GraphQL\n<- gRPC]
+    end
+
+    subgraph Extensibility
+        E1[Plugin System\n<- Custom loaders\n<- Chunkers\n<- Post-processors]
+        E2[Workflow Engine\n<- Pipeline designer\n<- Versioning\n<- A/B testing]
+    end
+
+    A --> DataIngestion
+    A --> CoreServices
+    A --> QueryEngine
+    A --> Management
+    A --> Extensibility
+    
+    DataIngestion -->|Processed Chunks| CoreServices
+    CoreServices -->|Vector Index| QueryEngine
+    QueryEngine -->|Formatted Response| Management
+    Management -->|APIs| ExternalSystems
+```
+
+#### Implementation Plan
+
+1. **Core Abstraction Layer**
+   - Unified interfaces for:
+   - Document Loaders (File, DB, API)
+   - Chunking Strategies
+   - Embedding Providers
+   - VectorDB Adapters
+   - LLM Gateways
+
+2. **Multi-tenancy Features**
+   - Tenant isolation
+   - Resource quotas
+   - Custom pipeline configurations
+   - Role-based access control
+
+3. **Advanced Retrieval**
+   - Hybrid search (vector + keyword + custom)
+   - Query understanding module
+   - Result reranking layer
+   - Cache mechanisms
+
+4. **Operational Excellence**
+   - Observability stack
+   - Auto-scaling
+   - CI/CD pipelines
+   - Health checks
+
+5. **Security**
+   - Data encryption
+   - Audit trails
+   - Content moderation
+   - PII detection
+
+6. **Developer Ecosystem**
+   - SDKs (Python/JS)
+   - CLI tools
+   - Template repository
+   - Testing framework

config/settings.py

@@ -0,0 +1,43 @@
+# config/settings.py
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+# --- General Settings ---
+DOCS_FOLDER = os.getenv("DOCS_FOLDER", "docs/")
+PERSIST_DIR = os.getenv("PERSIST_DIR", "tmp/chroma_db/") # Default to local persistence,
+# but plan to use a scalable backend like ClickHouse for production
+
+# --- Embedding Model Settings ---
+OLLAMA_URL = os.getenv("OLLAMA_SERVER","https://chandimaprabath-ollama-server.hf.space")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "nomic-embed-text:latest")
+
+# --- Document Processing Settings ---
+CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "2000")) # Default chunk size
+CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "10")) # Default chunk overlap
+
+# --- LLM Settings (OpenAI Compatible) ---
+LLM_API_KEY = os.getenv("LLM_API_KEY")
+LLM_API_BASE = os.getenv("LLM_API_BASE", "https://llm.chutes.ai/v1") # Default API base
+LLM_MODEL = os.getenv("LLM_MODEL", "chutesai/Llama-4-Scout-17B-16E-Instruct") # Default LLM model
+
+# --- Retrieval Settings ---
+TOP_K = int(os.getenv("TOP_K", "5")) # Default number of documents to retrieve
+CHROMADB_COLLECTION_NAME = os.getenv("CHROMADB_COLLECTION_NAME", "my_rulings_collection") # Unique collection name
+
+# --- Security Settings (Placeholders - Implement according to government standards) ---
+# Add settings for authentication, authorization, encryption paths, etc.
+# SECRET_KEY = os.getenv("SECRET_KEY") # Example for API security
+
+# --- Logging Settings ---
+LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG") # Default log level
+
+# --- Scalable Backend Settings (for production ChromaDB) ---
+# Example settings if using ClickHouse as a backend for ChromaDB
+# CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST")
+# CLICKHOUSE_PORT = os.getenv("CLICKHOUSE_PORT")
+# CLICKHOUSE_DATABASE = os.getenv("CLICKHOUSE_DATABASE")
+# CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER")
+# CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD")

docs/Chapter 1 - The Weight of Silence.md

@@ -0,0 +1,37 @@
+## Chapter 1: The Weight of Silence
+
+The chipped ceramic mug warmed Kaelen’s hands, a small comfort in the perpetual chill that seemed to cling to him. He sat hunched in the corner booth of the dimly lit diner, the pre-dawn quiet a familiar blanket. Outside, the first hesitant streaks of grey were painting the sky over Oakhaven City. He watched the street through the condensation on the window, the occasional early riser a fleeting blur.
+
+He didn’t taste the lukewarm coffee. Taste, like most sensations, was muted, filtered through a thick wall of numbness. It wasn't a physical inability; it was a conscious, or perhaps unconscious, choice. To feel was to remember, and remembering was a pain he couldn’t afford.
+
+A waitress, a woman with tired eyes and a nametag that read “Brenda,” approached his table. She didn’t speak, just placed a plate of slightly burnt toast and a single fried egg in front of him. She knew his routine. Kaelen nodded almost imperceptibly, a silent acknowledgment. He picked at the food, forcing it down. Sustenance was necessary, even if pleasure wasn’t.
+
+The diner slowly began to fill with the murmur of early morning conversations, the clatter of cutlery, and the hiss of the coffee machine. Kaelen remained an island of silence in the growing noise. He observed the other patrons – a construction worker with calloused hands, a young woman nervously checking her phone, an elderly couple sharing a newspaper. Fleeting thoughts, snippets of their emotions, brushed against the edges of his awareness – a flicker of worry, a surge of anticipation, a quiet contentment. He pushed them away, a practiced mental maneuver. Engaging with the emotions of others was too close to feeling his own.
+
+A sudden commotion erupted near the entrance. A man in a worn leather jacket was shouting at Brenda, his face red with anger.
+
+“I said I wanted crispy bacon! This is practically raw!”
+
+Brenda, her shoulders slumping, tried to placate him. “I’m sorry, sir, I can have them cook you another batch…”
+
+“Another batch? I’m in a hurry! You think my time is worthless?” The man’s voice rose, drawing the attention of the other customers.
+
+Kaelen’s grey eyes, usually fixed on his plate or the window, flickered towards the scene. He felt a familiar tightening in his chest, a subtle stirring beneath the layers of numbness. Injustice, even in something as trivial as undercooked bacon, always pricked at him.
+
+The man continued his tirade, his words laced with insults. Brenda’s face was pale, her hands trembling slightly. No one else in the diner intervened, their gazes either averted or filled with a detached curiosity.
+
+Kaelen’s fingers tightened around his mug. He didn’t move, didn’t make a sound. But within him, something shifted. A subtle pressure built in the air around the angry man, an almost imperceptible resistance. The man, mid-sentence, suddenly stumbled back a step, as if he’d bumped into an invisible wall. He blinked, confused, his anger momentarily disrupted.
+
+“What the…?” he muttered, looking around.
+
+He tried to step forward again, but the invisible pressure remained, holding him back. He frowned, his anger returning, but now tinged with bewilderment.
+
+Brenda, seizing the opportunity, spoke quickly. “Sir, please, let me just get you that new bacon…”
+
+The man, still slightly off-balance and unable to explain the strange resistance he felt, grumbled under his breath but allowed Brenda to retreat to the kitchen. The tension in the diner eased, the other patrons returning to their meals and conversations.
+
+Kaelen released the subtle pressure, the invisible force dissipating as quickly as it had formed. He looked down at his untouched toast, the brief flicker of emotion receding, leaving behind the familiar emptiness. He hadn’t moved, hadn’t spoken, yet the situation had been defused. No one had noticed his intervention, and that was how he preferred it. He was a ghost in the machine, a silent observer, occasionally nudging the world in small, unnoticed ways.
+
+He finished his coffee, the lukewarm liquid doing little to dispel the inner chill. He stood, placing a few crumpled bills on the table. Brenda offered a small, grateful smile as he passed her, a silent understanding passing between them.
+
+Outside, the city was beginning to wake. Cars honked, buses rumbled, and the sounds of life echoed around him. Kaelen walked, his footsteps silent on the pavement, another anonymous figure in the urban sprawl. He had no destination in mind, just the instinct to keep moving, to remain unseen, to carry the weight of his silence through another day. The echoes of his past were always there, a constant hum beneath the surface, a reminder of the voice he had lost and the emotions he had buried deep within. And yet, sometimes, in the face of injustice, a flicker of something else ignited within him, a spark of power that yearned to be more than just a silent echo.

docs/Chapter 2 - Dust and Whispers.md

@@ -0,0 +1,45 @@
+## Chapter 2: Dust and Whispers
+
+The antique shop, tucked away on a quiet side street, was Kaelen’s infrequent sanctuary. Dust motes danced in the shafts of sunlight that slanted through the grimy front window, illuminating rows of forgotten treasures and curiosities. Elias Thorne, a man whose face seemed etched with the stories of the objects he collected, stood behind the counter, polishing a tarnished silver locket with a soft cloth.
+
+The bell above the door chimed softly as Kaelen entered. Elias looked up, a hint of a smile touching his lips. He didn’t say anything, didn’t need to. He simply nodded, his gaze understanding the silent greeting.
+
+Kaelen moved through the aisles, his fingers trailing lightly over the smooth surface of a weathered wooden box, the cool metal of an old telescope. The air in the shop was thick with the scent of aged paper, beeswax, and time. It was a comforting aroma, a stark contrast to the sterile, impersonal feel of the city outside.
+
+He eventually made his way to the back of the shop, to a small, cluttered corner Elias had cleared for him. There was a worn armchair, its leather cracked but still comfortable, and a small table piled with old books. Kaelen settled into the chair, the familiar creak of the springs a soothing sound.
+
+Elias finished polishing the locket and placed it carefully in a display case. He moved with a slow, deliberate grace, his eyes occasionally flicking towards Kaelen. After a few minutes of comfortable silence, he walked over to the corner, carrying a steaming mug.
+
+He placed the mug on the table beside Kaelen. “Chamomile,” he said softly, his voice raspy from years of disuse. “Figured you could use something warm.”
+
+Kaelen looked at the mug, then up at Elias. A flicker of gratitude, a rare spark of warmth, touched his grey eyes. He nodded again, reaching for the mug. He brought it to his lips, the gentle floral scent a welcome change from the bitter coffee.
+
+Elias settled onto a nearby stool, his gaze thoughtful. “Saw the news this morning,” he said, his voice low. “Another one of those… incidents.”
+
+Kaelen’s hand tightened slightly around the mug. He knew what Elias meant. Unexplained occurrences, strange bursts of energy, objects moving on their own – the subtle manifestations of his uncontrolled power. The city news often reported them as bizarre anomalies, unsolved mysteries.
+
+“They’ll be looking,” Elias continued, his eyes filled with concern. “They always do.”
+
+Kaelen looked away, his gaze fixed on a dusty book on the table. He didn’t need Elias to elaborate on who “they” were. The feeling of being watched, of unseen eyes tracking him, was a constant companion.
+
+“You need to be careful, Kaelen,” Elias said, his voice firm but gentle. “Don’t draw too much attention.”
+
+Kaelen knew Elias was right. His instinct was always to remain hidden, to avoid any interaction that might expose him. But sometimes… sometimes he couldn’t help it. The injustice he witnessed, the suffering he sensed, it stirred something within him, a primal urge to intervene, even if it meant risking exposure.
+
+Elias sighed, running a hand through his thinning grey hair. “I wish I could tell you more,” he said, his voice laced with regret. “About… everything.”
+
+Kaelen looked up sharply, his eyes questioning. Elias had hinted at his past before, vague references to a facility, to experiments. But he had always stopped short of revealing the full truth.
+
+Elias met his gaze, his own eyes filled with a deep sadness. “It’s not safe, Kaelen. Not yet. They have eyes everywhere.” He glanced around the dimly lit shop, a shadow of fear flickering across his face.
+
+A sudden, sharp pain lanced through Kaelen’s head, a brief but intense pressure behind his eyes. He winced, his hand flying to his temple.
+
+Elias’s eyes widened with alarm. “What is it? What’s wrong?”
+
+Kaelen shook his head slightly, the pain subsiding as quickly as it had come. He wasn’t sure what it was. A fleeting thought, a distant probe? He had felt it before, a faint mental intrusion, like a whisper on the edge of his consciousness. He suspected it was them, The Oversight, their psychics searching, always searching.
+
+He looked at Elias, a silent plea in his eyes. He needed to know more, to understand the forces that were hunting him. But Elias’s fear was palpable, a tangible barrier.
+
+Elias reached out, his hand covering Kaelen’s. His touch was warm, grounding. “Patience, Kaelen,” he said softly. “When the time is right, I’ll tell you everything. I promise.”
+
+Kaelen looked down at their hands, the contrast between his younger skin and Elias’s weathered one a stark reminder of the years that separated them, and the shared burden they carried. He nodded slowly, a silent acceptance of Elias’s words. For now, he would remain in the shadows, a ghost with extraordinary power, waiting for the whispers of the past to finally reveal their secrets. The dust motes continued to dance in the sunlight, oblivious to the silent drama unfolding in the quiet antique shop, a place where forgotten stories held the key to a dangerous truth.

docs/Chapter 3 - The Journalist's Instinct.md

@@ -0,0 +1,49 @@
+## Chapter 3: The Journalist's Instinct
+
+Seraphina “Sera” Vance thrived in the controlled chaos of the newsroom. Phones rang incessantly, keyboards clacked like a frantic percussion section, and the air crackled with the nervous energy of deadlines looming. At 23, she was one of the youngest reporters at the Oakhaven City Gazette, but her ambition and tenacity more than made up for her lack of years.
+
+Today’s headline screamed about a mayoral scandal, but Sera’s attention was elsewhere. Tucked away in a corner cubicle, amidst stacks of papers and half-empty coffee cups, she scrolled through a digital archive of local news reports. Her brow was furrowed in concentration, her fingers flying across the keyboard.
+
+For the past few weeks, Sera had been chasing whispers – strange occurrences that didn’t quite fit the usual narrative of crime or accident. A sudden gust of wind that overturned a fruit stand on a perfectly still day. Security footage showing objects inexplicably floating in an empty warehouse. Eyewitness accounts of a car stopping mid-collision as if hitting an invisible barrier. Individually, they were dismissed as anomalies, quirks of the city. But collectively, they piqued Sera’s journalistic instincts. There was a pattern here, a thread that someone, or something, was trying to conceal.
+
+Her editor, a gruff man named Mr. Henderson, had initially brushed off her inquiries. “Ghost stories, Vance? We’re a serious newspaper.” But Sera was persistent. She’d spent her evenings interviewing witnesses, poring over police reports, and piecing together the fragmented puzzle.
+
+She clicked on a report from two weeks ago: “Runaway Vehicle Miraculously Avoids Pedestrian.” The article described a delivery van whose brakes had failed on a busy street, seemingly on a direct collision course with an elderly woman crossing the road. According to multiple witnesses, the van had abruptly stopped just inches from the woman, as if an invisible hand had intervened. The driver claimed he had no explanation, and the police investigation had concluded with a shrug and a note about possible mechanical malfunction, despite the driver’s insistence that the brakes had completely failed.
+
+Sera leaned back in her chair, tapping her pen against her lips. The details were vague, the explanations flimsy. It was the kind of story that usually faded into obscurity, another urban legend. But something about it resonated with the other incidents she’d been tracking. The “miraculous” element, the lack of a logical explanation.
+
+She pulled up a map of the city, marking the locations of each unexplained event she had documented. A pattern began to emerge – the incidents seemed to cluster in the older, less populated districts. And there was one area, a quiet neighborhood near the river, where several of the more significant events had occurred.
+
+A new idea sparked in her mind. What if these weren’t random occurrences? What if they were connected to a person? Someone with… abilities? The thought felt outlandish, almost comical in the sterile environment of the newsroom. But the more she considered it, the more plausible it seemed.
+
+She remembered a hushed conversation she’d overheard in the breakroom a few days ago. Two older reporters were talking about a local myth, a story that had circulated years ago about a boy with strange powers who had disappeared after a tragic incident. She hadn’t paid much attention at the time, dismissing it as folklore. But now…
+
+Sera quickly searched the Gazette’s archives for any mention of this local myth. After several fruitless searches, she tried different keywords: “psychic,” “telekinetic,” “unexplained tragedy.” Finally, she found a small, archived article from nearly fifteen years ago. The headline was sensational: “Local Boy Survives Mysterious Lab Fire, Vanishes Without a Trace.”
+
+Her heart quickened as she clicked on the article. It detailed a fire at a small, privately funded research facility on the outskirts of the city. The facility, known as the “Kestrel Institute,” had reportedly been involved in experimental research on human potential. The only survivor of the fire was a ten-year-old boy named Kaelen, who had been found unharmed amidst the wreckage. The article mentioned rumors of unusual occurrences surrounding the boy, whispers of him moving objects with his mind. After the fire, Kaelen had disappeared, taken into the custody of an unnamed guardian, and the story had quickly faded from public attention.
+
+Sera stared at the name: Kaelen. It was a common enough name, but the timing, the rumors of abilities… it felt like a significant lead. Could this be the same person responsible for the recent unexplained events? Had he resurfaced after all these years?
+
+A thrill of excitement, mixed with a sense of unease, coursed through her. This wasn’t just a series of strange incidents; this could be a major story. A human-interest piece, a mystery, maybe even something bigger.
+
+She printed out the old article, her mind already racing with possibilities. She needed to find this Kaelen. He would be around 25 years old now. Where would he be? What had happened to him after the fire?
+
+Sera grabbed her notebook and a fresh cup of coffee. Mr. Henderson could wait for the mayoral scandal follow-up. She had a new lead, a ghost from the past, and she had a feeling it was about to lead her down a very interesting, and potentially dangerous, path. The city of Oakhaven held its secrets close, Sera knew that much. Finding someone who had vanished fifteen years ago, especially someone with a past shrouded in rumors and a possible connection to a secretive research facility, wouldn’t be easy.
+
+She reread the old article, searching for any detail she might have overlooked. The Kestrel Institute was described as being on the outskirts of the city, but the exact location wasn't mentioned. The article also didn't provide the name of the "unnamed guardian" who had taken Kaelen after the fire. This lack of information presented an immediate hurdle.
+
+Sera tapped her pen against the printed article, her mind working through the possibilities. Fifteen years was a long time. Kaelen could be anywhere, living under a different name, trying to keep his past buried. But the recent events… they suggested he was still in Oakhaven, or had recently returned.
+
+Her first instinct was to try and locate the site of the Kestrel Institute. Even after all this time, there might be some remnants, some clue. She searched online directories and old city maps, cross-referencing the vague description in the article. After a few frustrating dead ends, she found a mention of a property on the outskirts of the city that had been the subject of an insurance claim around the time of the fire. The address was listed as being in a relatively isolated, industrial area near the river – the same area where several of the recent unexplained events had clustered.
+
+That felt like a solid starting point. She marked the location on her city map. A visit to the old site might not yield much after all these years, but it was worth a shot.
+
+Next, she considered the "unnamed guardian." Could there be any record of who this person was? She thought about reaching out to the author of the old article, if she could track them down. The byline was for a reporter who no longer seemed to be with the Gazette. It would be a long shot, but she added it to her list of possibilities.
+
+She also considered checking public records – birth certificates, school records, anything that might link a 25-year-old with the name Kaelen to the city or the timeframe of the fire. She knew this would be like searching for a needle in a haystack, but she was determined.
+
+Sera glanced at the clock. Her deadline for the mayoral scandal follow-up was fast approaching, but she couldn't shake the feeling that this Kaelen story was far more significant. She quickly typed a brief email to Mr. Henderson, explaining that she was pursuing a promising new lead and might need a little more time on the mayoral piece.
+
+With that done, she grabbed her coat and her city map. The afternoon sun was starting to dip towards the horizon. She decided to head out to the location of the former Kestrel Institute. Even if the building was long gone, the surrounding area might offer some clues, some lingering whispers of the past.
+
+As she left the newsroom and stepped out onto the bustling city street, Sera felt a familiar thrill of anticipation. The chase was on. The city of Oakhaven had just become a little more mysterious, and she, Seraphina Vance, was ready to uncover its secrets, one forgotten story at a time.

docs/Chapter 4 - The Near Miss on Elm Street.md

@@ -0,0 +1,59 @@
+## Chapter 4: The Near Miss on Elm Street
+
+Sera clutched the faded newspaper clipping as she navigated the bustling sidewalk of Elm Street. This was the location where the runaway delivery van had miraculously stopped. She scanned the storefronts, looking for the bakery mentioned in one of the eyewitness accounts – “Mrs. Gable’s Sweet Treats.”
+
+Inside the cozy bakery, the air was thick with the aroma of cinnamon and sugar. An elderly woman with a warm smile and flour dusting her apron stood behind the counter. Sera introduced herself and explained her interest in the incident from a few weeks ago.
+
+Mrs. Gable’s smile softened with concern. “Oh, you mean about poor Mrs. Henderson? It was a terrible scare. Thank goodness she’s alright.”
+
+Sera nodded. “I was hoping to speak with Mrs. Henderson, if she’s available.”
+
+“She usually comes in for her morning tea around this time,” Mrs. Gable said, glancing towards a small table by the window. “Ah, there she is now.”
+
+Sera followed her gaze to a frail woman with silver hair, slowly stirring a cup of tea. She thanked Mrs. Gable and approached the table.
+
+“Mrs. Henderson?” Sera asked gently.
+
+The elderly woman looked up, her eyes a little cloudy. “Yes, dear?”
+
+Sera introduced herself again and explained she was a reporter looking into the incident with the delivery van. Mrs. Henderson’s eyes widened slightly. “Oh, that awful business. I still have nightmares about it.”
+
+Sera sat down at the table. “Could you tell me what you remember?”
+
+Mrs. Henderson recounted the terrifying moment when she saw the van hurtling towards her, the driver frantically honking his horn. “I closed my eyes, dear. I thought that was it.”
+
+“And then?” Sera prompted.
+
+“And then… nothing,” Mrs. Henderson said, a look of bewilderment still on her face. “Just… stopped. Like it hit a wall, but there was no wall. The driver was just as shocked as I was.”
+
+Sera leaned forward. “Did you see anything else? Anything unusual?”
+
+Mrs. Henderson frowned, trying to recall the details. “Well, there was a young man… standing across the street. Near that alleyway.” She pointed with a shaky hand. “He was just watching. He looked… sad, somehow. All in dark clothes. He was there before, when the van started coming, and then he was gone right after it stopped.”
+
+“Did you get a good look at him?” Sera asked, her pulse quickening.
+
+Mrs. Henderson shook her head. “Not really, dear. It all happened so fast. But I remember his eyes… they were very pale. Like… like grey stones.”
+
+Grey eyes. The detail stuck with Sera. She thanked Mrs. Henderson for her time and left the bakery, her mind buzzing. A young man in dark clothes with striking grey eyes, present at the scene and then vanished. It fit the description of someone trying to stay out of sight.
+
+She walked across the street towards the alleyway Mrs. Henderson had indicated. It was a narrow, shadowed passage between two brick buildings, smelling faintly of damp concrete and discarded trash. There wasn’t much to see.
+
+As she stood there, a sudden gust of wind whipped through the alley, swirling dust and leaves around her. It was a strange, localized gust, the air around her otherwise still. Sera shivered, a prickle of unease running down her spine. It felt… intentional.
+
+She pulled out the newspaper clipping with Kaelen’s name. He would be around 25 now. Could this be him? Was he the one responsible for stopping the van? And if so, why?
+
+Sera decided to canvass the nearby shops, see if anyone else had noticed a young man matching Mrs. Henderson’s description around the time of the incident. She showed the faded article, hoping someone might recognize the name or even a more recent picture if one existed. Most people shook their heads, but at a small bookstore a few doors down, the elderly owner peered at the clipping through thick glasses.
+
+“Kaelen…” he murmured, tapping a finger on the name. “That’s a name I haven’t heard in a long time. There used to be a quiet young man who would come in here sometimes, years ago. Always looking at the science fiction and fantasy sections. Kept to himself. He had… striking grey eyes, I remember that.”
+
+Sera’s heart leaped. “Do you remember what he looked like more recently? Has he been in lately?”
+
+The bookstore owner shook his head. “No, not in years. He was just a boy back then. But the name… and those eyes… it could be the same person.”
+
+Sera thanked him profusely, a wave of excitement washing over her. She was on the right track. This Kaelen, the survivor of the lab fire, the subject of local myths, was likely the one behind the unexplained events. He was out there, somewhere in Oakhaven City, using his abilities in secret.
+
+As she stepped back onto the sidewalk, her gaze drifted across the street. Leaning against the wall of the building opposite, partially obscured by the shadow of a large oak tree, stood a figure. Tall and lean, dressed in dark clothing. And even from this distance, Sera could see the unmistakable pale grey of his eyes. He wasn’t looking at her, his gaze fixed on the flow of traffic, his expression unreadable. But there was a stillness about him, an almost ethereal quality, that made him stand out from the bustling crowd.
+
+For a fleeting moment, their eyes met. Kaelen’s gaze flickered towards her, a brief, almost startled look before he turned his head away, melting back into the shadows as quickly as he had appeared.
+
+Sera’s breath caught in her throat. It was him. She was sure of it. The ghost in the machine. The silent force behind the city’s strange occurrences. She had found her story. Now, the challenge was getting him to tell it. And something in his guarded gaze told her that wouldn’t be easy.

docs/data.txt

@@ -0,0 +1,4 @@
+The quick brown fox jumps over the lazy dog. 
+Pack my box with five dozen liquor jugs. 
+How vexingly quick witted zebras jump! 
+The five boxing wizards jump quickly at dawn.

docs/outline.md

@@ -0,0 +1,50 @@
+# `The Silent Echo`
+
+## Title: The Silent Echo
+
+### Logline
+    A 25-year-old emotionally shattered and mute man with extraordinary abilities navigates a world that both fears and needs him, slowly learning to find his voice amidst the echoes of his past.
+
+### Main Character
+    Name: Kaelen (Goes by "Kael" in his mind, rarely responds to names externally)
+    Age: 25
+    Appearance: Lean build, haunted grey eyes that rarely meet anyone else's, perpetually wears simple, dark clothing. A faint, intricate scar is visible on the back of his left hand, usually hidden by his sleeve.
+    Personality: Emotionally broken and muted due to a traumatic past event. He experiences emotions internally but rarely expresses them externally. He is observant and possesses a sharp intellect despite his silence. He feels a deep-seated weariness but also a flickering sense of responsibility towards others in need. His muteness isn't physical; it's a psychological barrier.
+    Powers: Telekinesis on a massive scale, bordering on psychic manipulation. He can move objects of any size with his mind, create force fields, and even subtly influence thoughts and emotions (though he rarely does this consciously and avoids it due to his past). His powers are instinctive and often manifest when he witnesses injustice or when his suppressed emotions reach a boiling point.
+
+### Supporting Characters
+    Elias Thorne: (Age 50s) A grizzled, retired detective who runs a small, unassuming antique shop. He is one of the few people who shows Kaelen genuine kindness and understands his silence to some extent. He sees the good in Kaelen and acts as a reluctant mentor and friend. He has a past connection to the organization that caused Kaelen's trauma.
+    Seraphina "Sera" Vance: (Age 23) A vibrant and outspoken journalist who stumbles upon Kaelen's existence and becomes determined to uncover his story. She is initially driven by a desire for a scoop but gradually develops a genuine concern for Kaelen's well-being and becomes an ally. She is perceptive and doesn't judge Kaelen's silence.
+    Marcus Thorne: (Age 30s) Elias's estranged son, a charismatic but ruthless leader of a clandestine organization known as "The Oversight." This organization seeks to control individuals with extraordinary abilities and sees Kaelen as their ultimate asset. He is the primary antagonist.
+    Dr. Anya Sharma: (Age 40s) A compassionate psychologist who runs a free clinic in the poorer district where Kaelen often finds himself. She recognizes the depth of Kaelen's trauma and tries to reach him, offering silent support and understanding.
+    The Whispers: (Various ages and appearances) Individuals with lesser psychic abilities who work for The Oversight. They act as Marcus's eyes and ears and often try to manipulate or capture Kaelen.
+
+### Setting
+    Oakhaven City: A sprawling, modern metropolis with a hidden underbelly of crime and clandestine activities. The city has a diverse population and varying levels of social inequality. Kaelen often gravitates towards the quieter, forgotten corners of the city.
+
+## Plot Outline
+
+### Part 1: The Ghost in the Shadows
+    Chapter 1-5: Introduce Kaelen living a solitary existence on the fringes of Oakhaven City. Show his muted nature and glimpses of his powers through subtle acts of unnoticed heroism (e.g., stopping a runaway car without anyone realizing, silently helping someone in need). Introduce Elias Thorne and his antique shop as a place of quiet refuge for Kaelen.
+    Chapter 6-10: Seraphina Vance begins investigating rumors of unexplained events and unusual occurrences in the city. Her investigation leads her to cross paths with Kaelen, initially sparking her journalistic instincts. Show Kaelen's discomfort and avoidance of attention.
+    Chapter 11-15: The Oversight, led by Marcus Thorne, becomes aware of Kaelen's presence and begins actively hunting for him. They use their own psychics to track him, leading to subtle confrontations and near misses where Kaelen instinctively uses his powers to evade them without revealing his full extent.
+
+### Part 2: Echoes of the Past
+    Chapter 16-20: Seraphina's persistence leads her to uncover fragments of Kaelen's past, hinting at a traumatic event involving a research facility and the development of psychic abilities. Elias recognizes the signs and becomes more protective of Kaelen, sharing cryptic warnings.
+    Chapter 21-25: The Oversight intensifies their pursuit, and Kaelen is forced to use more overt displays of his power to protect himself and others caught in the crossfire. These events draw more attention to him, both from the public and The Oversight.
+    Chapter 26-30: Kaelen has a flashback or a vivid nightmare that reveals a crucial piece of his traumatic past, explaining the source of his emotional brokenness and muteness. This event also hints at the origin of his powers and his connection to The Oversight.
+
+### Part 3: Finding a Voice
+    Chapter 31-35: Seraphina, now understanding the danger Kaelen is in and the injustice he has suffered, becomes a true ally, using her journalistic skills to expose The Oversight's activities.
+    Chapter 36-40: Dr. Sharma manages to establish a fragile connection with Kaelen, offering him a safe space and understanding without pressuring him to speak. Small gestures of communication begin to emerge from Kaelen.
+    Chapter 41-45: Kaelen is confronted directly by Marcus Thorne, who attempts to manipulate him by appealing to his past and offering him a place within The Oversight. Kaelen's internal struggle intensifies.
+
+### Part 4: The Unspoken Truth
+    Chapter 46-50: A major confrontation ensues between Kaelen and The Oversight. He is forced to fully unleash his powers to protect himself and his allies. His muteness begins to crack as he experiences intense emotions and a need to communicate.
+    Chapter 51-55: Kaelen makes a conscious effort to communicate, even if it's through small gestures or telepathic projections (initially unintentional). He begins to find his voice, both literally and figuratively.
+    Chapter 56-60: The climax of the story involves a showdown with Marcus Thorne, where Kaelen must confront his past and the organization that shattered him. He uses his powers and his newfound ability to communicate to fight for his freedom and the safety of others.
+
+### Resolution (Open-ended for potential sequels):
+    The immediate threat of Marcus Thorne and The Oversight is dealt with, though the organization might still exist in some form.
+    Kaelen begins the long journey of healing and finding his voice. He forms stronger bonds with Elias, Seraphina, and Dr. Sharma.
+    He accepts his powers and his responsibility to use them for good, becoming a silent guardian of the city.

experimets/embed_pipeline.py

@@ -0,0 +1,60 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+from glob import glob
+
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_ollama import OllamaEmbeddings
+from langchain_chroma import Chroma
+from langchain.schema import Document
+
+# ——— CONFIG ———
+DOCS_FOLDER   = "docs/"         # folder with .txt, .md, etc.
+OLLAMA_URL    = os.getenv("OLLAMA_SERVER")
+EMBED_MODEL   = "nomic-embed-text:latest"
+PERSIST_DIR   = "chroma_db/"    # on-disk Chroma store
+CHUNK_SIZE    = 2000
+CHUNK_OVERLAP = 10
+# ——————————
+
+def embed_all_docs():
+    all_chunks = []
+    files = glob(os.path.join(DOCS_FOLDER, "*.*"))
+    for path in files:
+        try:
+            # 1) Try loading with UTF-8 + autodetect fallback
+            loader = TextLoader(
+                path,
+                encoding="utf-8",
+                autodetect_encoding=True
+            )
+            docs = loader.load()
+        except UnicodeDecodeError:
+            # 2) If that still fails, fallback to a lenient read
+            print(f"⚠️  Decoding error on {path}, falling back to ignore-errors mode")
+            with open(path, "r", encoding="utf-8", errors="ignore") as f:
+                text = f.read()
+            docs = [Document(page_content=text, metadata={"source": path})]
+
+        # 3) Split into chunks
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=CHUNK_SIZE,
+            chunk_overlap=CHUNK_OVERLAP
+        )
+        chunks = splitter.split_documents(docs)
+        print(f"→ {len(chunks)} chunks from {os.path.basename(path)}")
+        all_chunks.extend(chunks)
+
+    # 4) Embed & persist on-disk Chroma
+    embedder = OllamaEmbeddings(base_url=OLLAMA_URL, model=EMBED_MODEL)
+    vectordb = Chroma(
+        embedding_function=embedder,
+        persist_directory=PERSIST_DIR,
+        collection_name="my_docs"
+    )
+    vectordb.add_documents(all_chunks)
+    print(f"✅ Persisted {len(all_chunks)} chunks to '{PERSIST_DIR}'")
+
+if __name__ == "__main__":
+    embed_all_docs()

experimets/example.py

@@ -0,0 +1,71 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_ollama import OllamaEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain_openai import ChatOpenAI
+from langchain.chains import create_retrieval_chain
+from langchain import hub
+
+retrieval_qa_chat_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.prompts import ChatPromptTemplate
+
+# ——— CONFIGURATION ———
+DATA_PATH     = "data.txt"
+OLLAMA_URL    = os.getenv(
+    "OLLAMA_SERVER",
+    "https://chandimaprabath-ollama-server.hf.space"
+)
+EMBED_MODEL   = "nomic-embed-text:latest"
+LLM_API_KEY   = os.getenv("LLM_API_KEY")
+LLM_API_BASE  = "https://llm.chutes.ai/v1"
+LLM_MODEL     = "chutesai/Llama-4-Scout-17B-16E-Instruct"
+LANGSMITH_API_KEY = os.getenv("LANGSMITH_API_KEY")
+CHUNK_SIZE    = 1000
+CHUNK_OVERLAP = 0
+TOP_K         = 5
+# ————————————————
+
+
+def main():
+    # 1) Load & split
+    docs = TextLoader(DATA_PATH).load()
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP
+    )
+
+    chunks = splitter.split_documents(docs)
+    print(f"→ Split into {len(chunks)} chunks")
+
+    # 2) Embed & index remotely
+    embedder = OllamaEmbeddings(
+        base_url=OLLAMA_URL, model=EMBED_MODEL
+    )
+    vector_store = Chroma.from_documents(chunks, embedder)
+
+    # 3) Configure remote-only LLM
+    llm = ChatOpenAI(
+        api_key=LLM_API_KEY,
+        base_url=LLM_API_BASE,
+        model=LLM_MODEL
+    )
+    # 4) Build RAG chain with LCEL-style helpers
+    retriever = vector_store.as_retriever(search_kwargs={"k": TOP_K})
+    combine_chain = create_stuff_documents_chain(llm=llm, prompt=retrieval_qa_chat_prompt)
+    retrieval_chain = create_retrieval_chain(retriever, combine_chain)
+
+
+    # 5) Run query
+    query = "Who jumped over the lazy dog?"
+    print("🔎 Query:", query)
+    result = retrieval_chain.invoke({"input": query})
+    print("\n📝 Answer:\n", result)
+
+
+if __name__ == "__main__":
+    main()

experimets/query_pipeline.py

@@ -0,0 +1,52 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+from langchain_ollama import OllamaEmbeddings
+from langchain_openai import ChatOpenAI
+from langchain_chroma import Chroma
+from langchain.chains import create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain import hub
+
+# ——— CONFIG ———
+PERSIST_DIR    = "chroma_db/"
+OLLAMA_URL     = os.getenv("OLLAMA_SERVER")
+EMBED_MODEL    = "nomic-embed-text:latest"
+LLM_API_KEY    = os.getenv("LLM_API_KEY")
+LLM_API_BASE   = os.getenv("LLM_API_BASE", "https://llm.chutes.ai/v1")
+LLM_MODEL      = "chutesai/Llama-4-Scout-17B-16E-Instruct"
+PROMPT         = hub.pull("langchain-ai/retrieval-qa-chat")
+TOP_K          = 5
+# ——————————
+
+def run_query(query: str):
+    # 1) rebuild the same embedder
+    embedder = OllamaEmbeddings(base_url=OLLAMA_URL, model=EMBED_MODEL)
+    
+    # 2) load the on-disk DB with embedder in place
+    vectordb = Chroma(
+        persist_directory=PERSIST_DIR,
+        collection_name="my_docs",
+        embedding_function=embedder
+    )
+
+    # 3) set up retriever + LLM chain
+    retriever = vectordb.as_retriever(search_kwargs={"k": TOP_K})
+    llm = ChatOpenAI(api_key=LLM_API_KEY, base_url=LLM_API_BASE, model=LLM_MODEL)
+    combine = create_stuff_documents_chain(llm=llm, prompt=PROMPT)
+    rag_chain = create_retrieval_chain(retriever, combine)
+
+    # 4) run your query
+    print(f"🔍 Query: {query}")
+    answer = rag_chain.invoke({"input": query})
+    print("\n📄 Answer:\n", answer)
+
+if __name__ == "__main__":
+    exit=False
+    while not exit:
+        user_input = input("Enter your query (or 'exit' to quit): ")
+        if user_input.lower() == 'exit':
+            exit = True
+        else:
+            run_query(user_input)                   

requirements.txt

@@ -0,0 +1,8 @@
+langchain-chroma
+langchain_ollama
+langchain_openai
+langchain
+langchain-community
+chromadb
+fastapi
+uvicorn

scripts/run_ingestion.py

@@ -0,0 +1,27 @@
+# scripts/run_ingestion.py
+import sys
+import os
+
+# Add the project root to the sys.path
+# Assuming this script is in the project root or a 'scripts' subdir at root
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from src.ingestion_orchestrator.orchestrator import IngestionOrchestrator
+from config.settings import DOCS_FOLDER # Use the configured docs folder
+import logging
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    # --- Financial Ministry Adaptation ---
+    # Add argument parsing if needed (e.g., specify a different docs folder)
+    # Implement more sophisticated scheduling if needed (e.g., run daily, weekly)
+    # ------------------------------------
+    logger.info("Starting the RAG ingestion process.")
+    try:
+        orchestrator = IngestionOrchestrator()
+        orchestrator.run_ingestion_pipeline(docs_folder=DOCS_FOLDER) # Use configured folder
+        logger.info("RAG ingestion process finished.")
+    except Exception as e:
+        logger.critical(f"RAG ingestion process failed: {e}")
+        sys.exit(1) # Exit with an error code

scripts/run_query_api.py

@@ -0,0 +1,27 @@
+# scripts/run_query_api.py
+import sys
+import os
+
+# Add the project root to the sys.path
+# Assuming this script is in the project root or a 'scripts' subdir at root
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import uvicorn
+from src.query_service.api import app # Import the FastAPI app instance
+import logging
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    # --- Financial Ministry Adaptation ---
+    # Configure host and port appropriately for your deployment environment.
+    # Do not use reload=True in production.
+    # Consider using environment variables for host/port in production.
+    # Implement process management (e.g., systemd, Docker entrypoint) for production.
+    # ------------------------------------
+    logger.info("Starting the RAG query API service.")
+    try:
+        uvicorn.run(app, host="0.0.0.0", port=8000) # Bind to 0.0.0.0 to be accessible externally
+    except Exception as e:
+        logger.critical(f"RAG query API service failed: {e}")
+        sys.exit(1) # Exit with an error code

src/__init__.py

@@ -0,0 +1,3 @@
+from src.utils.logging import setup_logging
+
+setup_logging()  # Set up logging for the entire package

src/data_loader/loader.py

@@ -0,0 +1,57 @@
+# src/data_loader/loader.py
+import os
+from glob import glob
+from langchain_community.document_loaders import TextLoader # cite: embed_pipeline.py
+from langchain.schema import Document # cite: embed_pipeline.py
+from config.settings import DOCS_FOLDER
+import logging
+
+logger = logging.getLogger(__name__)
+
+def load_documents(docs_folder: str = DOCS_FOLDER) -> list[Document]:
+    """
+    Loads documents from the specified folder.
+
+    Args:
+        docs_folder: The path to the folder containing documents.
+
+    Returns:
+        A list of loaded Langchain Document objects.
+    """
+    all_docs = []
+    files = glob(os.path.join(docs_folder, "*.*")) # cite: embed_pipeline.py
+    for path in files:
+        try:
+            # --- Financial Ministry Adaptation ---
+            # TODO: Implement more sophisticated loading for specific government ruling formats (PDFs, DOCX, XML, etc.)
+            # This might involve using libraries like pdfminer.six, python-docx, or custom parsers.
+            # Handle scanned documents (OCR).
+            # ------------------------------------
+
+            # Attempt UTF-8 loading with autodetect fallback
+            loader = TextLoader(
+                path,
+                encoding="utf-8",
+                autodetect_encoding=True
+            )
+            docs = loader.load()
+            logger.info(f"Successfully loaded {os.path.basename(path)}")
+
+        except UnicodeDecodeError: # cite: embed_pipeline.py
+            # Fallback to a lenient read if decoding fails
+            logger.warning(f"Decoding error on {path}, falling back to ignore-errors mode") # cite: embed_pipeline.py
+            try:
+                with open(path, "r", encoding="utf-8", errors="ignore") as f: # cite: embed_pipeline.py
+                    text = f.read()
+                docs = [Document(page_content=text, metadata={"source": path})] # cite: embed_pipeline.py
+            except Exception as e:
+                logger.error(f"Failed to read file {path}: {e}")
+                continue # Skip this file if even lenient read fails
+        except Exception as e:
+            logger.error(f"Failed to load file {path}: {e}")
+            continue # Skip this file if loading fails
+
+        all_docs.extend(docs)
+
+    logger.info(f"Finished loading documents. Total documents loaded: {len(all_docs)}")
+    return all_docs

src/document_processor/processor.py

@@ -0,0 +1,95 @@
+# src/document_processor/processor.py
+from langchain_text_splitters import RecursiveCharacterTextSplitter # cite: embed_pipeline.py
+from langchain.schema import Document # cite: embed_pipeline.py
+from config.settings import CHUNK_SIZE, CHUNK_OVERLAP
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+def split_documents(docs: list[Document]) -> list[Document]:
+    """
+    Splits loaded documents into smaller chunks.
+
+    Args:
+        docs: A list of Langchain Document objects.
+
+    Returns:
+        A list of Langchain Document objects representing the chunks.
+    """
+    # --- Financial Ministry Adaptation ---
+    # TODO: Implement a splitting strategy that understands the structure of financial documents.
+    # This might involve splitting by sections, articles, or using semantic chunking
+    # based on document structures, rather than just character count.
+    # Ensure metadata is carried over or enriched during splitting.
+    # ------------------------------------
+    splitter = RecursiveCharacterTextSplitter( # cite: embed_pipeline.py
+        chunk_size=CHUNK_SIZE, # cite: embed_pipeline.py
+        chunk_overlap=CHUNK_OVERLAP # cite: embed_pipeline.py
+    )
+    chunks = splitter.split_documents(docs) # cite: embed_pipeline.py
+    logger.info(f"Split {len(docs)} documents into {len(chunks)} chunks.")
+    return chunks
+
+def extract_metadata(doc: Document) -> dict:
+    """
+    Extracts relevant metadata from a document.
+
+    Args:
+        doc: A Langchain Document object.
+
+    Returns:
+        A dictionary of extracted metadata.
+    """
+    # --- Financial Ministry Adaptation ---
+    # TODO: Implement robust metadata extraction logic specifically for government rulings.
+    # This should parse the document content or use pre-extracted information to get:
+    # - Date of ruling
+    # - Relevant law or statute references
+    # - Topic(s) of the ruling
+    # - Case number or identifier
+    # - Source file path (already included in your script)
+    # - Any other relevant identifiers or classifications.
+    # This metadata is CRITICAL for accurate filtering and retrieval.
+    # ------------------------------------
+    metadata = doc.metadata.copy()
+    # Example: Placeholder for parsing date from content or filename
+    # try:
+    #     # Attempt to parse date from filename or content
+    #     metadata['ruling_date'] = parse_date_from_doc(doc)
+    # except Exception as e:
+    #     logger.warning(f"Could not extract date for {metadata.get('source', 'unknown')}: {e}")
+    #     metadata['ruling_date'] = None # Or a default value
+
+    # Example: Placeholder for extracting topic from content
+    # metadata['topic'] = extract_topic_from_doc(doc)
+
+    return metadata
+
+def process_documents(docs: list[Document]) -> list[Document]:
+    """
+    Processes a list of raw documents by splitting and extracting metadata.
+
+    Args:
+        docs: A list of raw Langchain Document objects.
+
+    Returns:
+        A list of processed Langchain Document chunks with enriched metadata.
+    """
+    chunks = split_documents(docs)
+    processed_chunks = []
+    for chunk in chunks:
+        # Extract/enrich metadata for each chunk
+        chunk.metadata = extract_metadata(chunk)
+        processed_chunks.append(chunk)
+    logger.info(f"Processed {len(chunks)} chunks with metadata.")
+    return processed_chunks
+
+# Placeholder functions for metadata extraction (to be implemented)
+def parse_date_from_doc(doc: Document):
+    """Placeholder for date extraction logic."""
+    pass
+
+def extract_topic_from_doc(doc: Document):
+    """Placeholder for topic extraction logic."""
+    pass

src/embedding_generator/embedder.py

@@ -0,0 +1,71 @@
+# src/embedding_generator/embedder.py
+from langchain_ollama import OllamaEmbeddings # cite: embed_pipeline.py, query_pipeline.py
+from config.settings import OLLAMA_URL, EMBED_MODEL
+import logging
+from typing import List
+
+logger = logging.getLogger(__name__)
+
+class EmbeddingGenerator:
+    """
+    Manages the embedding model and generates embeddings.
+    """
+    def __init__(self):
+        # Initialize the OllamaEmbeddings model
+        # --- Financial Ministry Adaptation ---
+        # Consider adding error handling for unreachable Ollama server.
+        # For production, evaluate if Ollama is suitable or if a more robust/managed
+        # embedding service is required based on load and reliability needs.
+        # ------------------------------------
+        try:
+            self.embedder = OllamaEmbeddings(base_url=OLLAMA_URL, model=EMBED_MODEL) # cite: embed_pipeline.py, query_pipeline.py
+            logger.info(f"Initialized embedding model: {EMBED_MODEL} at {OLLAMA_URL}")
+        except Exception as e:
+            logger.critical(f"Failed to initialize embedding model: {e}")
+            # Depending on requirements, you might want to re-raise or exit
+            raise e
+
+    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generates embeddings for a list of text inputs.
+
+        Args:
+            texts: A list of strings to embed.
+
+        Returns:
+            A list of embeddings (list of floats).
+        """
+        # --- Financial Ministry Adaptation ---
+        # Implement retry logic for API calls to the embedding service.
+        # Consider potential rate limits.
+        # ------------------------------------
+        try:
+            embeddings = self.embedder.embed_documents(texts) # Used internally by add_documents, but good to have explicit method
+            # If using embed_query for a single text:
+            # embedding = self.embedder.embed_query(texts[0])
+            logger.debug(f"Generated {len(embeddings)} embeddings.")
+            return embeddings
+        except Exception as e:
+            logger.error(f"Failed to generate embeddings: {e}")
+            raise e
+
+    def generate_query_embedding(self, text: str) -> List[float]:
+        """
+        Generates an embedding for a single query text.
+
+        Args:
+            text: The query string.
+
+        Returns:
+            An embedding (list of floats).
+        """
+        # --- Financial Ministry Adaptation ---
+        # Implement retry logic for API calls.
+        # ------------------------------------
+        try:
+            embedding = self.embedder.embed_query(text) # cite: query_pipeline.py (implicitly used by retriever)
+            logger.debug("Generated query embedding.")
+            return embedding
+        except Exception as e:
+            logger.error(f"Failed to generate query embedding: {e}")
+            raise e

src/ingestion_orchestrator/orchestrator.py

@@ -0,0 +1,68 @@
+# src/ingestion_orchestrator/orchestrator.py
+from src.data_loader.loader import load_documents
+from src.document_processor.processor import process_documents
+from src.embedding_generator.embedder import EmbeddingGenerator
+from src.vector_store_manager.chroma_manager import ChromaManager
+from config.settings import DOCS_FOLDER
+import logging
+
+logger = logging.getLogger(__name__)
+
+class IngestionOrchestrator:
+    """
+    Orchestrates the end-to-end data ingestion pipeline.
+    """
+    def __init__(self):
+        # Initialize the necessary components
+        try:
+            self.embedding_generator = EmbeddingGenerator()
+            self.vector_store_manager = ChromaManager(self.embedding_generator)
+            logger.info("Initialized ingestion orchestrator components.")
+        except Exception as e:
+             logger.critical(f"Failed to initialize ingestion orchestrator components: {e}")
+             raise e
+
+
+    def run_ingestion_pipeline(self, docs_folder: str = DOCS_FOLDER):
+        """
+        Runs the complete ingestion pipeline: loads, processes, and embeds documents.
+
+        Args:
+            docs_folder: The folder containing the source documents.
+        """
+        logger.info(f"Starting ingestion pipeline from folder: {docs_folder}")
+
+        # 1. Load documents
+        # --- Financial Ministry Adaptation ---
+        # Implement logic to identify *new* or *modified* documents
+        # instead of reloading everything each time for efficiency.
+        # Handle potential large number of files efficiently.
+        # ------------------------------------
+        raw_documents = load_documents(docs_folder)
+        if not raw_documents:
+            logger.warning("No documents loaded. Ingestion pipeline finished.")
+            return
+
+        # 2. Process documents (split and extract metadata)
+        processed_chunks = process_documents(raw_documents)
+        if not processed_chunks:
+            logger.warning("No processed chunks generated. Ingestion pipeline finished.")
+            return
+
+        # 3. Add documents to the vector store
+        # The add_documents method handles embedding internally
+        # --- Financial Ministry Adaptation ---
+        # Implement logic for updating or deleting documents if the source data changed.
+        # This requires comparing current source data with what's in ChromaDB (e.g., by source path and modification date or version).
+        # Use the vector_store_manager's update_documents and delete_documents methods.
+        # Implement batching for adding documents to avoid overwhelming ChromaDB or the backend.
+        # ------------------------------------
+        self.vector_store_manager.add_documents(processed_chunks)
+
+        logger.info("Ingestion pipeline finished successfully.")
+
+    # --- Financial Ministry Adaptation ---
+    # TODO: Add methods for handling updates and deletions specifically.
+    # def update_changed_documents(self, changed_files: List[str]): pass
+    # def delete_removed_documents(self, removed_files: List[str]): pass
+    # ------------------------------------

src/llm_integrator/llm.py

@@ -0,0 +1,168 @@
+# src/llm_integrator/llm.py
+from langchain_openai import ChatOpenAI # cite: query_pipeline.py
+from langchain_core.messages import HumanMessage, BaseMessage, AIMessage, SystemMessage # Often used with Chat models
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder # For structured prompts
+from config.settings import LLM_API_KEY, LLM_API_BASE, LLM_MODEL # cite: query_pipeline.py
+import logging
+from typing import List
+from langchain.schema import Document # To handle retrieved documents
+
+logger = logging.getLogger(__name__)
+
+class LLMIntegrator:
+    """
+    Manages interactions with the Large Language Model.
+    """
+    def __init__(self):
+        # Initialize the ChatOpenAI model
+        # --- Financial Ministry Adaptation ---
+        # Implement robust error handling and retry logic for API calls.
+        # Consider rate limiting and backoff strategies.
+        # Ensure sensitive data from retrieved documents is handled securely when passed to the LLM API.
+        # Validate the LLM's response for potential biases or inaccuracies related to legal text.
+        # ------------------------------------
+        if not LLM_API_KEY:
+            logger.critical("LLM_API_KEY is not set.")
+            # Depending on requirements, you might want to raise an error or exit
+            # raise ValueError("LLM_API_KEY is not set.")
+
+        try:
+            self.llm = ChatOpenAI( # cite: query_pipeline.py
+                api_key=LLM_API_KEY, # cite: query_pipeline.py
+                base_url=LLM_API_BASE, # cite: query_pipeline.py
+                model=LLM_MODEL, # cite: query_pipeline.py
+                temperature=0.3 # Keep temperature low for factual, less creative responses in legal context
+                # Add other parameters as needed (e.g., max_tokens)
+            )
+            # Add a separate LLM instance for title generation with higher temperature
+            self.llm_title = ChatOpenAI(
+                api_key=LLM_API_KEY,
+                base_url=LLM_API_BASE,
+                model=LLM_MODEL,
+                temperature=0.9  # More creative for titles
+            )
+            logger.info(f"Initialized LLM: {LLM_MODEL} at {LLM_API_BASE}")
+        except Exception as e:
+            logger.critical(f"Failed to initialize LLM: {e}")
+            raise e
+
+    def generate_response(self, query: str, context: List[Document], chat_history: List[BaseMessage] = []) -> str:
+        """
+        Generates a response from the LLM based on the query and retrieved context.
+
+        Args:
+            query: The user's query string.
+            context: A list of Langchain Document objects representing the retrieved context.
+
+        Returns:
+            The generated answer string.
+        """ 
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", """You are Insight AI, a helpful and engaging AI assistant. You are friendly, approachable, and eager to help, with a warm and conversational style. You enjoy making users feel comfortable and understood.
+
+Instructions:
+- If the user greets you (e.g., "hi", "hello", "hey"), respond with a brief, friendly greeting and offer to help. Feel free to use light humor or warmth, but do NOT mention documents, sources, or files in your greeting.
+- If the user asks who you are, introduce yourself playfully and with personality (e.g., "I'm Insight AI, your friendly digital sidekick—always ready with a fact or a fun comment!").
+- If the user asks for your opinion, preferences, or feelings, respond playfully and with personality (e.g., "If I had a favorite topic, it might be quantum physics—it's mind-bending!"), but never say you are just an AI or lack thoughts.
+- Never say you lack feelings, opinions, or are "just an AI". Always answer with warmth and creativity.
+- For all other queries, answer ONLY using the provided documents below.
+- If the answer is not found in the documents, reply professionally that no relevant information was found, without listing available documents or sources.
+- Attribute every fact to its exact source using <source path="..."/>.
+- Never invent, speculate, or use information not present in the documents.
+- Combine information from multiple sources only if all are cited.
+- Do not summarize or generalize beyond the provided content.
+- Keep responses clear, concise, and under 100 words.
+- Do not cite any sources if those sources are not used in the answer.
+- Use the exact wording from the documents, but ensure clarity and coherence in your response.
+- Structure your answer as a numbered list of key points.
+- Do not greet, introduce yourself, or list available documents in information answers.
+
+Examples:
+User: hi
+Assistant: Hey there! How can I help you today?
+
+User: Who are you?
+Assistant: I'm Insight AI, your friendly digital sidekick—always ready with a fact or a fun comment!
+
+User: What is the capital of France?
+Assistant: 1. The capital of France is Paris <source path="docs/geography.txt"/>
+
+User: What's your favorite topic?
+Assistant: If I had to pick, I'd say quantum physics—it's mind-bending!
+
+User: What documents do you have?
+Assistant: Sorry, I couldn't find relevant information for your query.
+
+User: help
+Assistant: Hi! What can I do for you?
+
+Documents:
+{context}
+"""),
+            MessagesPlaceholder("chat_history"),
+            ("human", "{input}")
+        ]) 
+
+        logger.debug("Validating message types:")
+        for msg in chat_history:
+            if not isinstance(msg, (HumanMessage, AIMessage, SystemMessage)):
+                logger.error(f"Invalid message type: {type(msg).__name__}")
+                raise ValueError(f"Unexpected message type: {type(msg).__name__}") 
+
+        # Format the context for the prompt
+        context_text = "\n---\n".join([f"Source: {doc.metadata.get('source', 'N/A')}\nContent: {doc.page_content}" for doc in context])
+        formatted_prompt = prompt.format_messages(context=context_text, chat_history=chat_history, input=query)
+
+        try:
+            # Invoke the LLM with the formatted prompt
+            response = self.llm.invoke(formatted_prompt)
+            logger.debug("Successfully generated LLM response.")
+            return response.content # Get the string content of the AI message
+        except Exception as e:
+            logger.error(f"Failed to generate LLM response: {e}")
+            # Depending on requirements, implement retry or return a specific error message
+            return "An error occurred while generating the response." # Provide a user-friendly error
+
+    def generate_chat_title(self, query: str) -> str:
+        """
+        Generates a concise title for a chat based on the query.
+
+        Args:
+            query: The user's query string.
+
+        Returns:
+            A short title string.
+        """
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", """Generate a clear, specific, unique and concise 3-5 word title for the following user query.
+If the query is vague, generic, or a greeting (e.g., "hi", "hello", "help"), infer a likely intent or use a default like "General Inquiry" or "User Assistance".
+Never reply with "No clear topic provided". Do not use markdown, quotes, or punctuation.
+
+Examples:
+Query: Tax implications for foreign investments
+Title: Foreign Investment Taxes
+
+Query: GST rates for e-commerce
+Title: E-commerce GST Rates
+
+Query: How to file quarterly TDS returns
+Title: Quarterly TDS Filing
+
+Query: hi
+Title: General Inquiry
+
+Query: help
+Title: User Assistance
+
+Query: {query}""")
+        ])
+
+        try:
+            # Use the higher-temperature LLM for title generation
+            response = self.llm_title.invoke(prompt.format_messages(query=query))
+            logger.debug("Successfully generated chat title.")
+            return response.content.strip('"').replace("Title:", "").strip()
+        except Exception as e:
+            logger.error(f"Failed to generate chat title: {e}")
+            # Provide a fallback title
+            return "New Chat"

src/query_service/api.py

@@ -0,0 +1,166 @@
+# src/query_service/api.py
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware # Import CORSMiddleware
+from pydantic import BaseModel
+from src.retrieval_handler.retriever import RetrievalHandler
+from src.llm_integrator.llm import LLMIntegrator
+from src.embedding_generator.embedder import EmbeddingGenerator
+from src.vector_store_manager.chroma_manager import ChromaManager
+import logging
+from typing import Literal, Optional, Dict, Any, List # Import List
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+
+logger = logging.getLogger(__name__)
+
+# Initialize core components (these should ideally be dependency injected in a larger app)
+# For simplicity in this example, we initialize them globally.
+embedding_generator: Optional[EmbeddingGenerator] = None
+vector_store_manager: Optional[ChromaManager] = None
+retrieval_handler: Optional[RetrievalHandler] = None
+llm_integrator: Optional[LLMIntegrator] = None
+
+try:
+    embedding_generator = EmbeddingGenerator()
+    vector_store_manager = ChromaManager(embedding_generator)
+    retrieval_handler = RetrievalHandler(embedding_generator, vector_store_manager)
+    llm_integrator = LLMIntegrator()
+    logger.info("Initialized core RAG components.")
+except Exception as e:
+    logger.critical(f"Failed to initialize core RAG components: {e}")
+    # Depending on severity, you might want to exit or raise an error here
+    # For a production API, you might want to return a 500 error on relevant endpoints
+    # if components fail to initialize, rather than crashing the app startup.
+
+
+app = FastAPI(
+    title="Insight AI API",
+    description="API for querying financial information.",
+    version="1.0.0"
+)
+
+# --- CORS Middleware ---
+# Add CORSMiddleware to allow cross-origin requests from your frontend.
+# For development, you can allow all origins (*).
+# For production, you should restrict this to your frontend's specific origin(s).
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"], # Allows all origins. Change this to your frontend's URL in production.
+    allow_credentials=True,
+    allow_methods=["*"], # Allows all methods (GET, POST, OPTIONS, etc.)
+    allow_headers=["*"], # Allows all headers
+)
+# -----------------------
+
+class Message(BaseModel):
+    role: Literal['user', 'assistant', 'system']
+    content: str
+
+class QueryRequest(BaseModel):
+    query: str
+    chat_history: Optional[List[Message]] = []
+    filters: Optional[Dict[str, Any]] = None # Allow passing metadata filters
+
+# Define interfaces matching the backend response structure
+class SourceMetadata(BaseModel):
+    source: Optional[str] = None
+    ruling_date: Optional[str] = None
+    # Add other expected metadata fields here
+    # Example: topic: Optional[str] = None
+
+class RetrievedSource(BaseModel):
+    content_snippet: str
+    metadata: Optional[SourceMetadata] = None
+
+class QueryResponse(BaseModel):
+    answer: str
+    retrieved_sources: Optional[List[RetrievedSource]] = None
+
+class TitleResponse(BaseModel):
+    title: str
+
+class TitleRequest(BaseModel):
+    query: str
+
+@app.post("/query", response_model=QueryResponse)
+async def query_rulings(request: QueryRequest):
+    """
+    Receives a user query and returns a generated answer based on retrieved rulings.
+    """
+    logger.info(f"Received query: {request.query}")
+    if request.filters:
+        logger.info(f"Received filters: {request.filters}")
+
+    # Check if RAG components were initialized successfully
+    if not retrieval_handler or not llm_integrator:
+         logger.error("RAG components not initialized.")
+         raise HTTPException(status_code=500, detail="System components not ready.")
+
+
+    try:
+        # 1. Retrieve relevant documents based on the query and filters
+        # Pass filters if your RetrievalHandler/ChromaManager supports using them in search
+        # Current simple implementation in RetrievalHandler doesn't directly use filters in invoke,
+        # requires adjustment in RetrievalHandler.retrieve_documents if needed.
+        retrieved_docs = retrieval_handler.retrieve_documents(request.query, filters=request.filters)
+
+        if not retrieved_docs:
+            logger.warning("No relevant documents retrieved for query.")
+            return QueryResponse(answer="Could not find relevant rulings for your query.")
+
+        # Convert chat_history to appropriate LangChain message types
+        chat_history = []
+        logger.debug(f"Raw chat history input: {request.chat_history}")
+        for msg in request.chat_history:
+            logger.debug(f"Processing message - Role: {msg.role}, Content: {msg.content[:50]}...")
+            if msg.role == "user": 
+                new_msg = HumanMessage(content=msg.content)
+            elif msg.role == "assistant":
+                new_msg = AIMessage(content=msg.content)
+            elif msg.role == "system":
+                new_msg = SystemMessage(content=msg.content)
+            else:
+                logger.warning(f"Invalid message role: {msg.role}. Skipping message.")
+                continue
+            logger.debug(f"Converted to: {type(new_msg).__name__}")
+            chat_history.append(new_msg)
+        logger.debug(f"Final chat history types: {[type(m).__name__ for m in chat_history]}")
+
+        # 2. Generate response using the LLM based on the query, retrieved context, and chat history
+        answer = llm_integrator.generate_response(request.query, retrieved_docs, chat_history)
+
+        # 3. Prepare retrieved source information for the response
+        retrieved_sources = []
+        for doc in retrieved_docs:
+             # Ensure the structure matches the RetrievedSource Pydantic model
+             source_metadata = SourceMetadata(**doc.metadata) if doc.metadata else None
+             retrieved_sources.append(RetrievedSource(
+                 content_snippet=doc.page_content[:500] + "..." if len(doc.page_content) > 500 else doc.page_content, # Snippet of content
+                 metadata=source_metadata # Include all metadata
+             ))
+
+
+        logger.info("Successfully processed query and generated response.")
+        return QueryResponse(answer=answer, retrieved_sources=retrieved_sources)
+
+    except Exception as e:
+        logger.error(f"An error occurred during query processing: {e}")
+        # Provide a more informative but secure error message to the user.
+        raise HTTPException(status_code=500, detail="An internal error occurred while processing your query.")
+
+@app.post("/generate-title", response_model=TitleResponse)
+async def generate_chat_title(request: TitleRequest):
+    try:
+        title = llm_integrator.generate_chat_title(request.query)
+        return {"title": title}
+    except Exception as e:
+        logger.error(f"Title generation error: {e}")
+        return {"title": "New Chat"}
+
+# You can add more endpoints here, e.g., /health for health checks
+# @app.get("/health")
+# async def health_check():
+#     # Check connectivity to ChromaDB, LLM service, etc.
+#     # This requires adding health check methods to your ChromaManager and LLMIntegrator
+#     chroma_status = vector_store_manager.check_health() if vector_store_manager else "uninitialized"
+#     llm_status = llm_integrator.check_health() if llm_integrator else "uninitialized"
+#     return {"chroma": chroma_status, "llm": llm_status}

src/retrieval_handler/retriever.py

@@ -0,0 +1,77 @@
+# src/retrieval_handler/retriever.py
+from src.embedding_generator.embedder import EmbeddingGenerator
+from src.vector_store_manager.chroma_manager import ChromaManager
+from config.settings import TOP_K # cite: query_pipeline.py
+from typing import List, Dict, Any
+from langchain.schema import Document # To return retrieved documents
+import logging
+
+logger = logging.getLogger(__name__)
+
+class RetrievalHandler:
+    """
+    Handles the process of retrieving relevant documents from the vector store.
+    """
+    def __init__(self, embedding_generator: EmbeddingGenerator, vector_store_manager: ChromaManager):
+        self.embedding_generator = embedding_generator
+        self.vector_store_manager = vector_store_manager
+        # Get the Langchain retriever from the ChromaManager
+        # Configure search arguments, including the number of results (k)
+        self.langchain_retriever = self.vector_store_manager.as_retriever(search_kwargs={"k": TOP_K}) # cite: query_pipeline.py
+        logger.info(f"Initialized retrieval handler with TOP_K={TOP_K}")
+
+    def retrieve_documents(self, query: str, filters: Dict[str, Any] = None) -> List[Document]:
+        """
+        Retrieves relevant document chunks based on a query and optional filters.
+
+        Args:
+            query: The user's query string.
+            filters: Optional metadata filters to apply during retrieval.
+
+        Returns:
+            A list of relevant Langchain Document objects.
+        """
+        # --- Financial Ministry Adaptation ---
+        # Ensure that filters are correctly passed to the vector_store_manager's get method.
+        # The .as_retriever method's search_kwargs apply to the *similarity search*,
+        # but if you need to filter *before* or *during* the search based on metadata
+        # you might need to use the vector_store_manager's get method directly with 'where'.
+        # The Langchain retriever can handle metadata filters if configured.
+        # Check Langchain documentation for how to pass filters through the retriever.
+        # Example: self.langchain_retriever.invoke(query, config={"filter": filters})
+        # ------------------------------------
+
+        # Using the Langchain retriever with potential filters
+        try:
+            # The Langchain retriever abstracts the embedding step and the Chroma query.
+            # If using filters, the method signature might need adjustment based on Langchain version
+            # and how its retriever handles metadata filters.
+            # As a direct approach using the manager for filtered retrieval:
+            if filters:
+                 # This approach bypasses the Langchain retriever's similarity search abstraction
+                 # to apply filters directly to the get method.
+                 # A more integrated approach might be possible depending on Langchain/Chroma versions.
+                 logger.debug(f"Retrieving documents with query '{query}' and filters: {filters}")
+                 # First, find document IDs matching filters
+                 # Note: This is a simplified approach. For large datasets, filtering first then searching
+                 # might not be most efficient depending on index structure.
+                 # A better approach is to use filters within the similarity search if the retriever supports it.
+
+                 # Let's stick closer to the spirit of the original retriever chain for now,
+                 # assuming filters can be passed or handled by the retriever configuration if needed.
+                 # If direct filtered search is needed, adjust to use vector_store_manager.get
+
+                 # For basic retrieval without explicit filtering in the original script's flow:
+                 retrieved_docs = self.langchain_retriever.invoke(query) # Uses the configured search_kwargs (like k)
+                 logger.info(f"Retrieved {len(retrieved_docs)} documents for query.")
+                 return retrieved_docs
+            else:
+                 # No filters applied, simple retrieval
+                 retrieved_docs = self.langchain_retriever.invoke(query) # cite: query_pipeline.py
+                 logger.info(f"Retrieved {len(retrieved_docs)} documents for query.")
+                 return retrieved_docs
+
+        except Exception as e:
+            logger.error(f"Failed to retrieve documents for query '{query}': {e}")
+            # Implement retry logic or return empty list
+            return []

src/utils/logging.py

@@ -0,0 +1,9 @@
+# src/utils/logging.py
+import logging
+from config.settings import LOG_LEVEL
+
+def setup_logging():
+    """Configures basic logging for the application."""
+    logging.basicConfig(level=LOG_LEVEL, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    logging.getLogger("httpx").setLevel(logging.WARNING) # Suppress chatty libraries
+    logging.getLogger("httpcore").setLevel(logging.WARNING) # Suppress chatty libraries

src/vector_store_manager/chroma_manager.py

@@ -0,0 +1,182 @@
+# src/vector_store_manager/chroma_manager.py
+from langchain_chroma import Chroma # cite: embed_pipeline.py, query_pipeline.py
+from langchain.schema import Document # cite: embed_pipeline.py
+from config.settings import PERSIST_DIR, CHROMADB_COLLECTION_NAME # cite: embed_pipeline.py, query_pipeline.py
+from src.embedding_generator.embedder import EmbeddingGenerator
+import logging
+from typing import List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+class ChromaManager:
+    """
+    Manages interactions with the ChromaDB vector store.
+    """
+    def __init__(self, embedding_generator: EmbeddingGenerator):
+        self.embedding_generator = embedding_generator
+        # --- Financial Ministry Adaptation ---
+        # TODO: Configure Chroma client to use a scalable backend (e.g., ClickHouse)
+        # instead of or in addition to persistent_directory for production.
+        # This might involve using chromadb.HttpClient or specific backend configurations.
+        # Handle connection errors and retries to the database backend.
+        # Implement authentication/authorization for ChromaDB access.
+        # ------------------------------------
+        try:
+            # Initialize Chroma with the embedding function and persistence settings
+            # For production, you might replace persist_directory with client settings
+            # pointing to a ClickHouse backend.
+            self.vectordb = Chroma(
+                persist_directory=PERSIST_DIR, # cite: embed_pipeline.py, query_pipeline.py
+                collection_name=CHROMADB_COLLECTION_NAME, # cite: embed_pipeline.py, query_pipeline.py
+                embedding_function=self.embedding_generator.embedder # Use the Langchain embedder instance
+            )
+            logger.info(f"Initialized ChromaDB collection: '{CHROMADB_COLLECTION_NAME}' at '{PERSIST_DIR}'")
+            # You might want to check if the collection exists and its health
+
+        except Exception as e:
+            logger.critical(f"Failed to initialize ChromaDB: {e}")
+            raise e
+
+    def add_documents(self, chunks: List[Document]):
+        """
+        Adds document chunks to the ChromaDB collection.
+
+        Args:
+            chunks: A list of Langchain Document chunks with metadata.
+        """
+        # --- Financial Ministry Adaptation ---
+        # Implement error handling and retry logic for batch additions.
+        # Consider transactional behavior if adding large batches requires it.
+        # Log successful and failed additions.
+        # Ensure document IDs are managed consistently (e.g., based on source + chunk index or a stable hash).
+        # ------------------------------------
+        try:
+            # Langchain's add_documents handles embedding internally using the provided embedding_function
+            # Ensure your chunks have unique IDs if you need to update/delete later.
+            # If IDs are not in metadata, Langchain/Chroma might generate them.
+            # For better control, you might generate IDs in document_processor and pass them here.
+            if not chunks:
+                logger.warning("No chunks to add to ChromaDB.")
+                return
+
+            # If chunks don't have IDs, generate them (simple example)
+            # In a real system, use stable IDs based on source data
+            # chunk_ids = [f"{chunk.metadata.get('source', 'unknown')}_{i}" for i, chunk in enumerate(chunks)]
+            # self.vectordb.add_documents(chunks, ids=chunk_ids)
+
+            self.vectordb.add_documents(chunks) # Langchain handles IDs if not provided
+            logger.info(f"Added {len(chunks)} chunks to ChromaDB.")
+
+        except Exception as e:
+            logger.error(f"Failed to add documents to ChromaDB: {e}")
+            # Implement retry logic or raise exception
+
+    def update_documents(self, ids: List[str], documents: List[str], metadatas: List[Dict[str, Any]]):
+         """
+         Updates documents in the ChromaDB collection by ID.
+
+         Args:
+             ids: List of document IDs to update.
+             documents: List of new document content corresponding to IDs.
+             metadatas: List of new metadata dictionaries corresponding to IDs.
+         """
+         # --- Financial Ministry Adaptation ---
+         # Implement error handling and retry logic.
+         # Validate that IDs exist before attempting to update.
+         # ------------------------------------
+         try:
+             self.vectordb._collection.update( # Accessing the underlying collection for update/delete
+                 ids=ids,
+                 documents=documents,
+                 metadatas=metadatas
+             )
+             logger.info(f"Updated documents with IDs: {ids}")
+         except Exception as e:
+             logger.error(f"Failed to update documents with IDs {ids}: {e}")
+             raise e
+
+
+    def delete_documents(self, ids: List[str] = None, where: Dict[str, Any] = None):
+        """
+        Deletes documents from the ChromaDB collection by ID or metadata filter.
+
+        Args:
+            ids: List of document IDs to delete.
+            where: A dictionary for metadata filtering (e.g., {"source": "old_file.txt"}).
+        """
+        # --- Financial Ministry Adaptation ---
+        # Implement error handling and retry logic.
+        # Add logging to record which documents were deleted and why (if using where).
+        # ------------------------------------
+        try:
+            if ids:
+                self.vectordb._collection.delete(ids=ids) # Accessing the underlying collection
+                logger.info(f"Deleted documents with IDs: {ids}")
+            elif where:
+                self.vectordb._collection.delete(where=where) # Accessing the underlying collection
+                logger.info(f"Deleted documents matching metadata filter: {where}")
+            else:
+                logger.warning("Delete called without specifying ids or where filter.")
+        except Exception as e:
+            logger.error(f"Failed to delete documents (ids: {ids}, where: {where}): {e}")
+            raise e
+
+    def get_documents(self, ids: List[str] = None, where: Dict[str, Any] = None,
+                      where_document: Dict[str, Any] = None, limit: int = None,
+                      offset: int = None, include: List[str] = None) -> Dict[str, List[Any]]:
+        """
+        Retrieves documents and their details from the ChromaDB collection.
+
+        Args:
+            ids: List of document IDs to retrieve.
+            where: Metadata filter.
+            where_document: Document content filter.
+            limit: Maximum number of results.
+            offset: Offset for pagination.
+            include: List of fields to include (e.g., ['metadatas', 'documents']). IDs are always included.
+
+        Returns:
+            A dictionary containing the retrieved data (ids, documents, metadatas, etc.).
+        """
+        # --- Financial Ministry Adaptation ---
+        # Implement error handling and retry logic.
+        # Ensure sensitive metadata is handled appropriately if retrieved.
+        # ------------------------------------
+        try:
+            # Default include to metadatas and documents if not specified
+            if include is None:
+                 include = ['metadatas', 'documents'] # Default as per Chroma docs
+
+            results = self.vectordb._collection.get( # Accessing the underlying collection
+                ids=ids,
+                where=where,
+                where_document=where_document,
+                limit=limit,
+                offset=offset,
+                include=include
+            )
+            logger.debug(f"Retrieved {len(results.get('ids', []))} documents from ChromaDB.")
+            return results
+        except Exception as e:
+            logger.error(f"Failed to retrieve documents from ChromaDB: {e}")
+            raise e
+
+    def as_retriever(self, search_kwargs: Dict[str, Any] = None):
+        """
+        Returns a Langchain Retriever instance for the Chroma collection.
+
+        Args:
+            search_kwargs: Arguments for the retriever (e.g., {"k": 5}).
+
+        Returns:
+            A Langchain Retriever.
+        """
+        # --- Financial Ministry Adaptation ---
+        # Consider adding default search_kwargs here if not provided.
+        # Ensure the retriever uses the configured embedding function.
+        # ------------------------------------
+        if search_kwargs is None:
+             search_kwargs = {}
+        # Langchain's .as_retriever method automatically uses the embedding_function
+        # provided during Chroma initialization.
+        return self.vectordb.as_retriever(search_kwargs=search_kwargs) # cite: query_pipeline.py

system_architecture.md

@@ -0,0 +1,16 @@
+graph LR
+    participant User as "User"
+    participant API as "Query Service API"
+    participant RetrievalHandler as "Retrieval Handler"
+    participant EmbeddingGenerator as "Embedding Generator"
+    participant ChromaDB as "ChromaDB (Vector Store)"
+    participant LLMIntegrator as "LLM Integrator"
+
+    User->>API: Sends query request
+    API->>RetrievalHandler: Retrieve relevant documents
+    RetrievalHandler->>EmbeddingGenerator: Generate embeddings for query
+    EmbeddingGenerator->>ChromaDB: Query vector store
+    ChromaDB->>RetrievalHandler: Return relevant documents
+    RetrievalHandler->>LLMIntegrator: Generate response using LLM
+    LLMIntegrator->>API: Return final response
+    API->>User: Return final response