Merge all API endpoints into the app API

api/docs.py

@@ -26,7 +26,7 @@ from litellm.router import Router
 from schemas import DataRequest, DataResponse, DocRequirements, DownloadRequest, MeetingsRequest, MeetingsResponse, RequirementsRequest, RequirementsResponse
 
 # API router for requirement extraction from docs / doc list retrieval / download
-router = APIRouter()
+router = APIRouter(tags=["document extraction"])
 
 # ==================================================== Utilities =================================================================
 

api/requirements.py

@@ -1,10 +1,11 @@
 from fastapi import APIRouter, Depends, HTTPException
+from jinja2 import Environment
 from litellm.router import Router
-from dependencies import get_llm_router
-from schemas import ReqSearchLLMResponse, ReqSearchRequest, ReqSearchResponse
+from dependencies import get_llm_router, get_prompt_templates
+from schemas import _ReqGroupingCategory, _ReqGroupingOutput, ReqGroupingCategory, ReqGroupingRequest, ReqGroupingResponse, ReqSearchLLMResponse, ReqSearchRequest, ReqSearchResponse
 
-# Router for all requirements
-router = APIRouter()
+# Router for requirement processing
+router = APIRouter(tags=["requirement processing"])
 
 
 @router.post("/get_reqs_from_query", response_model=ReqSearchResponse)
@@ -16,7 +17,6 @@ def find_requirements_from_problem_description(req: ReqSearchRequest, llm_router
 
     requirements_text = "\n".join(
         [f"[Selection ID: {r.req_id} | Document: {r.document} | Context: {r.context} | Requirement: {r.requirement}]" for r in requirements])
-    print("Called the LLM")
     resp_ai = llm_router.completion(
         model="gemini-v2",
         messages=[{"role": "user", "content": f"Given all the requirements : \n {requirements_text} \n and the problem description \"{query}\", return a list of 'Selection ID' for the most relevant corresponding requirements that reference or best cover the problem. If none of the requirements covers the problem, simply return an empty list"}],
@@ -33,3 +33,63 @@ def find_requirements_from_problem_description(req: ReqSearchRequest, llm_router
             status_code=500, detail="LLM error : Generated a wrong index, please try again.")
 
     return ReqSearchResponse(requirements=[requirements[i] for i in out_llm])
+
+
+@router.post("/categorize_requirements")
+async def categorize_reqs(params: ReqGroupingRequest, prompt_env: Environment = Depends(get_prompt_templates), llm_router: Router = Depends(get_llm_router)) -> ReqGroupingResponse:
+    """Categorize the given service requirements into categories"""
+
+    MAX_ATTEMPTS = 5
+
+    categories: list[_ReqGroupingCategory] = []
+    messages = []
+
+    # categorize the requirements using their indices
+    req_prompt = await prompt_env.get_template("classify.txt").render_async(**{
+        "requirements": [rq.model_dump() for rq in params.requirements],
+        "max_n_categories": params.max_n_categories,
+        "response_schema": _ReqGroupingOutput.model_json_schema()})
+
+    # add system prompt with requirements
+    messages.append({"role": "user", "content": req_prompt})
+
+    # ensure all requirements items are processed
+    for attempt in range(MAX_ATTEMPTS):
+        req_completion = await llm_router.acompletion(model="gemini-v2", messages=messages, response_format=_ReqGroupingOutput)
+        output = _ReqGroupingOutput.model_validate_json(
+            req_completion.choices[0].message.content)
+
+        # quick check to ensure no requirement was left out by the LLM by checking all IDs are contained in at least a single category
+        valid_ids_universe = set(range(0, len(params.requirements)))
+        assigned_ids = {
+            req_id for cat in output.categories for req_id in cat.items}
+
+        # keep only non-hallucinated, valid assigned ids
+        valid_assigned_ids = assigned_ids.intersection(valid_ids_universe)
+
+        # check for remaining requirements assigned to none of the categories
+        unassigned_ids = valid_ids_universe - valid_assigned_ids
+
+        if len(unassigned_ids) == 0:
+            categories.extend(output.categories)
+            break
+        else:
+            messages.append(req_completion.choices[0].message)
+            messages.append(
+                {"role": "user", "content": f"You haven't categorized the following requirements in at least one category {unassigned_ids}. Please do so."})
+
+            if attempt == MAX_ATTEMPTS - 1:
+                raise Exception("Failed to classify all requirements")
+
+    # build the final category objects
+    # remove the invalid (likely hallucinated) requirement IDs
+    final_categories = []
+    for idx, cat in enumerate(output.categories):
+        final_categories.append(ReqGroupingCategory(
+            id=idx,
+            title=cat.title,
+            requirements=[params.requirements[i]
+                          for i in cat.items if i < len(params.requirements)]
+        ))
+
+    return ReqGroupingResponse(categories=final_categories)

api/solutions.py

@@ -0,0 +1,219 @@
+import asyncio
+import json
+import logging
+from fastapi import APIRouter, Depends, HTTPException
+from httpx import AsyncClient
+from jinja2 import Environment
+from litellm.router import Router
+from dependencies import INSIGHT_FINDER_BASE_URL, get_http_client, get_llm_router, get_prompt_templates
+from typing import Awaitable, Callable, TypeVar
+from schemas import _RefinedSolutionModel, _ReqGroupingCategory, _ReqGroupingOutput, _SearchedSolutionModel, _SolutionCriticismOutput, CriticizeSolutionsRequest, CritiqueResponse, InsightFinderConstraintsList, ReqGroupingCategory, ReqGroupingRequest, ReqGroupingResponse, ReqSearchLLMResponse, ReqSearchRequest, ReqSearchResponse, SolutionCriticism, SolutionModel, SolutionSearchResponse, SolutionSearchV2Request, TechnologyData
+
+# Router for solution generation and critique
+router = APIRouter(tags=["solution generation and critique"])
+
+
+# ============== utilities =======================
+T = TypeVar("T")
+A = TypeVar("A")
+
+
+async def retry_until(
+    func: Callable[[A], Awaitable[T]],
+    arg: A,
+    predicate: Callable[[T], bool],
+    max_retries: int,
+) -> T:
+    """Retries the given async function until the passed in validation predicate returns true."""
+    last_value = await func(arg)
+    for _ in range(max_retries):
+        if predicate(last_value):
+            return last_value
+        last_value = await func(arg)
+    return last_value
+
+# =================================================
+
+
+@router.post("/search_solutions_gemini/v2", response_model=SolutionSearchResponse)
+async def search_solutions(params: SolutionSearchV2Request, prompt_env: Environment = Depends(get_prompt_templates), llm_router: Router = Depends(get_llm_router)) -> SolutionSearchResponse:
+    """Searches solutions solving the given grouping params and respecting the user constraints using Gemini and grounded on google search"""
+
+    logging.info(f"Searching solutions for categories: {params}")
+
+    async def _search_inner(cat: ReqGroupingCategory) -> SolutionModel:
+        # ================== generate the solution with web grounding
+        req_prompt = await prompt_env.get_template("search_solution_v2.txt").render_async(**{
+            "category": cat.model_dump(),
+            "user_constraints": params.user_constraints,
+        })
+
+        # generate the completion in non-structured mode.
+        # the googleSearch tool enables grounding gemini with google search
+        # this also forces gemini to perform a tool call
+        req_completion = await llm_router.acompletion(model="gemini-v2", messages=[
+            {"role": "user", "content": req_prompt}
+        ], tools=[{"googleSearch": {}}], tool_choice="required")
+
+        # ==================== structure the solution as a json ===================================
+
+        structured_prompt = await prompt_env.get_template("structure_solution.txt").render_async(**{
+            "solution": req_completion.choices[0].message.content,
+            "response_schema": _SearchedSolutionModel.model_json_schema()
+        })
+
+        structured_completion = await llm_router.acompletion(model="gemini-v2", messages=[
+            {"role": "user", "content": structured_prompt}
+        ], response_format=_SearchedSolutionModel)
+        solution_model = _SearchedSolutionModel.model_validate_json(
+            structured_completion.choices[0].message.content)
+
+        # ======================== build the final solution object ================================
+
+        sources_metadata = []
+        # extract the source metadata from the search items, if gemini actually called the tools to search .... and didn't hallucinated
+        try:
+            sources_metadata.extend([{"name": a["web"]["title"], "url": a["web"]["uri"]}
+                                     for a in req_completion["vertex_ai_grounding_metadata"][0]['groundingChunks']])
+        except KeyError as ke:
+            pass
+
+        final_sol = SolutionModel(
+            Context="",
+            Requirements=[
+                cat.requirements[i].requirement for i in solution_model.requirement_ids
+            ],
+            Problem_Description=solution_model.problem_description,
+            Solution_Description=solution_model.solution_description,
+            References=sources_metadata,
+            Category_Id=cat.id,
+        )
+        return final_sol
+
+    solutions = await asyncio.gather(*[retry_until(_search_inner, cat, lambda v: len(v.References) > 0, 2) for cat in params.categories], return_exceptions=True)
+    logging.info(solutions)
+    final_solutions = [
+        sol for sol in solutions if not isinstance(sol, Exception)]
+
+    return SolutionSearchResponse(solutions=final_solutions)
+
+
+@router.post("/criticize_solution", response_model=CritiqueResponse)
+async def criticize_solution(params: CriticizeSolutionsRequest, prompt_env: Environment = Depends(get_prompt_templates), llm_router: Router = Depends(get_llm_router)) -> CritiqueResponse:
+    """Criticize the challenges, weaknesses and limitations of the provided solutions."""
+
+    async def __criticize_single(solution: SolutionModel):
+        req_prompt = await prompt_env.get_template("criticize.txt").render_async(**{
+            "solutions": [solution.model_dump()],
+            "response_schema": _SolutionCriticismOutput.model_json_schema()
+        })
+
+        req_completion = await llm_router.acompletion(
+            model="gemini-v2",
+            messages=[{"role": "user", "content": req_prompt}],
+            response_format=_SolutionCriticismOutput
+        )
+
+        criticism_out = _SolutionCriticismOutput.model_validate_json(
+            req_completion.choices[0].message.content
+        )
+
+        return SolutionCriticism(solution=solution, criticism=criticism_out.criticisms[0])
+
+    critiques = await asyncio.gather(*[__criticize_single(sol) for sol in params.solutions], return_exceptions=False)
+    return CritiqueResponse(critiques=critiques)
+
+
+# =================================================================== Refine solution ====================================
+
+@router.post("/refine_solutions", response_model=SolutionSearchResponse)
+async def refine_solutions(params: CritiqueResponse, prompt_env: Environment = Depends(get_prompt_templates), llm_router: Router = Depends(get_llm_router)) -> SolutionSearchResponse:
+    """Refines the previously critiqued solutions."""
+
+    async def __refine_solution(crit: SolutionCriticism):
+        req_prompt = await prompt_env.get_template("refine_solution.txt").render_async(**{
+            "solution": crit.solution.model_dump(),
+            "criticism": crit.criticism,
+            "response_schema": _RefinedSolutionModel.model_json_schema(),
+        })
+
+        req_completion = await llm_router.acompletion(model="gemini-v2", messages=[
+            {"role": "user", "content": req_prompt}
+        ], response_format=_RefinedSolutionModel)
+
+        req_model = _RefinedSolutionModel.model_validate_json(
+            req_completion.choices[0].message.content)
+
+        # copy previous solution model
+        refined_solution = crit.solution.model_copy(deep=True)
+        refined_solution.Problem_Description = req_model.problem_description
+        refined_solution.Solution_Description = req_model.solution_description
+
+        return refined_solution
+
+    refined_solutions = await asyncio.gather(*[__refine_solution(crit) for crit in params.critiques], return_exceptions=False)
+
+    return SolutionSearchResponse(solutions=refined_solutions)
+
+# =============================================================== Search solutions =========================================
+
+
+@router.post("/search_solutions_if")
+async def search_solutions_if(req: SolutionSearchV2Request, prompt_env: Environment = Depends(get_prompt_templates), llm_router: Router = Depends(get_llm_router), http_client: AsyncClient = Depends(get_http_client)) -> SolutionSearchResponse:
+
+    async def _search_solution_inner(cat: ReqGroupingCategory):
+        # process requirements into insight finder format
+        fmt_completion = await llm_router.acompletion("gemini-v2", messages=[
+            {
+                "role": "user",
+                "content": await prompt_env.get_template("if/format_requirements.txt").render_async(**{
+                    "category": cat.model_dump(),
+                    "response_schema": InsightFinderConstraintsList.model_json_schema()
+                })
+            }], response_format=InsightFinderConstraintsList)
+
+        fmt_model = InsightFinderConstraintsList.model_validate_json(
+            fmt_completion.choices[0].message.content)
+
+        # translate from a structured output to a dict for insights finder
+        formatted_constraints = {'constraints': {
+            cons.title: cons.description for cons in fmt_model.constraints}}
+
+        # fetch technologies from insight finder
+        technologies_req = await http_client.post(INSIGHT_FINDER_BASE_URL + "process-constraints", content=json.dumps(formatted_constraints))
+        technologies = TechnologyData.model_validate(technologies_req.json())
+
+        # =============================================================== synthesize solution using LLM =========================================
+
+        format_solution = await llm_router.acompletion("gemini-v2", messages=[{
+            "role": "user",
+            "content": await prompt_env.get_template("if/synthesize_solution.txt").render_async(**{
+                "category": cat.model_dump(),
+                "technologies": technologies.model_dump()["technologies"],
+                "user_constraints": req.user_constraints,
+                "response_schema": _SearchedSolutionModel.model_json_schema()
+            })}
+        ], response_format=_SearchedSolutionModel)
+
+        format_solution_model = _SearchedSolutionModel.model_validate_json(
+            format_solution.choices[0].message.content)
+
+        final_solution = SolutionModel(
+            Context="",
+            Requirements=[
+                cat.requirements[i].requirement for i in format_solution_model.requirement_ids
+            ],
+            Problem_Description=format_solution_model.problem_description,
+            Solution_Description=format_solution_model.solution_description,
+            References=[],
+            Category_Id=cat.id,
+        )
+
+        # ========================================================================================================================================
+
+        return final_solution
+
+    tasks = await asyncio.gather(*[_search_solution_inner(cat) for cat in req.categories], return_exceptions=True)
+    final_solutions = [sol for sol in tasks if not isinstance(sol, Exception)]
+
+    return SolutionSearchResponse(solutions=final_solutions)

app.py

@@ -7,6 +7,7 @@ import warnings
 import os
 from fastapi import Depends, FastAPI, BackgroundTasks, HTTPException, Request
 from fastapi.staticfiles import StaticFiles
+import api.solutions
 from dependencies import get_llm_router, init_dependencies
 import api.docs
 import api.requirements
@@ -43,4 +44,5 @@ app.add_middleware(CORSMiddleware, allow_credentials=True, allow_headers=[
 
 app.include_router(api.docs.router, prefix="/docs")
 app.include_router(api.requirements.router, prefix="/requirements")
+app.include_router(api.solutions.router, prefix="/solutions")
 app.mount("/", StaticFiles(directory="static", html=True), name="static")

dependencies.py

@@ -1,15 +1,22 @@
 import os
+from httpx import AsyncClient
 from litellm.router import Router
+from jinja2 import Environment, StrictUndefined, FileSystemLoader
 
 # Declare all global app dependencies here
 # - Setup your dependency global inside init_dependencies()
 # - Create a get_xxxx_() function to retrieve the dependency inside the FastAPI router
 
 
+INSIGHT_FINDER_BASE_URL = "https://organizedprogrammers-insight-finder.hf.space/"
+
 def init_dependencies():
     """Initialize the application global dependencies"""
 
     global llm_router
+    global prompt_templates
+    global http_client
+
     llm_router = Router(model_list=[
         {
             "model_name": "gemini-v1",
@@ -27,7 +34,7 @@ def init_dependencies():
             "model_name": "gemini-v2",
             "litellm_params":
             {
-                "model": "gemini/gemini-2.5-flash",
+                "model": "gemini/gemini-2.5-flash-lite-preview-06-17",
                 "api_key": os.environ.get("GEMINI"),
                 "max_retries": 5,
                 "rpm": 10,
@@ -36,7 +43,23 @@ def init_dependencies():
             }
         }], fallbacks=[{"gemini-v2": ["gemini-v1"]}], num_retries=10, retry_after=30)
 
+    prompt_templates = Environment(loader=FileSystemLoader(
+        "prompts"), enable_async=True, undefined=StrictUndefined)
+
+    http_client = AsyncClient(verify=os.environ.get(
+        "NO_SSL", "0") == "1", timeout=None)
+
 
 def get_llm_router() -> Router:
     """Retrieves the LLM router"""
     return llm_router
+
+
+def get_prompt_templates() -> Environment:
+    """Retrieves the Jinja2 prompt templates environment"""
+    return prompt_templates
+
+
+def get_http_client() -> AsyncClient:
+    """Retrieves the HTTP Client"""
+    return http_client

index.html

@@ -1,341 +0,0 @@
-<!DOCTYPE html>
-<html lang="fr">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Requirements Extractor</title>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/marked/16.0.0/lib/marked.umd.min.js"
-        integrity="sha512-ygzQGrI/8Bfkm9ToUkBEuSMrapUZcHUys05feZh4ScVrKCYEXJsCBYNeVWZ0ghpH+n3Sl7OYlRZ/1ko01pYUCQ=="
-        crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-    <link href="https://cdn.jsdelivr.net/npm/daisyui@5" rel="stylesheet" type="text/css" />
-    <script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
-</head>
-
-<body class="bg-gray-100 min-h-screen">
-    <!-- Loading Overlay -->
-    <div id="loading-overlay" class="fixed inset-0 bg-black/50 flex items-center justify-center z-50 hidden">
-        <div class="bg-white p-6 rounded-lg shadow-lg text-center">
-            <span class="loading loading-spinner loading-xl"></span>
-            <p id="progress-text" class="text-gray-700">Chargement en cours...</p>
-        </div>
-    </div>
-
-    <div class="container mx-auto p-6">
-        <h1 class="text-3xl font-bold text-center mb-8">Requirements Extractor</h1>
-        <div id="selection-container" class="mb-6">
-            <div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4">
-                <!-- Working Group -->
-                <div>
-                    <label for="working-group-select" class="block text-sm font-medium text-gray-700 mb-2">Working
-                        Group</label>
-                    <select id="working-group-select" class="w-full p-2 border border-gray-300 rounded-md">
-                        <option value="" selected>Select a working group</option>
-                        <option value="SA1">SA1</option>
-                        <option value="SA2">SA2</option>
-                        <option value="SA3">SA3</option>
-                        <option value="SA4">SA4</option>
-                        <option value="SA5">SA5</option>
-                        <option value="SA6">SA6</option>
-                        <option value="CT1">CT1</option>
-                        <option value="CT2">CT2</option>
-                        <option value="CT3">CT3</option>
-                        <option value="CT4">CT4</option>
-                        <option value="CT5">CT5</option>
-                        <option value="CT6">CT6</option>
-                        <option value="RAN1">RAN1</option>
-                        <option value="RAN2">RAN2</option>
-                    </select>
-                </div>
-
-                <!-- Meeting -->
-                <div>
-                    <label for="meeting-select" class="block text-sm font-medium text-gray-700 mb-2">Meeting</label>
-                    <select id="meeting-select" class="w-full p-2 border border-gray-300 rounded-md">
-                        <option value="">Select a meeting</option>
-                    </select>
-                </div>
-            </div>
-
-            <!-- Buttons -->
-            <div class="flex flex-col sm:flex-row gap-2">
-                <!-- <button id="get-meetings-btn" class="px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
-                    Get Meetings
-                </button> -->
-                <button id="get-tdocs-btn" class="px-4 py-2 bg-green-500 text-white rounded-md hover:bg-green-600">
-                    Get TDocs
-                </button>
-            </div>
-        </div>
-
-        <!-- Add an horizontal separation -->
-        <hr>
-        <!-- Tab list for subsections -->
-        <div role="tablist" class="tabs tabs-border tabs-xl" id="tab-container">
-            <a role="tab" class="tab tab-active" id="doc-table-tab" onclick="switchTab('doc-table-tab')">📝
-                Documents</a>
-            <a role="tab" class="tab tab-disabled" id="requirements-tab" onclick="switchTab('requirements-tab')">
-                <div class="flex items-center gap-1">
-                    <div class="badge badge-neutral badge-outline badge-xs" id="requirements-tab-badge">0</div>
-                    <span>Requirements</span>
-                </div>
-            </a>
-            <a role="tab" class="tab tab-disabled" id="solutions-tab" onclick="switchTab('solutions-tab')">Group and
-                Solve</a>
-            <a role="tab" class="tab tab-disabled" id="query-tab" onclick="switchTab('query-tab')">🔎 Find relevant
-                requirements</a>
-        </div>
-
-        <div id="doc-table-tab-contents" class="hidden">
-            <!-- Filters -->
-            <div id="filters-container" class="mb-6 hidden pt-10">
-                <div class="grid grid-cols-1 md:grid-cols-3 gap-4">
-                    <!-- Type Filter Dropdown -->
-                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
-                        <div tabindex="0" role="button" class="btn w-full justify-between" id="doc-type-filter-btn">
-                            <span id="doc-type-filter-label">Type</span>
-                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
-                                stroke="currentColor">
-                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                                    d="M19 9l-7 7-7-7" />
-                            </svg>
-                        </div>
-                        <ul tabindex="0"
-                            class="dropdown-content z-[1] menu p-2 shadow bg-base-100 rounded-box w-full min-w-[200px] max-h-60 overflow-y-auto"
-                            id="doc-type-filter-menu">
-                            <!-- Rempli par JS -->
-                        </ul>
-                    </div>
-
-                    <!-- Status Filter Dropdown -->
-                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
-                        <div tabindex="0" role="button" class="btn w-full justify-between" id="status-dropdown-btn">
-                            <span id="status-filter-label">Status (Tous)</span>
-                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
-                                stroke="currentColor">
-                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                                    d="M19 9l-7 7-7-7" />
-                            </svg>
-                        </div>
-                        <ul tabindex="0"
-                            class="dropdown-content z-[1] p-2 shadow bg-base-100 rounded-box w-full max-h-60 overflow-y-auto">
-                            <li class="border-b pb-2 mb-2">
-                                <label class="flex items-center gap-2 cursor-pointer">
-                                    <input type="checkbox" class="status-checkbox" value="all" checked>
-                                    <span class="font-semibold">Tous</span>
-                                </label>
-                            </li>
-                            <div id="status-options" class="flex flex-col gap-1"></div>
-                        </ul>
-                    </div>
-
-                    <!-- Agenda Item Filter Dropdown -->
-                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
-                        <div tabindex="0" role="button" class="btn w-full justify-between" id="agenda-dropdown-btn">
-                            <span id="agenda-filter-label">Agenda Item (Tous)</span>
-                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
-                                stroke="currentColor">
-                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                                    d="M19 9l-7 7-7-7" />
-                            </svg>
-                        </div>
-                        <ul tabindex="0"
-                            class="dropdown-content z-[1] p-2 shadow bg-base-100 rounded-box w-full max-h-60 overflow-y-auto">
-                            <li class="border-b pb-2 mb-2">
-                                <label class="flex items-center gap-2 cursor-pointer">
-                                    <input type="checkbox" class="agenda-checkbox" value="all" checked>
-                                    <span class="font-semibold">Tous</span>
-                                </label>
-                            </li>
-                            <div id="agenda-options" class="flex flex-col gap-1"></div>
-                        </ul>
-                    </div>
-                </div>
-            </div>
-
-            <!-- Data Table Informations -->
-            <div class="flex justify-between items-center mb-2 pt-5" id="data-table-info-container">
-                <!-- Left side: buttons -->
-                <div class="flex gap-2 items-center">
-                    <div class="tooltip" data-tip="Extract requirements from selected documents">
-                        <button id="extract-requirements-btn"
-                            class="bg-orange-300 text-white text-sm rounded px-3 py-1 shadow hover:bg-orange-600">
-                            <svg class="w-6 h-6 text-gray-800 dark:text-white" aria-hidden="true"
-                                xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none"
-                                viewBox="0 0 24 24">
-                                <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"
-                                    stroke-width="2"
-                                    d="M9 8h6m-6 4h6m-6 4h6M6 3v18l2-2 2 2 2-2 2 2 2-2 2 2V3l-2 2-2-2-2 2-2-2-2 2-2-2Z" />
-                            </svg>Extract Requirements
-                        </button>
-                    </div>
-                    <button id="download-tdocs-btn" class="text-sm rounded px-3 py-1 shadow cursor-pointer">
-                        📦 Download Selected TDocs
-                    </button>
-                </div>
-
-                <!-- Right side: document counts -->
-                <div class="flex gap-2 items-center">
-                    <span id="displayed-count" class="text-sm text-gray-700 bg-white rounded px-3 py-1 shadow">
-                        0 total documents
-                    </span>
-                    <span id="selected-count" class="text-sm text-blue-700 bg-blue-50 rounded px-3 py-1 shadow">
-                        0 selected documents
-                    </span>
-                </div>
-            </div>
-
-
-            <!-- Data Table -->
-            <div id="data-table-container" class="mb-6">
-                <table id="data-table" class="w-full bg-white rounded-lg shadow overflow-hidden">
-                    <thead class="bg-gray-50">
-                        <tr>
-                            <th class="px-4 py-2 text-left">
-                                <input type="checkbox" id="select-all-checkbox">
-                            </th>
-                            <th class="px-4 py-2 text-left">TDoc</th>
-                            <th class="px-4 py-2 text-left">Title</th>
-                            <th class="px-4 py-2 text-left">Type</th>
-                            <th class="px-4 py-2 text-left">Status</th>
-                            <th class="px-4 py-2 text-left">Agenda Item</th>
-                            <th class="px-4 py-2 text-left">URL</th>
-                        </tr>
-                    </thead>
-                    <tbody></tbody>
-                </table>
-            </div>
-        </div>
-
-        <div id="requirements-tab-contents" class="hidden pt-10">
-            <!-- Requirement list container -->
-            <div id="requirements-container" class="mb-6">
-                <div class="flex">
-                    <h2 class="text-2xl font-bold mb-4">Extracted requirement list</h2>
-                    <div class="justify-end pl-5">
-                        <!--Copy ALL reqs button-->
-                        <div class="tooltip" data-tip="Copy ALL requirements to clipboard">
-                            <button class="btn btn-square" id="copy-all-reqs-btn" aria-label="Copy">
-                                📋
-                            </button>
-                        </div>
-                    </div>
-                </div>
-                <div id="requirements-list"></div>
-            </div>
-        </div>
-
-
-        <!-- Query Requirements -->
-        <div id="query-tab-contents" class="hidden pt-10">
-            <div id="query-requirements-container" class="mb-6">
-                <h2 class="text-2xl font-bold mb-4">Find relevant requirements for a query</h2>
-                <div class="flex gap-2">
-                    <input type="text" id="query-input" class="flex-1 p-2 border border-gray-300 rounded-md"
-                        placeholder="Enter your query...">
-                    <button id="search-requirements-btn"
-                        class="px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
-                        Search
-                    </button>
-                </div>
-                <div id="query-results" class="mt-4"></div>
-            </div>
-        </div>
-
-        <!-- Solution tab -->
-        <div id="solutions-tab-contents" class="hidden pt-10">
-            <!--Header de catégorisation des requirements-->
-            <div class="w-full mx-auto mt-4 p-4 bg-base-100 rounded-box shadow-lg border border-base-200">
-                <!-- Séléction du nb de catégories / mode auto-->
-                <div class="flex flex-wrap justify-begin items-center gap-6">
-                    <!-- Number input -->
-                    <div class="form-control flex flex-col items-center justify-center">
-                        <label class="label" for="category-count">
-                            <span class="label-text text-base-content"># Categories</span>
-                        </label>
-                        <input id="category-count" type="number" class="input input-bordered w-24" min="1" value="3" />
-                    </div>
-
-                    <!-- Auto detect toggle -->
-                    <div class="form-control flex flex-col items-center justify-center">
-                        <label class="label">
-                            <span class="label-text text-base-content">Number of categories</span>
-                        </label>
-                        <input type="checkbox" class="toggle toggle-primary" id="auto-detect-toggle" checked="true" />
-                        <div class="text-xs mt-1 text-base-content">Manual | Auto detect</div>
-                    </div>
-
-                    <!-- Action Button -->
-                    <button id="categorize-requirements-btn" class="btn btn-primary btn-l self-center">
-                        Categorize
-                    </button>
-                </div>
-                <div class="flex flex-wrap justify-end items-center gap-6">
-                    <div class="tooltip" data-tip="Use Insight Finder's API to generate solutions">
-                        <label class="label">
-                            <span class="label-text text-base-content">Use insight finder solver</span>
-                        </label>
-                        <input type="checkbox" class="toggle toggle-primary" id="use-insight-finder-solver"
-                            checked="false" />
-                    </div>
-                </div>
-            </div>
-            <div id="categorized-requirements-container pt-10" class="mb-6">
-                <div class="flex mb-4 pt-10">
-                    <div class="justify-begin">
-                        <h2 class="text-2xl font-bold">Requirements categories list</h2>
-                    </div>
-                    <div class="justify-end pl-5">
-                        <!--Copy reqs button-->
-                        <div class="tooltip" data-tip="Copy selected requirements to clipboard">
-                            <button class="btn btn-square" id="copy-reqs-btn" aria-label="Copy">
-                                📋
-                            </button>
-                        </div>
-                    </div>
-                </div>
-                <div id="categorized-requirements-list"></div>
-            </div>
-
-            <!-- Solutions Action Buttons -->
-            <div id="solutions-action-buttons-container" class="mb-6 hidden">
-                <div class="flex flex-wrap gap-2 justify-center items-center">
-                    <div class="join">
-                        <input id="solution-gen-nsteps" type="number" class="input join-item w-24" min="1" value="3" />
-                        <button id="get-solutions-btn"
-                            class="btn join-item px-4 py-2 bg-indigo-500 text-white rounded-md hover:bg-indigo-600">
-                            Run multiple steps
-                        </button>
-                    </div>
-                    <button id="get-solutions-step-btn"
-                        class="px-4 py-2 bg-pink-500 text-white rounded-md hover:bg-pink-600">
-                        Get Solutions (Step-by-step)
-                    </button>
-                    <div class="dropdown dropdown-center">
-                        <div id="additional-gen-instr-btn" tabindex="0" role="button" class="btn m-1">🎨 Additional
-                            generation constraints</div>
-                        <div tabindex="0" class="dropdown-content card card-sm bg-base-100 z-1 w-256 shadow-md">
-                            <div class="card-body">
-                                <h3>Add additional constraints to follow when generating the solutions</h3>
-                                <textarea id="additional-gen-instr"
-                                    class="textarea textarea-bordered w-full"></textarea>
-                            </div>
-                        </div>
-                    </div>
-                </div>
-
-            </div>
-
-            <!-- Solutions Container -->
-            <div id="solutions-container" class="mb-6">
-                <h2 class="text-2xl font-bold mb-4">Solutions</h2>
-                <div id="solutions-list"></div>
-            </div>
-        </div>
-
-        <script src="/static/sse.js"></script>
-        <script src="/static/ui-utils.js"></script>
-        <script src="/static/script.js"></script>
-</body>
-
-</html>

prompts/classify.txt

@@ -0,0 +1,21 @@
+<role>You are an useful assistant who excels at categorizing technical extracted requirements</role>
+<task>You are tasked with classifying each element of a list of technical requirements into categories which you may arbitrarily define. 
+For each category indicate which requirements belong in that category using their ID. An item may appear in multiple categories at a time. 
+</task>
+{% if max_n_categories is none -%}
+<number_of_categories>You may have at most as much categories as you think is needed</number_of_categories>
+{% else -%}
+<number_of_categories>You may have at most {{max_n_categories}} categories</number_of_categories>
+{% endif-%}
+
+Here are the requirements:
+<requirements>
+{% for req in requirements -%}
+- {{ loop.index0 }}. {{ req["requirement"] }}
+{% endfor -%}
+</requirements>
+
+<response_format>
+Reply in JSON using the following schema:
+{{response_schema}}
+</response_format> 

prompts/criticize.txt

@@ -0,0 +1,21 @@
+<role>You are an expert system designer</role>
+<task>
+You are tasked with criticizing multiple solutions solving a set of requirements on different points, 
+namely the technical challenges of the solutions, the weaknesses and limitations of the proposed solutions.
+</task>
+
+Here are the solutions:
+<solutions>
+{% for solution in solutions %}
+## Solution
+- Context: {{solution["Context"]}}
+- Problem description: {{solution["Problem_Description"]}}
+- Solution description: {{solution["Solution_Description"]}}
+---
+{% endfor -%}
+</solutions>
+
+<response_format>
+Reply in JSON using the following schema:
+{{response_schema}}
+</response_format> 

prompts/if/format_requirements.txt

@@ -0,0 +1,23 @@
+<role>You are an expert system designer</role>
+<task>
+Your task is to extract from a category of requirements the top most technical constraints that must be solved.
+Give the most important constraints that must be solved to address the general problem of the category.
+Give at most the 10 constraints.
+</task>
+
+<requirements>
+Here is the category item and the associated requirements:
+Category Title: {{category["title"]}}
+Context: {{category["requirements"][0]["context"]}}
+Requirements:
+{% for req in category["requirements"] -%}
+- {{loop.index0}} {{req["requirement"]}}
+{% endfor -%}
+</requirements>
+
+<response_format>
+Reply in JSON using the following schema:
+{{response_schema}}
+</response_format> 
+
+

prompts/if/synthesize_solution.txt

@@ -0,0 +1,43 @@
+<role>You are an expert system designer</role>
+<task>
+Your task is to create a solution that should cover the maximum possible of requirements at once.
+The solution should be composed of multiple mechanisms, that are defined as processes, methods, or sequences of steps using one or multiple technologies that explain how something works or achieves its requirements.
+
+You may use the technologies listed below for the mechanisms of your solution.
+
+You must aim for the following goals:
+- The solution must aim to maximize requirement satisfaction while respecting the context.
+- Provide a list of requirements addressed by the solution (provide the requirement IDs)
+- Please also detail each mechanism and how they work in the final solution description.
+- Please describe the solution description using markdown in a consistent format.
+</task>
+
+<requirements>
+Here is the category item and the associated requirements:
+Category Title: {{category["title"]}}
+Context: {{category["requirements"][0]["context"]}}
+Requirements:
+{% for req in category["requirements"] -%}
+- {{loop.index0}} {{req["requirement"]}}
+{% endfor -%}
+</requirements>
+
+<available_technologies>
+Here are the technologies you may use for the mechanisms that compose the solution:
+{% for tech in technologies -%}
+- {{tech["title"]}} : {{tech["purpose"]}}
+  * Key Components : {{tech["key_components"]}}
+{% endfor %}
+</available_technologies>
+
+{% if user_constraints is not none %}
+<user_constraints>
+Here are additional user constraints the solution must respect:
+    {{user_constraints}}
+</user_constraints> 
+{% endif %}
+
+<response_format>
+Reply in JSON using the following schema:
+{{response_schema}}
+</response_format> 

prompts/refine_solution.txt

@@ -0,0 +1,45 @@
+<role>You are an expert system designer</role>
+<task>
+Your task is to refine a solution to account for the technical challenges, weaknesses and limitations that were critiqued. 
+No need to include that the solution is refined.
+</task>
+
+Here is the solution:
+<solution>
+# Solution Context: 
+{{solution['Context']}}
+
+# Requirements solved by the solution
+{% for req in solution['Requirements'] -%}
+- {{req}}
+{% endfor %}
+
+# Problem description associated to the solution
+{{solution['Problem_Description']}}
+
+# Description of the solution
+{{solution['Solution_Description']}}
+</solution>
+
+Here is the criticism:
+<criticism>
+# Technical Challenges
+{% for challenge in criticism['technical_challenges'] -%}
+    - {{challenge}}
+{% endfor %}
+
+# Weaknesses
+{% for weakness in criticism['weaknesses'] -%}
+    - {{weakness}}
+{% endfor %}
+
+# Limitations
+{% for limitation in criticism['limitations'] -%}
+    - {{limitation}}
+{% endfor %}
+</criticism>
+
+<response_format>
+Reply in JSON using the following response schema:
+{{response_schema}}
+</response_format>

prompts/search_solution.txt

@@ -0,0 +1,23 @@
+<role>You are an expert system designer</role>
+<task>
+Your task is to create a solution that should cover the maximum possible of requirements at once.
+The solution should be composed of multiple mechanisms, that are defined as processes, methods, or sequences of steps using a technology that explain how something works or achieves its requirements.
+You may for that search mechanisms for the different requirements.
+**Please actually make searches and do not simulate them.**
+
+You must aim for the following goals:
+    - The solution must aim to maximize requirement satisfaction while respecting the context.
+    - Provide a list of requirements addressed by the solution (provide only the requirement IDs)
+    - Please also detail each mechanism and how they work in the final solution description.
+    - Please describe the solution description using markdown in a consistent format.
+</task>
+
+Here is the category item and the associated requirements:
+<requirements>
+Category Title: {{category["title"]}}
+Context: {{category["requirements"][0]["context"]}}
+Requirements:
+{% for req in category["requirements"] -%}
+- {{loop.index0}} {{req["requirement"]}}
+{% endfor -%}
+</requirements>

prompts/search_solution_v2.txt

@@ -0,0 +1,31 @@
+<role>You are an expert system designer</role>
+<task>
+Your task is to create a solution that should cover the maximum possible of requirements at once.
+The solution should be composed of multiple mechanisms, that are defined as processes, methods, or sequences of steps using a technology that explain how something works or achieves its requirements.
+You may for that search mechanisms for the different requirements.
+
+**Please actually make searches and do not simulate them.**
+
+You must aim for the following goals:
+- The solution must aim to maximize requirement satisfaction while respecting the context.
+- Provide a list of requirements addressed by the solution (provide only the requirement IDs)
+- Please also detail each mechanism and how they work in the final solution description.
+- Please describe the solution description using markdown in a consistent format.
+</task>
+
+Here is the category item and the associated requirements:
+<requirements>
+Category Title: {{category["title"]}}
+Context: {{category["requirements"][0]["context"]}}
+Requirements:
+{% for req in category["requirements"] -%}
+- {{loop.index0}} {{req["requirement"]}}
+{% endfor -%}
+</requirements>
+
+{% if user_constraints is not none %}
+Here are additional user constraints the solution must respect:
+<user_constraints>
+    {{user_constraints}}
+</user_constraints> 
+{% endif %}

prompts/structure_solution.txt

@@ -0,0 +1,12 @@
+<role>You are an expert system designer</role>
+<task>Your task is to take a solution you've created previously and structure it into a JSON object.</task>
+
+Here is the solution
+<solution>
+{{solution}}
+</solution>
+
+<response_format>
+Reply in JSON using the following schema:
+{{response_schema}}
+</response_format> 

requirements.txt

@@ -10,4 +10,5 @@ openpyxl
 beautifulsoup4
 aiolimiter
 nltk
-httpx
+httpx
+Jinja2

schemas.py

@@ -7,6 +7,7 @@ class MeetingsRequest(BaseModel):
 class MeetingsResponse(BaseModel):
     meetings: Dict[str, str]
 # --------------------------------------
+
 class DataRequest(BaseModel):
     working_group: str
     meeting: str
@@ -32,23 +33,180 @@ class RequirementsResponse(BaseModel):
     requirements: List[DocRequirements]
 
 # --------------------------------------
-class SingleRequirement(BaseModel):
-    req_id: int
-    document: str
-    context: str
-    requirement: str
+
+class RequirementInfo(BaseModel):
+    req_id: int = Field(..., description="The ID of this requirement")
+    context: str = Field(..., description="Context for the requirement.")
+    requirement: str = Field(..., description="The requirement itself.")
+    document: str = Field(...,
+                          description="The document the requirement is extracted from.")
+
 
 class ReqSearchLLMResponse(BaseModel):
     selected: List[int]
 
+
 class ReqSearchRequest(BaseModel):
     query: str
-    requirements: List[SingleRequirement]
+    requirements: List[RequirementInfo]
+
 
 class ReqSearchResponse(BaseModel):
-    requirements: List[SingleRequirement]
+    requirements: List[RequirementInfo]
 
 # --------------------------------------
 
+
 class DownloadRequest(BaseModel):
-    documents: List[str] = Field(description="List of document IDs to download")
+    documents: List[str] = Field(
+        description="List of document IDs to download")
+
+
+class ReqGroupingCategory(BaseModel):
+    """Represents the category of requirements grouped together"""
+    id: int = Field(..., description="ID of the grouping category")
+    title: str = Field(..., description="Title given to the grouping category")
+    requirements: List[RequirementInfo] = Field(
+        ..., description="List of grouped requirements")
+
+
+class SolutionModel(BaseModel):
+    Context: str = Field(...,
+                         description="Full context provided for this category.")
+    Requirements: List[str] = Field(...,
+                                    description="List of each requirement as string.")
+    Problem_Description: str = Field(..., alias="Problem Description",
+                                     description="Description of the problem being solved.")
+    Solution_Description: str = Field(..., alias="Solution Description",
+                                      description="Detailed description of the solution.")
+    References: list[dict] = Field(
+        ..., description="References to documents used for the solution.")
+    Category_Id: int = Field(
+        ..., description="ID of the requirements category the solution is based on")
+
+    class Config:
+        validate_by_name = True  # Enables alias handling on input/output
+
+
+# ============================================================= Categorize requirements endpoint
+
+
+class ReqGroupingRequest(BaseModel):
+    """Request schema of a requirement grouping call."""
+    requirements: list[RequirementInfo]
+    max_n_categories: Optional[int] = Field(
+        default=None, description="Max number of categories to construct. Defaults to None")
+
+
+class ReqGroupingResponse(BaseModel):
+    """Response of a requirement grouping call."""
+    categories: List[ReqGroupingCategory]
+
+
+# INFO: keep in sync with prompt
+class _ReqGroupingCategory(BaseModel):
+    title: str = Field(..., description="Title given to the grouping category")
+    items: list[int] = Field(
+        ..., description="List of the IDs of the requirements belonging to the category.")
+
+
+class _ReqGroupingOutput(BaseModel):
+    categories: list[_ReqGroupingCategory] = Field(
+        ..., description="List of grouping categories")
+
+
+# =================================================================== search solution response endpoint
+
+class _SolutionSearchOutput(BaseModel):
+    solution: SolutionModel
+
+
+class _SearchedSolutionModel(BaseModel):
+    """"Internal model used for solutions searched using gemini"""
+    requirement_ids: List[int] = Field(...,
+                                       description="List of each requirement ID addressed by the solution")
+    problem_description: str = Field(...,
+                                     description="Description of the problem being solved.")
+    solution_description: str = Field(...,
+                                      description="Detailed description of the solution.")
+
+
+class SolutionSearchResponse(BaseModel):
+    """Response model for solution search"""
+    solutions: list[SolutionModel]
+
+
+class SolutionSearchV2Request(BaseModel):
+    """Response of a requirement grouping call."""
+    categories: List[ReqGroupingCategory]
+    user_constraints: Optional[str] = Field(
+        default=None, description="Additional user constraints to respect when generating the solutions.")
+
+# =========================================================== Criticize solution endpoint
+
+
+class CriticizeSolutionsRequest(BaseModel):
+    solutions: list[SolutionModel]
+
+
+class _SolutionCriticism(BaseModel):
+    technical_challenges: List[str] = Field(
+        ..., description="Technical challenges encountered by the solution")
+    weaknesses: List[str] = Field(...,
+                                  description="Identified weaknesses of the solution")
+    limitations: List[str] = Field(...,
+                                   description="Identified limitations of the solution")
+
+
+class _SolutionCriticismOutput(BaseModel):
+    criticisms: List[_SolutionCriticism]
+
+
+class SolutionCriticism(BaseModel):
+    solution: SolutionModel
+    criticism: _SolutionCriticism
+
+
+class CritiqueResponse(BaseModel):
+    critiques: List[SolutionCriticism]
+
+
+# ==========================================================================
+
+class _RefinedSolutionModel(BaseModel):
+    """Internal model used for solution refining"""
+
+    problem_description: str = Field(...,
+                                     description="New description of the problem being solved.")
+    solution_description: str = Field(...,
+                                      description="New detailed description of the solution.")
+
+
+# ================================================================= search solutions using insight finder endpoints
+
+# Helpers to extract constraints for Insights Finder
+
+class InsightFinderConstraintItem(BaseModel):
+    title: str
+    description: str
+
+
+class InsightFinderConstraintsList(BaseModel):
+    constraints: list[InsightFinderConstraintItem]
+
+# =================================================
+
+
+class Technology(BaseModel):
+    """Represents a single technology entry with its details."""
+    title: str
+    purpose: str
+    key_components: str
+    advantages: str
+    limitations: str
+    id: int
+
+
+class TechnologyData(BaseModel):
+    """Represents the top-level object containing a list of technologies."""
+    technologies: List[Technology]

static/js/script.js

@@ -417,7 +417,7 @@ async function categorizeRequirements(max_categories) {
     toggleElementsEnabled(['categorize-requirements-btn'], false);
 
     try {
-        const response = await fetch('https://organizedprogrammers-reqxtract-api.hf.space/reqs/categorize_requirements', {
+        const response = await fetch('/requirements/categorize_requirements', {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ requirements: formattedRequirements, "max_n_categories": max_categories })
@@ -1054,7 +1054,7 @@ async function generateSolutionsV1(selected_categories, user_constraints = null,
     input_req.user_constraints = user_constraints;
     console.log(useIFSolver);
 
-    const base_url = useIFSolver ? "https://organizedprogrammers-reqxtract-api.hf.space/solution/search_solutions_if" : "https://organizedprogrammers-reqxtract-api.hf.space/solution/search_solutions_gemini/v2"
+    const base_url = useIFSolver ? "/solutions/search_solutions_if" : "/solutions/search_solutions_gemini/v2"
 
     let response = await fetch(base_url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(input_req) })
     let responseObj = await response.json()
@@ -1062,13 +1062,13 @@ async function generateSolutionsV1(selected_categories, user_constraints = null,
 }
 
 async function generateCriticismsV1(solutions) {
-    let response = await fetch('https://organizedprogrammers-reqxtract-api.hf.space/solution/criticize_solution', { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(solutions) })
+    let response = await fetch('/solutions/criticize_solution', { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(solutions) })
     let responseObj = await response.json()
     solutionsCriticizedVersions.push(responseObj)
 }
 
 async function refineSolutionsV1(critiques) {
-    let response = await fetch('https://organizedprogrammers-reqxtract-api.hf.space/solution/refine_solutions', { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(critiques) })
+    let response = await fetch('/solutions/refine_solutions', { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(critiques) })
     let responseObj = await response.json()
     await generateCriticismsV1(responseObj)
 }