Upload folder using huggingface_hub

.env.example

@@ -0,0 +1,28 @@
+# --- Required settings even when working locally. ---
+
+# OpenAI API Config
+OPENAI_MODEL_ID=gpt-4o-mini
+OPENAI_API_KEY=str
+
+# Huggingface API Config
+HUGGINGFACE_ACCESS_TOKEN=str
+
+# Comet ML (during training and inference) 
+COMET_API_KEY=str
+
+# --- Required settings when deploying the code. ---
+# --- Otherwise, default values work fine. ---
+
+# MongoDB database
+DATABASE_HOST="mongodb://llm_engineering:[email protected]:27017"
+
+# Qdrant vector database
+USE_QDRANT_CLOUD=false
+QDRANT_CLOUD_URL=str
+QDRANT_APIKEY=str
+
+# AWS Authentication
+AWS_ARN_ROLE=str
+AWS_REGION=eu-central-1
+AWS_ACCESS_KEY=str
+AWS_SECRET_KEY=str

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Chat[[:space:]]with[[:space:]]your[[:space:]]Video[[:space:]]Library[[:space:]]–[[:space:]]Engineering[[:space:]]AI[[:space:]]Agents.pdf filter=lfs diff=lfs merge=lfs -text
+clips/clip_eFgkZKhNUdM_1270_0.847.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/clip_eFgkZKhNUdM_642_0.847.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/clip_eFgkZKhNUdM_874_0.838.mp4 filter=lfs diff=lfs merge=lfs -text
+en_core_web_sm-3.7.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
+images/cover_plus.png filter=lfs diff=lfs merge=lfs -text
+images/crazy_cat.jpg filter=lfs diff=lfs merge=lfs -text

.github/workflows/cd.yaml

@@ -0,0 +1,43 @@
+name: CD
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build & Push Docker Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build images & push to ECR
+        id: build-image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/${{ secrets.AWS_ECR_NAME }}:${{ github.sha }}
+            ${{ steps.login-ecr.outputs.registry }}/${{ secrets.AWS_ECR_NAME }}:latest
+          push: true

.github/workflows/ci.yaml

@@ -0,0 +1,69 @@
+name: CI
+
+on:
+  pull_request:
+    
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qa:
+    name: QA
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: 1.8.3
+
+      - name: Install packages
+        run: |
+          poetry install --only dev
+          poetry self add 'poethepoet[poetry_plugin]'
+
+      - name: gitleaks check
+        run: poetry poe gitleaks-check
+
+      - name: Lint check [Python]
+        run: poetry poe lint-check
+
+      - name: Format check [Python]
+        run: poetry poe format-check
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: 1.8.3
+
+      - name: Install packages
+        run: |
+          poetry install
+          poetry self add 'poethepoet[poetry_plugin]'
+
+      - name: Run tests
+        run: |
+          echo "Running tests..."
+          poetry poe test

.gitignore

@@ -0,0 +1,177 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# IDEs
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# MacOs
+.DS_Store
+
+# VS Code
+.vscode/**/launch.json
+
+# Data
+output/
+sagemaker_*.json
+run_ids.txt
+
+# Virtual environments
+*_venv
+*_myenv

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
+    hooks:
+      - id: ruff # Run the linter.
+      - id: ruff-format # Run the formatter.
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.18.2
+    hooks:
+      - id: gitleaks

.python-version

@@ -0,0 +1 @@
+3.11.8

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "[python]": {
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.fixAll": "explicit",
+            "source.organizeImports": "explicit"
+        },
+        "editor.defaultFormatter": "charliermarsh.ruff"
+    },
+    "notebook.formatOnSave.enabled": true,
+    "notebook.codeActionsOnSave": {
+        "notebook.source.fixAll": "explicit",
+        "notebook.source.organizeImports": "explicit"
+    },
+}

Chat with your Video Library – Engineering AI Agents.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c32250914f76da5d6294d4dd86e6540cff5278bee30cbc4e4d1571fe26403c46
+size 2216057

Dockerfile

@@ -0,0 +1,47 @@
+FROM python:3.11-slim-bullseye AS release
+
+ENV WORKSPACE_ROOT=/app/
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV POETRY_VERSION=1.8.3
+ENV DEBIAN_FRONTEND=noninteractive
+ENV POETRY_NO_INTERACTION=1
+
+# Install Google Chrome
+RUN apt-get update -y && \
+    apt-get install -y gnupg wget curl --no-install-recommends && \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/google-linux-signing-key.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/google-linux-signing-key.gpg] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \
+    apt-get update -y && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install other system dependencies.
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends build-essential \
+    gcc \
+    python3-dev \
+    build-essential \
+    libglib2.0-dev \
+    libnss3-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Poetry using pip and clear cache
+RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
+RUN poetry config installer.max-workers 20
+
+WORKDIR $WORKSPACE_ROOT
+
+# Copy the poetry lock file and pyproject.toml file to install dependencies
+COPY pyproject.toml poetry.lock $WORKSPACE_ROOT
+
+# Install the dependencies and clear cache
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root --no-interaction --no-cache --without dev && \
+    poetry self add 'poethepoet[poetry_plugin]' && \
+    rm -rf ~/.cache/pypoetry/cache/ && \
+    rm -rf ~/.cache/pypoetry/artifacts/
+
+# Copy the rest of the code.
+COPY . $WORKSPACE_ROOT

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Packt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,666 @@
 ---
-title: LLM Engineers Handbook
-emoji: 🦀
-colorFrom: indigo
-colorTo: blue
+title: LLM-Engineers-Handbook
+app_file: demonstration.ipynb
 sdk: gradio
 sdk_version: 5.29.0
-app_file: app.py
-pinned: false
 ---
+<div align="center">
+  <h1>👷 LLM Engineer's Handbook</h1>
+  <p class="tagline">Official repository of the <a href="https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072/">LLM Engineer's Handbook</a> by <a href="https://github.com/iusztinpaul">Paul Iusztin</a> and <a href="https://github.com/mlabonne">Maxime Labonne</a></p>
+</div>
+</br>
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<p align="center">
+  <a href="https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072/">
+    <img src="images/cover_plus.png" alt="Book cover">
+  </a>
+</p>
+
+<p align="center">
+  Find the book on <a href="https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072/">Amazon</a> or <a href="https://www.packtpub.com/en-us/product/llm-engineers-handbook-9781836200062">Packt</a>
+</p>
+
+## 🌟 Features
+
+The goal of this book is to create your own end-to-end LLM-based system using best practices:
+
+- 📝 Data collection & generation
+- 🔄 LLM training pipeline
+- 📊 Simple RAG system
+- 🚀 Production-ready AWS deployment
+- 🔍 Comprehensive monitoring
+- 🧪 Testing and evaluation framework
+
+You can download and use the final trained model on [Hugging Face](https://huggingface.co/mlabonne/TwinLlama-3.1-8B-DPO).
+
+> [!IMPORTANT]
+> The code in this GitHub repository is actively maintained and may contain updates not reflected in the book. **Always refer to this repository for the latest version of the code.**
+
+## 🔗 Dependencies
+
+### Local dependencies
+
+To install and run the project locally, you need the following dependencies.
+
+| Tool | Version | Purpose | Installation Link |
+|------|---------|---------|------------------|
+| pyenv | ≥2.3.36 | Multiple Python versions (optional) | [Install Guide](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation) |
+| Python | 3.11 | Runtime environment | [Download](https://www.python.org/downloads/) |
+| Poetry | >= 1.8.3 and < 2.0 | Package management | [Install Guide](https://python-poetry.org/docs/#installation) |
+| Docker | ≥27.1.1 | Containerization | [Install Guide](https://docs.docker.com/engine/install/) |
+| AWS CLI | ≥2.15.42 | Cloud management | [Install Guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) |
+| Git | ≥2.44.0 | Version control | [Download](https://git-scm.com/downloads) |
+
+### Cloud services
+
+The code also uses and depends on the following cloud services. For now, you don't have to do anything. We will guide you in the installation and deployment sections on how to use them:
+
+| Service | Purpose |
+|---------|---------|
+| [HuggingFace](https://huggingface.com/) | Model registry |
+| [Comet ML](https://www.comet.com/site/products/opik/?utm_source=llm_handbook&utm_medium=github&utm_campaign=opik) | Experiment tracker |
+| [Opik](https://www.comet.com/site/products/opik/?utm_source=llm_handbook&utm_medium=github&utm_campaign=opik) | Prompt monitoring |
+| [ZenML](https://www.zenml.io/) | Orchestrator and artifacts layer |
+| [AWS](https://aws.amazon.com/) | Compute and storage |
+| [MongoDB](https://www.mongodb.com/) | NoSQL database |
+| [Qdrant](https://qdrant.tech/) | Vector database |
+| [GitHub Actions](https://github.com/features/actions) | CI/CD pipeline |
+
+In the [LLM Engineer's Handbook](https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072/), Chapter 2 will walk you through each tool. Chapters 10 and 11 provide step-by-step guides on how to set up everything you need.
+
+## 🗂️ Project Structure
+
+Here is the directory overview:
+
+```bash
+.
+├── code_snippets/       # Standalone example code
+├── configs/             # Pipeline configuration files
+├── llm_engineering/     # Core project package
+│   ├── application/    
+│   ├── domain/         
+│   ├── infrastructure/ 
+│   ├── model/         
+├── pipelines/           # ML pipeline definitions
+├── steps/               # Pipeline components
+├── tests/               # Test examples
+├── tools/               # Utility scripts
+│   ├── run.py
+│   ├── ml_service.py
+│   ├── rag.py
+│   ├── data_warehouse.py
+```
+
+`llm_engineering/`  is the main Python package implementing LLM and RAG functionality. It follows Domain-Driven Design (DDD) principles:
+
+- `domain/`: Core business entities and structures
+- `application/`: Business logic, crawlers, and RAG implementation
+- `model/`: LLM training and inference
+- `infrastructure/`: External service integrations (AWS, Qdrant, MongoDB, FastAPI)
+
+The code logic and imports flow as follows: `infrastructure` → `model` → `application` → `domain`
+
+`pipelines/`: Contains the ZenML ML pipelines, which serve as the entry point for all the ML pipelines. Coordinates the data processing and model training stages of the ML lifecycle.
+
+`steps/`: Contains individual ZenML steps, which are reusable components for building and customizing ZenML pipelines. Steps perform specific tasks (e.g., data loading, preprocessing) and can be combined within the ML pipelines.
+
+`tests/`: Covers a few sample tests used as examples within the CI pipeline.
+
+`tools/`: Utility scripts used to call the ZenML pipelines and inference code:
+- `run.py`: Entry point script to run ZenML pipelines.
+- `ml_service.py`: Starts the REST API inference server.
+- `rag.py`: Demonstrates usage of the RAG retrieval module.
+- `data_warehouse.py`: Used to export or import data from the MongoDB data warehouse through JSON files.
+
+`configs/`: ZenML YAML configuration files to control the execution of pipelines and steps.
+
+`code_snippets/`: Independent code examples that can be executed independently.
+
+## 💻 Installation
+
+> [!NOTE]
+> If you are experiencing issues while installing and running the repository, consider checking the [Issues](https://github.com/PacktPublishing/LLM-Engineers-Handbook/issues) GitHub section for other people who solved similar problems or directly asking us for help.
+
+### 1. Clone the Repository
+
+Start by cloning the repository and navigating to the project directory:
+
+```bash
+git clone https://github.com/PacktPublishing/LLM-Engineers-Handbook.git
+cd LLM-Engineers-Handbook 
+```
+
+Next, we have to prepare your Python environment and its adjacent dependencies. 
+
+### 2. Set Up Python Environment
+
+The project requires Python 3.11. You can either use your global Python installation or set up a project-specific version using pyenv.
+
+#### Option A: Using Global Python (if version 3.11 is installed)
+
+Verify your Python version:
+
+```bash
+python --version  # Should show Python 3.11.x
+```
+
+#### Option B: Using pyenv (recommended)
+
+1. Verify pyenv installation:
+
+```bash
+pyenv --version   # Should show pyenv 2.3.36 or later
+```
+
+2. Install Python 3.11.8:
+
+```bash
+pyenv install 3.11.8
+```
+
+3. Verify the installation:
+
+```bash
+python --version  # Should show Python 3.11.8
+```
+
+4. Confirm Python version in the project directory:
+
+```bash
+python --version
+# Output: Python 3.11.8
+```
+
+> [!NOTE]  
+> The project includes a `.python-version` file that automatically sets the correct Python version when you're in the project directory.
+
+### 3. Install Dependencies
+
+The project uses Poetry for dependency management.
+
+1. Verify Poetry installation:
+
+```bash
+poetry --version  # Should show Poetry version 1.8.3 or later
+```
+
+2. Set up the project environment and install dependencies:
+
+```bash
+poetry env use 3.11
+poetry install --without aws
+poetry run pre-commit install
+```
+
+This will:
+
+- Configure Poetry to use Python 3.11
+- Install project dependencies (excluding AWS-specific packages)
+- Set up pre-commit hooks for code verification
+
+### 4. Activate the Environment
+
+As our task manager, we run all the scripts using [Poe the Poet](https://poethepoet.natn.io/index.html).
+
+1. Start a Poetry shell:
+
+```bash
+poetry shell
+```
+
+2. Run project commands using Poe the Poet:
+
+```bash
+poetry poe ...
+```
+
+<details>
+<summary>🔧 Troubleshooting Poe the Poet Installation</summary>
+
+### Alternative Command Execution
+
+If you're experiencing issues with `poethepoet`, you can still run the project commands directly through Poetry. Here's how:
+
+1. Look up the command definition in `pyproject.toml`
+2. Use `poetry run` with the underlying command
+
+#### Example:
+Instead of:
+```bash
+poetry poe local-infrastructure-up
+```
+Use the direct command from pyproject.toml:
+```bash
+poetry run <actual-command-from-pyproject-toml>
+```
+Note: All project commands are defined in the [tool.poe.tasks] section of pyproject.toml
+</details>
+
+Now, let's configure our local project with all the necessary credentials and tokens to run the code locally.
+
+### 5. Local Development Setup
+
+After you have installed all the dependencies, you must create and fill a `.env` file with your credentials to appropriately interact with other services and run the project. Setting your sensitive credentials in a `.env` file is a good security practice, as this file won't be committed to GitHub or shared with anyone else. 
+
+1. First, copy our example by running the following:
+
+```bash
+cp .env.example .env # The file must be at your repository's root!
+```
+
+2. Now, let's understand how to fill in all the essential variables within the `.env` file to get you started. The following are the mandatory settings we must complete when working locally:
+
+#### OpenAI
+
+To authenticate to OpenAI's API, you must fill out the `OPENAI_API_KEY` env var with an authentication token.
+
+```env
+OPENAI_API_KEY=your_api_key_here
+```
+
+→ Check out this [tutorial](https://platform.openai.com/docs/quickstart) to learn how to provide one from OpenAI.
+
+#### Hugging Face
+
+To authenticate to Hugging Face, you must fill out the `HUGGINGFACE_ACCESS_TOKEN` env var with an authentication token.
+
+```env
+HUGGINGFACE_ACCESS_TOKEN=your_token_here
+```
+
+→ Check out this [tutorial](https://huggingface.co/docs/hub/en/security-tokens) to learn how to provide one from Hugging Face.
+
+#### Comet ML & Opik
+
+To authenticate to Comet ML (required only during training) and Opik, you must fill out the `COMET_API_KEY` env var with your authentication token.
+
+```env
+COMET_API_KEY=your_api_key_here
+```
+
+→ Check out this [tutorial](https://www.comet.com/docs/opik/?utm_source=llm_handbook&utm_medium=github&utm_campaign=opik) to learn how to get started with Opik. You can also access Opik's dashboard using 🔗[this link](https://www.comet.com/opik?utm_source=llm_handbook&utm_medium=github&utm_content=opik).
+
+### 6. Deployment Setup
+
+When deploying the project to the cloud, we must set additional settings for Mongo, Qdrant, and AWS. If you are just working locally, the default values of these env vars will work out of the box. Detailed deployment instructions are available in Chapter 11 of the [LLM Engineer's Handbook](https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072/).
+
+#### MongoDB
+
+We must change the `DATABASE_HOST` env var with the URL pointing to your cloud MongoDB cluster.
+
+```env
+DATABASE_HOST=your_mongodb_url
+```
+
+→ Check out this [tutorial](https://www.mongodb.com/resources/products/fundamentals/mongodb-cluster-setup) to learn how to create and host a MongoDB cluster for free.
+
+#### Qdrant
+
+Change `USE_QDRANT_CLOUD` to `true`, `QDRANT_CLOUD_URL` with the URL point to your cloud Qdrant cluster, and `QDRANT_APIKEY` with its API key.
+
+```env
+USE_QDRANT_CLOUD=true
+QDRANT_CLOUD_URL=your_qdrant_cloud_url
+QDRANT_APIKEY=your_qdrant_api_key
+```
+
+→ Check out this [tutorial](https://qdrant.tech/documentation/cloud/create-cluster/) to learn how to create a Qdrant cluster for free
+
+#### AWS
+
+For your AWS set-up to work correctly, you need the AWS CLI installed on your local machine and properly configured with an admin user (or a user with enough permissions to create new SageMaker, ECR, and S3 resources; using an admin user will make everything more straightforward).
+
+Chapter 2 provides step-by-step instructions on how to install the AWS CLI, create an admin user on AWS, and get an access key to set up the `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` environment variables. If you already have an AWS admin user in place, you have to configure the following env vars in your `.env` file:
+
+```bash
+AWS_REGION=eu-central-1 # Change it with your AWS region.
+AWS_ACCESS_KEY=your_aws_access_key
+AWS_SECRET_KEY=your_aws_secret_key
+```
+
+AWS credentials are typically stored in `~/.aws/credentials`. You can view this file directly using `cat` or similar commands:
+
+```bash
+cat ~/.aws/credentials
+```
+
+> [!IMPORTANT]
+> Additional configuration options are available in [settings.py](https://github.com/PacktPublishing/LLM-Engineers-Handbook/blob/main/llm_engineering/settings.py). Any variable in the `Settings` class can be configured through the `.env` file. 
+
+## 🏗️ Infrastructure
+
+### Local infrastructure (for testing and development)
+
+When running the project locally, we host a MongoDB and Qdrant database using Docker. Also, a testing ZenML server is made available through their Python package.
+
+> [!WARNING]
+> You need Docker installed (>= v27.1.1)
+
+For ease of use, you can start the whole local development infrastructure with the following command:
+```bash
+poetry poe local-infrastructure-up
+```
+
+Also, you can stop the ZenML server and all the Docker containers using the following command:
+```bash
+poetry poe local-infrastructure-down
+```
+
+> [!WARNING]  
+> When running on MacOS, before starting the server, export the following environment variable:
+> `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`
+> Otherwise, the connection between the local server and pipeline will break. 🔗 More details in [this issue](https://github.com/zenml-io/zenml/issues/2369).
+> This is done by default when using Poe the Poet.
+
+Start the inference real-time RESTful API:
+```bash
+poetry poe run-inference-ml-service
+```
+
+> [!IMPORTANT]
+> The LLM microservice, called by the RESTful API, will work only after deploying the LLM to AWS SageMaker.
+
+#### ZenML
+
+Dashboard URL: `localhost:8237`
+
+Default credentials:
+  - `username`: default
+  - `password`: 
+
+→ Find out more about using and setting up [ZenML](https://docs.zenml.io/).
+
+#### Qdrant
+
+REST API URL: `localhost:6333`
+
+Dashboard URL: `localhost:6333/dashboard`
+
+→ Find out more about using and setting up [Qdrant with Docker](https://qdrant.tech/documentation/quick-start/).
+
+#### MongoDB
+
+Database URI: `mongodb://llm_engineering:[email protected]:27017`
+
+Database name: `twin`
+
+Default credentials:
+  - `username`: llm_engineering
+  - `password`: llm_engineering
+
+→ Find out more about using and setting up [MongoDB with Docker](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-community-with-docker).
+
+You can search your MongoDB collections using your **IDEs MongoDB plugin** (which you have to install separately), where you have to use the database URI to connect to the MongoDB database hosted within the Docker container: `mongodb://llm_engineering:[email protected]:27017`
+
+> [!IMPORTANT]
+> Everything related to training or running the LLMs (e.g., training, evaluation, inference) can only be run if you set up AWS SageMaker, as explained in the next section on cloud infrastructure.
+
+### Cloud infrastructure (for production)
+
+Here we will quickly present how to deploy the project to AWS and other serverless services. We won't go into the details (as everything is presented in the book) but only point out the main steps you have to go through.
+
+First, reinstall your Python dependencies with the AWS group:
+```bash
+poetry install --with aws
+```
+
+#### AWS SageMaker
+
+> [!NOTE]
+> Chapter 10 provides step-by-step instructions in the section "Implementing the LLM microservice using AWS SageMaker".
+
+By this point, we expect you to have AWS CLI installed and your AWS CLI and project's env vars (within the `.env` file) properly configured with an AWS admin user.
+
+To ensure best practices, we must create a new AWS user restricted to creating and deleting only resources related to AWS SageMaker. Create it by running:
+```bash
+poetry poe create-sagemaker-role
+```
+It will create a `sagemaker_user_credentials.json` file at the root of your repository with your new `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` values. **But before replacing your new AWS credentials, also run the following command to create the execution role (to create it using your admin credentials).**
+
+To create the IAM execution role used by AWS SageMaker to access other AWS resources on our behalf, run the following:
+```bash
+poetry poe create-sagemaker-execution-role
+```
+It will create a `sagemaker_execution_role.json` file at the root of your repository with your new `AWS_ARN_ROLE` value. Add it to your `.env` file. 
+
+Once you've updated the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`, and `AWS_ARN_ROLE` values in your `.env` file, you can use AWS SageMaker. **Note that this step is crucial to complete the AWS setup.**
+
+#### Training
+
+We start the training pipeline through ZenML by running the following:
+```bash
+poetry poe run-training-pipeline
+```
+This will start the training code using the configs from `configs/training.yaml` directly in SageMaker. You can visualize the results in Comet ML's dashboard.
+
+We start the evaluation pipeline through ZenML by running the following:
+```bash
+poetry poe run-evaluation-pipeline
+```
+This will start the evaluation code using the configs from `configs/evaluating.yaml` directly in SageMaker. You can visualize the results in `*-results` datasets saved to your Hugging Face profile.
+
+#### Inference
+
+To create an AWS SageMaker Inference Endpoint, run:
+```bash
+poetry poe deploy-inference-endpoint
+```
+To test it out, run:
+```bash
+poetry poe test-sagemaker-endpoint
+```
+To delete it, run:
+```bash
+poetry poe delete-inference-endpoint
+```
+
+#### AWS: ML pipelines, artifacts, and containers
+
+The ML pipelines, artifacts, and containers are deployed to AWS by leveraging ZenML's deployment features. Thus, you must create an account with ZenML Cloud and follow their guide on deploying a ZenML stack to AWS. Otherwise, we provide step-by-step instructions in **Chapter 11**, section **Deploying the LLM Twin's pipelines to the cloud** on what you must do.  
+
+#### Qdrant & MongoDB
+
+We leverage Qdrant's and MongoDB's serverless options when deploying the project. Thus, you can either follow [Qdrant's](https://qdrant.tech/documentation/cloud/create-cluster/) and [MongoDB's](https://www.mongodb.com/resources/products/fundamentals/mongodb-cluster-setup) tutorials on how to create a freemium cluster for each or go through **Chapter 11**, section **Deploying the LLM Twin's pipelines to the cloud** and follow our step-by-step instructions.
+
+#### GitHub Actions
+
+We use GitHub Actions to implement our CI/CD pipelines. To implement your own, you have to fork our repository and set the following env vars as Actions secrets in your forked repository:
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `AWS_ECR_NAME`
+- `AWS_REGION`
+
+Also, we provide instructions on how to set everything up in **Chapter 11**, section **Adding LLMOps to the LLM Twin**.
+
+#### Comet ML & Opik
+
+You can visualize the results on their self-hosted dashboards if you create a Comet account and correctly set the `COMET_API_KEY` env var. As Opik is powered by Comet, you don't have to set up anything else along Comet:
+- [Comet ML (for experiment tracking)](https://www.comet.com/?utm_source=llm_handbook&utm_medium=github&utm_campaign=opik)
+- [Opik (for prompt monitoring)](https://www.comet.com/opik?utm_source=llm_handbook&utm_medium=github&utm_campaign=opik)
+
+### 💰 Running the Project Costs
+
+We will mostly stick to free tiers for all the services except for AWS and OpenAI's API, which are both pay-as-you-go services. The cost of running the project once, with our default values, will be roughly ~$25 (most of it comes from using AWS SageMaker for training and inference).
+
+## ⚡ Pipelines
+
+All the ML pipelines will be orchestrated behind the scenes by [ZenML](https://www.zenml.io/). A few exceptions exist when running utility scrips, such as exporting or importing from the data warehouse.
+
+The ZenML pipelines are the entry point for most processes throughout this project. They are under the `pipelines/` folder. Thus, when you want to understand or debug a workflow, starting with the ZenML pipeline is the best approach.
+
+To see the pipelines running and their results:
+- go to your ZenML dashboard
+- go to the `Pipelines` section
+- click on a specific pipeline (e.g., `feature_engineering`)
+- click on a specific run (e.g., `feature_engineering_run_2024_06_20_18_40_24`)
+- click on a specific step or artifact of the DAG to find more details about it
+
+Now, let's explore all the pipelines you can run. From data collection to training, we will present them in their natural order to go through the LLM project end-to-end.
+
+### Data pipelines
+
+Run the data collection ETL:
+```bash
+poetry poe run-digital-data-etl
+```
+
+> [!WARNING]
+> You must have Chrome (or another Chromium-based browser) installed on your system for LinkedIn and Medium crawlers to work (which use Selenium under the hood). Based on your Chrome version, the Chromedriver will be automatically installed to enable Selenium support. Another option is to run everything using our Docker image if you don't want to install Chrome. For example, to run all the pipelines combined you can run `poetry poe run-docker-end-to-end-data-pipeline`. Note that the command can be tweaked to support any other pipeline.
+>
+> If, for any other reason, you don't have a Chromium-based browser installed and don't want to use Docker, you have two other options to bypass this Selenium issue:
+> - Comment out all the code related to Selenium, Chrome and all the links that use Selenium to crawl them (e.g., Medium), such as the `chromedriver_autoinstaller.install()` command from [application.crawlers.base](https://github.com/PacktPublishing/LLM-Engineers-Handbook/blob/main/llm_engineering/application/crawlers/base.py) and other static calls that check for Chrome drivers and Selenium.
+> - Install Google Chrome using your CLI in environments such as GitHub Codespaces or other cloud VMs using the same command as in our [Docker file](https://github.com/PacktPublishing/LLM-Engineers-Handbook/blob/main/Dockerfile#L10).
+
+To add additional links to collect from, go to `configs/digital_data_etl_[author_name].yaml` and add them to the `links` field. Also, you can create a completely new file and specify it at run time, like this: `python -m llm_engineering.interfaces.orchestrator.run --run-etl --etl-config-filename configs/digital_data_etl_[your_name].yaml`
+
+Run the feature engineering pipeline:
+```bash
+poetry poe run-feature-engineering-pipeline
+```
+
+Generate the instruct dataset:
+```bash
+poetry poe run-generate-instruct-datasets-pipeline
+```
+
+Generate the preference dataset:
+```bash
+poetry poe run-generate-preference-datasets-pipeline
+```
+
+Run all of the above compressed into a single pipeline:
+```bash
+poetry poe run-end-to-end-data-pipeline
+```
+
+### Utility pipelines
+
+Export the data from the data warehouse to JSON files:
+```bash
+poetry poe run-export-data-warehouse-to-json
+```
+
+Import data to the data warehouse from JSON files (by default, it imports the data from the `data/data_warehouse_raw_data` directory):
+```bash
+poetry poe run-import-data-warehouse-from-json
+```
+
+Export ZenML artifacts to JSON:
+```bash
+poetry poe run-export-artifact-to-json-pipeline
+```
+
+This will export the following ZenML artifacts to the `output` folder as JSON files (it will take their latest version):
+- cleaned_documents.json
+- instruct_datasets.json
+- preference_datasets.json
+- raw_documents.json
+
+You can configure what artifacts to export by tweaking the `configs/export_artifact_to_json.yaml` configuration file.
+
+### Training pipelines
+
+Run the training pipeline:
+```bash
+poetry poe run-training-pipeline
+```
+
+Run the evaluation pipeline:
+```bash
+poetry poe run-evaluation-pipeline
+```
+
+> [!WARNING]
+> For this to work, make sure you properly configured AWS SageMaker as described in [Set up cloud infrastructure (for production)](#set-up-cloud-infrastructure-for-production).
+
+### Inference pipelines
+
+Call the RAG retrieval module with a test query:
+```bash
+poetry poe call-rag-retrieval-module
+```
+
+Start the inference real-time RESTful API:
+```bash
+poetry poe run-inference-ml-service
+```
+
+Call the inference real-time RESTful API with a test query:
+```bash
+poetry poe call-inference-ml-service
+```
+
+Remember that you can monitor the prompt traces on [Opik](https://www.comet.com/opik).
+
+> [!WARNING]
+> For the inference service to work, you must have the LLM microservice deployed to AWS SageMaker, as explained in the setup cloud infrastructure section.
+
+### Linting & formatting (QA)
+
+Check or fix your linting issues:
+```bash
+poetry poe lint-check
+poetry poe lint-fix
+```
+
+Check or fix your formatting issues:
+```bash
+poetry poe format-check
+poetry poe format-fix
+```
+
+Check the code for leaked credentials:
+```bash
+poetry poe gitleaks-check
+```
+
+### Tests
+
+Run all the tests using the following command:
+```bash
+poetry poe test
+```
+
+## 🏃 Run project
+
+Based on the setup and usage steps described above, assuming the local and cloud infrastructure works and the `.env` is filled as expected, follow the next steps to run the LLM system end-to-end:
+
+### Data
+
+1. Collect data: `poetry poe run-digital-data-etl`
+
+2. Compute features: `poetry poe run-feature-engineering-pipeline`
+
+3. Compute instruct dataset: `poetry poe run-generate-instruct-datasets-pipeline`
+
+4. Compute preference alignment dataset: `poetry poe run-generate-preference-datasets-pipeline`
+
+### Training
+
+> [!IMPORTANT]
+> From now on, for these steps to work, you need to properly set up AWS SageMaker, such as running `poetry install --with aws` and filling in the AWS-related environment variables and configs.
+
+5. SFT fine-tuning Llamma 3.1: `poetry poe run-training-pipeline`
+
+6. For DPO, go to `configs/training.yaml`, change `finetuning_type` to `dpo`, and run `poetry poe run-training-pipeline` again
+
+7. Evaluate fine-tuned models: `poetry poe run-evaluation-pipeline`
+
+### Inference
+
+> [!IMPORTANT]
+> From now on, for these steps to work, you need to properly set up AWS SageMaker, such as running `poetry install --with aws` and filling in the AWS-related environment variables and configs.
+
+8. Call only the RAG retrieval module: `poetry poe call-rag-retrieval-module`
+
+9. Deploy the LLM Twin microservice to SageMaker: `poetry poe deploy-inference-endpoint`
+
+10. Test the LLM Twin microservice: `poetry poe test-sagemaker-endpoint`
+
+11. Start end-to-end RAG server: `poetry poe run-inference-ml-service`
+
+12. Test RAG server: `poetry poe call-inference-ml-service`
+
+## 📄 License
+
+This course is an open-source project released under the MIT license. Thus, as long you distribute our LICENSE and acknowledge our work, you can safely clone or fork this project and use it as a source of inspiration for whatever you want (e.g., university projects, college degree projects, personal projects, etc.).

clips/clip_eFgkZKhNUdM_1270_0.847.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e29cb777c6b6984f5599a455d760923147593388b1a9076e6f9afe0bbf679d3c
+size 7315834

clips/clip_eFgkZKhNUdM_642_0.847.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5653d4b661233328997880cd40df4aba4e74d3aca5198685f3b4aaaac0e03a3
+size 1628484

clips/clip_eFgkZKhNUdM_874_0.838.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be8086177badfba4783ccca4040c990ea0a16ebc8d33185dce7a354e43720317
+size 3585253

code_snippets/03_custom_odm_example.py

@@ -0,0 +1,10 @@
+from llm_engineering.domain.documents import ArticleDocument, UserDocument
+
+if __name__ == "__main__":
+    user = UserDocument.get_or_create(first_name="Paul", last_name="Iusztin")
+    articles = ArticleDocument.bulk_find(author_id=str(user.id))
+
+    print(f"User ID: {user.id}")  # noqa
+    print(f"User name: {user.first_name} {user.last_name}")  # noqa
+    print(f"Number of articles: {len(articles)}")  # noqa
+    print("First article link:", articles[0].link)  # noqa

code_snippets/03_orm.py

@@ -0,0 +1,37 @@
+from sqlalchemy import Column, Integer, String, create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+# Create virtual environment, install dependencies and run the code:
+# 1. Create: python3 -m venv orm_venv
+# 2. Activate: source orm_venv/bin/activate
+# 3. Install: pip install sqlalchemy==2.0.35
+# 4. Run the code: python code_snippets/03_orm.py
+
+if __name__ == "__main__":
+    Base = declarative_base()
+
+    # Define  a class that maps to the users table.
+    class User(Base):
+        __tablename__ = "users"
+
+        id = Column(Integer, primary_key=True)
+        name = Column(String)
+
+    # Create an SQLite database in memory.
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(engine)
+
+    # Create a session used to interact with the database.
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    # Add a new user.
+    new_user = User(name="Alice")
+    session.add(new_user)
+    session.commit()
+
+    # Query the database.
+    user = session.query(User).first()
+    if user:
+        print(f"User ID: {user.id}")  # noqa
+        print(f"User name: {user.name}")  # noqa

code_snippets/08_instructor_embeddings.py

@@ -0,0 +1,18 @@
+from sentence_transformers import SentenceTransformer
+
+# Create virtual environment, install dependencies and run the code:
+# 1. Create: python3 -m venv instructor_venv
+# 2. Activate: source instructor_venv/bin/activate
+# 3. Install: pip install sentence-transformers==3.3.0
+# 4. Run the code: python code_snippets/08_instructor_embeddings.py
+
+if __name__ == "__main__":
+    model = SentenceTransformer("hkunlp/instructor-base")
+
+    sentence = "RAG Fundamentals First"
+
+    instruction = "Represent the title of an article about AI:"
+
+    embeddings = model.encode([[instruction, sentence]])
+    print(embeddings.shape)  # noqa
+    # Output: (1, 768)

code_snippets/08_text_embeddings.py

@@ -0,0 +1,28 @@
+from sentence_transformers import SentenceTransformer
+
+# Leverage the Poetry virtual environment to run the code:
+# poetry run python code_snippets/08_text_embeddings.py
+
+if __name__ == "__main__":
+    # 1. Load a pretrained Sentence Transformer model.
+    model = SentenceTransformer("all-MiniLM-L6-v2")
+
+    # The sentences to encode.
+    sentences = ["The dog sits outside waiting for a treat.", "I am going swimming.", "The dog is swimming."]
+
+    # 2. Calculate embeddings.
+    embeddings = model.encode(sentences)
+    print(embeddings.shape)  # noqa
+    # Output: [3, 384]
+
+    # 3. Calculate the embedding similarities using cosine similarity.
+    similarities = model.similarity(embeddings, embeddings)
+    print(similarities)  # noqa
+    # Output:
+    # tensor([[ 1.0000, -0.0389,  0.2692],
+    #     [-0.0389,  1.0000,  0.3837],
+    #     [ 0.2692,  0.3837,  1.0000]])
+    #
+    # similarities[0, 0] = The similarity between the first sentence and itself.
+    # similarities[0, 1] = The similarity between the first and second sentence.
+    # similarities[2, 1] = The similarity between the third and second sentence.

code_snippets/08_text_image_embeddings.py

@@ -0,0 +1,37 @@
+from io import BytesIO
+
+import requests
+from PIL import Image
+from sentence_transformers import SentenceTransformer
+
+# Leverage the Poetry virtual environment to run the code:
+# poetry run python code_snippets/08_text_image_embeddings.py
+
+if __name__ == "__main__":
+    # Load an image with a crazy cat.
+    response = requests.get(
+        "https://github.com/PacktPublishing/LLM-Engineering/blob/main/images/crazy_cat.jpg?raw=true"
+    )
+    image = Image.open(BytesIO(response.content))
+
+    # Load CLIP model.
+    model = SentenceTransformer("clip-ViT-B-32")
+
+    # Encode the loaded image.
+    img_emb = model.encode(image)
+
+    # Encode text descriptions.
+    text_emb = model.encode(
+        [
+            "A crazy cat smiling.",
+            "A white and brown cat with a yellow bandana.",
+            "A man eating in the garden.",
+        ]
+    )
+    print(text_emb.shape)  # noqa
+    # Output: (3, 512)
+
+    # Compute similarities.
+    similarity_scores = model.similarity(img_emb, text_emb)
+    print(similarity_scores)  # noqa
+    # Output: tensor([[0.3068, 0.3300, 0.1719]])

configs/digital_data_etl_maxime_labonne.yaml

@@ -0,0 +1,38 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+    
+parameters:
+  user_full_name: Maxime Labonne # [First Name(s)] [Last Name]
+  links:
+    # Personal Blog
+    - https://mlabonne.github.io/blog/posts/2024-07-29_Finetune_Llama31.html
+    - https://mlabonne.github.io/blog/posts/2024-07-15_The_Rise_of_Agentic_Data_Generation.html
+    # Substack
+    - https://maximelabonne.substack.com/p/uncensor-any-llm-with-abliteration-d30148b7d43e
+    - https://maximelabonne.substack.com/p/create-mixtures-of-experts-with-mergekit-11b318c99562
+    - https://maximelabonne.substack.com/p/merge-large-language-models-with-mergekit-2118fb392b54
+    - https://maximelabonne.substack.com/p/fine-tune-a-mistral-7b-model-with-direct-preference-optimization-708042745aac
+    - https://maximelabonne.substack.com/p/exllamav2-the-fastest-library-to-run-llms-32aeda294d26
+    - https://maximelabonne.substack.com/p/quantize-llama-models-with-ggml-and-llama-cpp-3612dfbcc172
+    - https://maximelabonne.substack.com/p/a-beginners-guide-to-llm-fine-tuning-4bae7d4da672
+    - https://maximelabonne.substack.com/p/graph-convolutional-networks-introduction-to-gnns-24b3f60d6c95
+    - https://maximelabonne.substack.com/p/4-bit-quantization-with-gptq-36b0f4f02c34
+    - https://maximelabonne.substack.com/p/fine-tune-your-own-llama-2-model-in-a-colab-notebook-df9823a04a32
+    - https://maximelabonne.substack.com/p/introduction-to-weight-quantization-2494701b9c0c
+    - https://maximelabonne.substack.com/p/decoding-strategies-in-large-language-models-9733a8f70539
+    - https://maximelabonne.substack.com/p/the-art-of-spending-optimizing-your-marketing-budget-with-nonlinear-optimization-6c8a39afb3c2
+    - https://maximelabonne.substack.com/p/create-a-bot-to-find-diamonds-in-minecraft-d836606a993a
+    - https://maximelabonne.substack.com/p/constraint-programming-67ac16fa0c81
+    - https://maximelabonne.substack.com/p/how-to-design-the-most-powerful-graph-neural-network-3d18b07a6e66
+    - https://maximelabonne.substack.com/p/introduction-to-graphsage-in-python-a9e7f9ecf9d7
+    - https://maximelabonne.substack.com/p/graph-attention-networks-in-python-975736ac5c0c
+    - https://maximelabonne.substack.com/p/integer-programming-vs-linear-programming-in-python-f1be5bb4e60e
+    - https://maximelabonne.substack.com/p/introduction-to-linear-programming-in-python-9261e7eb44b
+    - https://maximelabonne.substack.com/p/what-is-a-tensor-in-deep-learning-6dedd95d6507
+    - https://maximelabonne.substack.com/p/efficiently-iterating-over-rows-in-a-pandas-dataframe-7dd5f9992c01
+    - https://maximelabonne.substack.com/p/q-learning-for-beginners-2837b777741
+    - https://maximelabonne.substack.com/p/how-to-start-machine-learning-for-developers-in-2022-390af12b193f

configs/digital_data_etl_paul_iusztin.yaml

@@ -0,0 +1,62 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+    
+parameters:
+  user_full_name: Paul Iusztin # [First Name(s)] [Last Name]
+  links:
+    # Medium (only articles that are not under the paid wall work)
+    - https://medium.com/decodingml/an-end-to-end-framework-for-production-ready-llm-systems-by-building-your-llm-twin-2cc6bb01141f
+    - https://medium.com/decodingml/a-real-time-retrieval-system-for-rag-on-social-media-data-9cc01d50a2a0
+    - https://medium.com/decodingml/sota-python-streaming-pipelines-for-fine-tuning-llms-and-rag-in-real-time-82eb07795b87
+    - https://medium.com/decodingml/the-4-advanced-rag-algorithms-you-must-know-to-implement-5d0c7f1199d2
+    - https://medium.com/decodingml/architect-scalable-and-cost-effective-llm-rag-inference-pipelines-73b94ef82a99
+    # Substack
+    - https://decodingml.substack.com/p/real-time-feature-pipelines-with?r=1ttoeh
+    - https://decodingml.substack.com/p/building-ml-systems-the-right-way?r=1ttoeh
+    - https://decodingml.substack.com/p/reduce-your-pytorchs-code-latency?r=1ttoeh
+    - https://decodingml.substack.com/p/llm-agents-demystified?r=1ttoeh
+    - https://decodingml.substack.com/p/scalable-rag-ingestion-pipeline-using?r=1ttoeh
+    - https://decodingml.substack.com/p/the-ultimate-mlops-tool?r=1ttoeh
+    - https://decodingml.substack.com/p/the-new-king-of-infrastructure-as?r=1ttoeh
+    - https://decodingml.substack.com/p/highly-scalable-data-ingestion-architecture?r=1ttoeh
+    - https://decodingml.substack.com/p/2-key-llmops-concepts?r=1ttoeh
+    - https://decodingml.substack.com/p/the-llm-twin-free-course-on-production?r=1ttoeh
+    - https://decodingml.substack.com/p/a-blueprint-for-designing-production?r=1ttoeh
+    - https://decodingml.substack.com/p/the-difference-between-development?r=1ttoeh
+    - https://decodingml.substack.com/p/architect-scalable-and-cost-effective?r=1ttoeh
+    - https://decodingml.substack.com/p/7-tips-to-reduce-your-vram-when-training?r=1ttoeh
+    - https://decodingml.substack.com/p/using-this-python-package-you-can?r=1ttoeh
+    - https://decodingml.substack.com/p/the-4-advanced-rag-algorithms-you?r=1ttoeh
+    - https://decodingml.substack.com/p/problems-deploying-your-ml-models?r=1ttoeh
+    - https://decodingml.substack.com/p/sota-python-streaming-pipelines-for?r=1ttoeh
+    - https://decodingml.substack.com/p/ready-for-production-ml-here-are?r=1ttoeh
+    - https://decodingml.substack.com/p/my-ml-monthly-learning-resource-recommendations?r=1ttoeh
+    - https://decodingml.substack.com/p/an-end-to-end-framework-for-production?r=1ttoeh
+    - https://decodingml.substack.com/p/upskill-your-llm-knowledge-base-with?r=1ttoeh
+    - https://decodingml.substack.com/p/want-to-learn-an-end-to-end-framework?r=1ttoeh
+    - https://decodingml.substack.com/p/my-favorite-way-to-implement-a-configuration?r=1ttoeh
+    - https://decodingml.substack.com/p/a-real-time-retrieval-system-for?r=1ttoeh
+    - https://decodingml.substack.com/p/4-key-decoding-strategies-for-llms?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-new-year-the-new-and-improved?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-8-types-of-mlops-tools-that-must?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-this-is-what-you-need-to-build?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-7-steps-on-how-to-fine-tune-an?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-how-do-you-generate-a-q-and-a?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-what-do-you-need-to-fine-tune?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-why-and-when-do-you-need-to-fine?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-how-to-implement-a-streaming?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-why-and-what-do-you-need-a-streaming?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-unwrapping-the-3-pipeline-design?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-how-to-design-an-llm-system-for?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-synced-vector-dbs-a-guide-to?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-what-is-the-difference-between?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-7-steps-to-build-a-production?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-chain-of-thought-reasoning-write?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-build-and-serve-a-production?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-4-key-ideas-you-must-know-to?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-how-to-add-real-time-monitoring?r=1ttoeh
+    - https://decodingml.substack.com/p/dml-top-6-ml-platform-features-you?r=1ttoeh

configs/end_to_end_data.yaml

@@ -0,0 +1,87 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  # Data ETL & Feature engineering pipelines parameters
+  author_links:
+    - user_full_name: Paul Iusztin # [First Name(s)] [Last Name]
+      links:
+        # Medium (only articles that are not under the paid wall work)
+        - https://medium.com/decodingml/an-end-to-end-framework-for-production-ready-llm-systems-by-building-your-llm-twin-2cc6bb01141f
+        - https://medium.com/decodingml/a-real-time-retrieval-system-for-rag-on-social-media-data-9cc01d50a2a0
+        - https://medium.com/decodingml/sota-python-streaming-pipelines-for-fine-tuning-llms-and-rag-in-real-time-82eb07795b87
+        - https://medium.com/decodingml/the-4-advanced-rag-algorithms-you-must-know-to-implement-5d0c7f1199d2
+        - https://medium.com/decodingml/architect-scalable-and-cost-effective-llm-rag-inference-pipelines-73b94ef82a99
+        # Substack
+        - https://decodingml.substack.com/p/a-blueprint-for-designing-production?r=1ttoeh
+        - https://decodingml.substack.com/p/the-difference-between-development?r=1ttoeh
+        - https://decodingml.substack.com/p/architect-scalable-and-cost-effective?r=1ttoeh
+        - https://decodingml.substack.com/p/7-tips-to-reduce-your-vram-when-training?r=1ttoeh
+        - https://decodingml.substack.com/p/using-this-python-package-you-can?r=1ttoeh
+        - https://decodingml.substack.com/p/the-4-advanced-rag-algorithms-you?r=1ttoeh
+        - https://decodingml.substack.com/p/problems-deploying-your-ml-models?r=1ttoeh
+        - https://decodingml.substack.com/p/sota-python-streaming-pipelines-for?r=1ttoeh
+        - https://decodingml.substack.com/p/ready-for-production-ml-here-are?r=1ttoeh
+        - https://decodingml.substack.com/p/ready-for-production-ml-here-are?r=1ttoeh
+        - https://decodingml.substack.com/p/my-ml-monthly-learning-resource-recommendations?r=1ttoeh
+        - https://decodingml.substack.com/p/an-end-to-end-framework-for-production?r=1ttoeh
+        - https://decodingml.substack.com/p/upskill-your-llm-knowledge-base-with?r=1ttoeh
+        - https://decodingml.substack.com/p/want-to-learn-an-end-to-end-framework?r=1ttoeh
+        - https://decodingml.substack.com/p/my-favorite-way-to-implement-a-configuration?r=1ttoeh
+        - https://decodingml.substack.com/p/a-real-time-retrieval-system-for?r=1ttoeh
+        - https://decodingml.substack.com/p/4-key-decoding-strategies-for-llms?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-new-year-the-new-and-improved?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-8-types-of-mlops-tools-that-must?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-this-is-what-you-need-to-build?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-7-steps-on-how-to-fine-tune-an?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-how-do-you-generate-a-q-and-a?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-what-do-you-need-to-fine-tune?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-why-and-when-do-you-need-to-fine?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-how-to-implement-a-streaming?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-why-and-what-do-you-need-a-streaming?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-unwrapping-the-3-pipeline-design?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-how-to-design-an-llm-system-for?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-synced-vector-dbs-a-guide-to?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-what-is-the-difference-between?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-7-steps-to-build-a-production?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-chain-of-thought-reasoning-write?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-build-and-serve-a-production?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-4-key-ideas-you-must-know-to?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-how-to-add-real-time-monitoring?r=1ttoeh
+        - https://decodingml.substack.com/p/dml-top-6-ml-platform-features-you?r=1ttoeh
+    - user_full_name: Maxime Labonne # [First Name(s)] [Last Name]
+      links:
+        # Substack
+        - https://maximelabonne.substack.com/p/uncensor-any-llm-with-abliteration-d30148b7d43e
+        - https://maximelabonne.substack.com/p/create-mixtures-of-experts-with-mergekit-11b318c99562
+        - https://maximelabonne.substack.com/p/merge-large-language-models-with-mergekit-2118fb392b54
+        - https://maximelabonne.substack.com/p/fine-tune-a-mistral-7b-model-with-direct-preference-optimization-708042745aac
+        - https://maximelabonne.substack.com/p/exllamav2-the-fastest-library-to-run-llms-32aeda294d26
+        - https://maximelabonne.substack.com/p/quantize-llama-models-with-ggml-and-llama-cpp-3612dfbcc172
+        - https://maximelabonne.substack.com/p/a-beginners-guide-to-llm-fine-tuning-4bae7d4da672
+        - https://maximelabonne.substack.com/p/graph-convolutional-networks-introduction-to-gnns-24b3f60d6c95
+        - https://maximelabonne.substack.com/p/4-bit-quantization-with-gptq-36b0f4f02c34
+        - https://maximelabonne.substack.com/p/fine-tune-your-own-llama-2-model-in-a-colab-notebook-df9823a04a32
+        - https://maximelabonne.substack.com/p/introduction-to-weight-quantization-2494701b9c0c
+        - https://maximelabonne.substack.com/p/decoding-strategies-in-large-language-models-9733a8f70539
+        - https://maximelabonne.substack.com/p/the-art-of-spending-optimizing-your-marketing-budget-with-nonlinear-optimization-6c8a39afb3c2
+        - https://maximelabonne.substack.com/p/create-a-bot-to-find-diamonds-in-minecraft-d836606a993a
+        - https://maximelabonne.substack.com/p/constraint-programming-67ac16fa0c81
+        - https://maximelabonne.substack.com/p/how-to-design-the-most-powerful-graph-neural-network-3d18b07a6e66
+        - https://maximelabonne.substack.com/p/introduction-to-graphsage-in-python-a9e7f9ecf9d7
+        - https://maximelabonne.substack.com/p/graph-attention-networks-in-python-975736ac5c0c
+        - https://maximelabonne.substack.com/p/integer-programming-vs-linear-programming-in-python-f1be5bb4e60e
+        - https://maximelabonne.substack.com/p/introduction-to-linear-programming-in-python-9261e7eb44b
+        - https://maximelabonne.substack.com/p/what-is-a-tensor-in-deep-learning-6dedd95d6507
+        - https://maximelabonne.substack.com/p/efficiently-iterating-over-rows-in-a-pandas-dataframe-7dd5f9992c01
+        - https://maximelabonne.substack.com/p/q-learning-for-beginners-2837b777741
+        - https://maximelabonne.substack.com/p/how-to-start-machine-learning-for-developers-in-2022-390af12b193f
+  # Generate instruct dataset pipeline parameters
+  test_split_size: 0.1
+  push_to_huggingface: false
+  dataset_id: pauliusztin/llmtwin
+  mock: false

configs/evaluating.yaml

@@ -0,0 +1,9 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  is_dummy: true # Change this to 'false' to run the evaluation on the full dataset.

configs/export_artifact_to_json.yaml

@@ -0,0 +1,13 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  artifact_names:
+    - raw_documents
+    - cleaned_documents
+    - instruct_datasets
+    - preference_datasets

configs/feature_engineering.yaml

@@ -0,0 +1,11 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+    
+parameters:
+  author_full_names:
+    - Maxime Labonne
+    - Paul Iusztin

configs/generate_instruct_datasets.yaml

@@ -0,0 +1,13 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  test_split_size: 0.1
+  dataset_type: "instruction"
+  push_to_huggingface: true
+  dataset_id: pauliusztin/llmtwin
+  mock: false

configs/generate_preference_datasets.yaml

@@ -0,0 +1,13 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  test_split_size: 0.05
+  dataset_type: "preference"
+  push_to_huggingface: true
+  dataset_id: pauliusztin/llmtwin-dpo
+  mock: false

configs/training.yaml

@@ -0,0 +1,14 @@
+settings:
+  docker:
+    parent_image: 992382797823.dkr.ecr.eu-central-1.amazonaws.com/zenml-rlwlcs:latest
+    skip_build: True
+  orchestrator.sagemaker:
+    synchronous: false
+
+parameters:
+  finetuning_type: sft
+  num_train_epochs: 3
+  per_device_train_batch_size: 2
+  learning_rate: 3e-4
+  dataset_huggingface_workspace: mlabonne
+  is_dummy: true # Change this to 'false' to run the training with the full dataset and epochs.

data/data_warehouse_raw_data/PostDocument.json

@@ -0,0 +1 @@
+[]

data/data_warehouse_raw_data/RepositoryDocument.json

@@ -0,0 +1 @@
+[]

data/data_warehouse_raw_data/UserDocument.json

@@ -0,0 +1 @@
+[{"first_name": "Maxime", "last_name": "Labonne", "_id": "eff74089-0271-4319-8543-745c087f4f61"}, {"first_name": "Paul", "last_name": "Iusztin", "_id": "b5fa1f08-75f0-402d-8e88-d1357e346d9e"}]

demonstration.ipynb

@@ -0,0 +1,1027 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OBCrx5fSG4Qm"
+      },
+      "source": [
+        "# CS-UY 4613: Project\n",
+        "\n",
+        "Yufei Zhen\n",
+        "\n",
+        "macOS: Ventura 13.3.1 (a), GPU: Apple M2 Max"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IptBGhoVG790"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "* video source: [https://www.youtube.com/@pantelism](https://www.youtube.com/@pantelism)\n",
+        "\n",
+        "* **option 1** (repository source: [https://github.com/PacktPublishing/LLM-Engineers-Handbook](https://github.com/PacktPublishing/LLM-Engineers-Handbook))\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "8i3CcnpG_VPn",
+        "outputId": "597a492a-6305-43a6-e94e-b74fa8a12d7b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Cloning into 'LLM-Engineers-Handbook'...\n",
+            "remote: Enumerating objects: 1970, done.\u001b[K\n",
+            "remote: Counting objects: 100% (515/515), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (138/138), done.\u001b[K\n",
+            "remote: Total 1970 (delta 414), reused 377 (delta 377), pack-reused 1455 (from 2)\u001b[K\n",
+            "Receiving objects: 100% (1970/1970), 4.77 MiB | 21.22 MiB/s, done.\n",
+            "Resolving deltas: 100% (1263/1263), done.\n"
+          ]
+        }
+      ],
+      "source": [
+        "# !git clone https://github.com/PacktPublishing/LLM-Engineers-Handbook.git"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# !poetry env use 3.11\n",
+        "# !poetry install --without aws\n",
+        "# !poetry run pre-commit install"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "MPS available: True\n",
+            "CUDA available: False\n"
+          ]
+        }
+      ],
+      "source": [
+        "import torch\n",
+        "print(f\"MPS available: {torch.backends.mps.is_available()}\")\n",
+        "print(f\"CUDA available: {torch.cuda.is_available()}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ufyNDhgOYiUh"
+      },
+      "source": [
+        "## RAG Architecture\n",
+        "\n",
+        "- Integrating into [https://github.com/PacktPublishing/LLM-Engineers-Handbook/tree/main/llm_engineering/application/rag](https://github.com/PacktPublishing/LLM-Engineers-Handbook/tree/main/llm_engineering/application/rag):\n",
+        "\n",
+        "- Directory overview: \n",
+        "\n",
+        "```\n",
+        ".\n",
+        "├── ... \n",
+        "├── clips/               # Generated video clip responses\n",
+        "├── llm_engineering/     # Core project package\n",
+        "│   ├── application/\n",
+        "│   │   ├── ...\n",
+        "│   │   ├── rag          # Main RAG architecture\n",
+        "│   │   │   ├── __init__.py\n",
+        "│   │   │   ├── base.py\n",
+        "│   │   │   ├── multimodel_dispatcher.py (new)\n",
+        "│   │   │   ├── pipeline.py (new)\n",
+        "│   │   │   ├── prompt_templates.py\n",
+        "│   │   │   ├── query_expansion.py\n",
+        "│   │   │   ├── reranking.py\n",
+        "│   │   │   ├── retriever.py (modified)\n",
+        "│   │   │   ├── self_query.py\n",
+        "│   │   │   ├── topic_retriever.py (new)\n",
+        "│   │   │   ├── video_ingetser.py (new)\n",
+        "│   │   │   ├── video_processor.py (new)\n",
+        "│   ├── domain/\n",
+        "│   │   ├── ...\n",
+        "│   │   ├── queries.py (modified)\n",
+        "│   │   ├── video_chunks.py (new)\n",
+        "├── demonstration.ipynb (YOU'RE HERE)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Video Ingestion"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "video_db = \"/Users/yufeizhen/Desktop/project/videos\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "\u001b[32m2025-05-04 03:25:21.777\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mllm_engineering.settings\u001b[0m:\u001b[36mload_settings\u001b[0m:\u001b[36m94\u001b[0m - \u001b[1mLoading settings from the ZenML secret store.\u001b[0m\n",
+            "\u001b[32m2025-05-04 03:25:21.929\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mllm_engineering.settings\u001b[0m:\u001b[36mload_settings\u001b[0m:\u001b[36m99\u001b[0m - \u001b[33m\u001b[1mFailed to load settings from the ZenML secret store. Defaulting to loading the settings from the '.env' file.\u001b[0m\n",
+            "\u001b[32m2025-05-04 03:25:22.015\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mllm_engineering.infrastructure.db.mongo\u001b[0m:\u001b[36m__new__\u001b[0m:\u001b[36m20\u001b[0m - \u001b[1mConnection to MongoDB with URI successful: mongodb://llm_engineering:[email protected]:27017\u001b[0m\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\u001b[1;35mPyTorch version 2.2.2 available.\u001b[0m\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "\u001b[32m2025-05-04 03:25:23.410\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mllm_engineering.infrastructure.db.qdrant\u001b[0m:\u001b[36m__new__\u001b[0m:\u001b[36m29\u001b[0m - \u001b[1mConnection to Qdrant DB with URI successful: str\u001b[0m\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\u001b[1;35mLoad pretrained SentenceTransformer: all-MiniLM-L6-v2\u001b[0m\n",
+            "Initializing fallback TextEmbedder\n",
+            "\u001b[1;35mLoad pretrained SentenceTransformer: all-MiniLM-L6-v2\u001b[0m\n",
+            "Loading CLIP model: openai/clip-vit-base-patch32\n",
+            "CLIP model loaded successfully\n",
+            "Initialized embedders\n",
+            "Loaded NLP model\n",
+            "Loaded BERTopic\n",
+            "Processing videos from: /Users/yufeizhen/Desktop/project/videos\n",
+            "Already processed 8 videos\n",
+            "Previously processed videos:\n",
+            "  - 9CGGh6ivg68\n",
+            "  - FCQ-rih6cHY\n",
+            "  - TV-DjM8242s\n",
+            "  - WXoOohWU28Y\n",
+            "  - eFgkZKhNUdM\n",
+            "  - eQ6UE968Xe4\n",
+            "  - lb_5AdUpfuA\n",
+            "  - rCVlIVKqqGE\n",
+            "Found 8 video folders\n",
+            "Will process 0 videos (8 skipped)\n",
+            "Skipping TV-DjM8242s (already processed)\n",
+            "Skipping eFgkZKhNUdM (already processed)\n",
+            "Skipping eQ6UE968Xe4 (already processed)\n",
+            "Skipping rCVlIVKqqGE (already processed)\n",
+            "Skipping lb_5AdUpfuA (already processed)\n",
+            "Skipping FCQ-rih6cHY (already processed)\n",
+            "Skipping 9CGGh6ivg68 (already processed)\n",
+            "Skipping WXoOohWU28Y (already processed)\n",
+            "\n",
+            "All videos processed!\n",
+            "Total processed videos: 8\n"
+          ]
+        }
+      ],
+      "source": [
+        "from llm_engineering.application.rag.video_ingester import VideoIngester\n",
+        "\n",
+        "ingester = VideoIngester(video_root=video_db)\n",
+        "# ingester.process_video_library(force_reprocess=True)\n",
+        "ingester.process_video_library()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Total stored vectors: 403\n"
+          ]
+        }
+      ],
+      "source": [
+        "from qdrant_client import QdrantClient\n",
+        "\n",
+        "client = QdrantClient(path=\"/Users/yufeizhen/Desktop/project/qdrant_storage\")\n",
+        "print(\"Total stored vectors:\", client.count(\"video_chunks\").count)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Video Q&A"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Initializing VideoQAEngine\n",
+            "Video root: /Users/yufeizhen/Desktop/project/videos\n",
+            "Qdrant storage path: /Users/yufeizhen/Desktop/project/qdrant_storage\n",
+            "Connected to Qdrant storage at: /Users/yufeizhen/Desktop/project/qdrant_storage\n",
+            "Available collections: collections=[CollectionDescription(name='video_chunks')]\n",
+            "Found video_chunks collection with 403 points\n",
+            "Initializing fallback TextEmbedder\n",
+            "\u001b[1;35mLoad pretrained SentenceTransformer: all-MiniLM-L6-v2\u001b[0m\n",
+            "Loading CLIP model: openai/clip-vit-base-patch32\n",
+            "CLIP model loaded successfully\n",
+            "VideoQAEngine initialized successfully\n"
+          ]
+        }
+      ],
+      "source": [
+        "from llm_engineering.application.rag.pipeline import VideoQAEngine\n",
+        "\n",
+        "engine = VideoQAEngine(video_root=video_db)\n",
+        "\n",
+        "def respond(question):\n",
+        "    clips = engine.ask(question)\n",
+        "    return [(str(clip[\"path\"]), f\"Relevance: {clip['score']:.2f}\") for clip in clips]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "question = \"Using only the videos, explain the the binary cross entropy loss function.\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\n",
+            "--- Processing query: 'Using only the videos, explain the the binary cross entropy loss function.' ---\n",
+            "Retrieving relevant video segments...\n",
+            "Encoding query with CLIP: 'Using only the videos, explain the the binary cros...'\n",
+            "Cleaned text for CLIP: Using only the videos, explain the the binary cros...\n",
+            "Query embedded successfully\n",
+            "Sending search request to Qdrant (attempt 1/5)\n",
+            "Creating fresh connection to Qdrant...\n",
+            "Search successful, found 3 results\n",
+            "Retrieval completed in 0.07 seconds\n",
+            "Found 3 relevant video segments\n",
+            "\n",
+            "Processing result 1/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 1270.0s - 1302.0s\n",
+            "  Score: 0.8472\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_1270_0.847.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processing result 2/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 642.0s - 647.0s\n",
+            "  Score: 0.8467\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_642_0.847.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processing result 3/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 874.0s - 882.0s\n",
+            "  Score: 0.8379\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_874_0.838.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processed 3 clips successfully\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "[('clips/clip_eFgkZKhNUdM_1270_0.847.mp4', 'Relevance: 0.85'),\n",
+              " ('clips/clip_eFgkZKhNUdM_642_0.847.mp4', 'Relevance: 0.85'),\n",
+              " ('clips/clip_eFgkZKhNUdM_874_0.838.mp4', 'Relevance: 0.84')]"
+            ]
+          },
+          "execution_count": 5,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "respond(question)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Gradio App"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttps://api.gradio.app/pkg-version\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n"
+          ]
+        }
+      ],
+      "source": [
+        "import gradio as gr\n",
+        "\n",
+        "interface = gr.Interface(\n",
+        "    fn=respond,\n",
+        "    inputs=gr.Textbox(label=\"Ask about the video content\"),\n",
+        "    outputs=gr.Gallery(label=\"Relevant Video Clips\"),\n",
+        "    examples=[\n",
+        "        [\"Using only the videos, explain how ResNets work.\"],\n",
+        "        [\"Using only the videos, explain the advantages of CNNs over fully connected networks.\"],\n",
+        "        [\"Using only the videos, explain the the binary cross entropy loss function.\"]\n",
+        "    ]\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "* Running on local URL:  http://127.0.0.1:7860\n",
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttp://127.0.0.1:7860/gradio_api/startup-events\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "\u001b[1;35mHTTP Request: HEAD \u001b[0m\u001b[34mhttp://127.0.0.1:7860/\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttps://api.gradio.app/v3/tunnel-request\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "* Running on public URL: https://382d4d0bacff86ee02.gradio.live\n",
+            "\n",
+            "This share link expires in 1 week. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n",
+            "\u001b[1;35mHTTP Request: HEAD \u001b[0m\u001b[34mhttps://382d4d0bacff86ee02.gradio.live\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n"
+          ]
+        },
+        {
+          "data": {
+            "text/html": [
+              "<div><iframe src=\"https://382d4d0bacff86ee02.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+            ],
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/plain": []
+          },
+          "execution_count": 5,
+          "metadata": {},
+          "output_type": "execute_result"
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\n",
+            "--- Processing query: 'Using only the videos, explain the the binary cross entropy loss function.' ---\n",
+            "Retrieving relevant video segments...\n",
+            "Encoding query with CLIP: 'Using only the videos, explain the the binary cros...'\n",
+            "Cleaned text for CLIP: Using only the videos, explain the the binary cross entropy loss function....\n",
+            "Cleaned text for CLIP: Using only the videos, explain the the binary cros...\n",
+            "Query embedded successfully\n",
+            "Sending search request to Qdrant (attempt 1/5)\n",
+            "Search successful, found 3 results\n",
+            "Retrieval completed in 0.34 seconds\n",
+            "Found 3 relevant video segments\n",
+            "\n",
+            "Processing result 1/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 1270.0s - 1302.0s\n",
+            "  Score: 0.8472\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_1270_0.847.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processing result 2/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 642.0s - 647.0s\n",
+            "  Score: 0.8467\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_642_0.847.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processing result 3/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 874.0s - 882.0s\n",
+            "  Score: 0.8379\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_874_0.838.mp4\n",
+            "  Clip created successfully\n",
+            "\n",
+            "Processed 3 clips successfully\n"
+          ]
+        }
+      ],
+      "source": [
+        "interface.launch(share=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Initializing VideoQAEngine\n",
+            "Video root: /Users/yufeizhen/Desktop/project/videos\n",
+            "Qdrant storage path: /Users/yufeizhen/Desktop/project/qdrant_storage\n",
+            "Connected to Qdrant storage at: /Users/yufeizhen/Desktop/project/qdrant_storage\n",
+            "Available collections: collections=[CollectionDescription(name='video_chunks')]\n",
+            "Found video_chunks collection with 403 points\n",
+            "Initializing fallback TextEmbedder\n",
+            "\u001b[1;35mLoad pretrained SentenceTransformer: all-MiniLM-L6-v2\u001b[0m\n",
+            "Loading CLIP model: openai/clip-vit-base-patch32\n",
+            "CLIP model loaded successfully\n",
+            "VideoQAEngine initialized successfully\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttps://api.gradio.app/pkg-version\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n"
+          ]
+        }
+      ],
+      "source": [
+        "import gradio as gr\n",
+        "from llm_engineering.application.rag.pipeline import VideoQAEngine\n",
+        "\n",
+        "# Initialize the VideoQAEngine with the video database\n",
+        "video_db = \"/Users/yufeizhen/Desktop/project/videos\"\n",
+        "engine = VideoQAEngine(video_root=video_db)\n",
+        "\n",
+        "# Define the chat function that processes messages and returns relevant video clips\n",
+        "def chat(message, history):\n",
+        "    # Process message to get relevant clips\n",
+        "    clips = engine.ask(message)\n",
+        "    \n",
+        "    # Format for display\n",
+        "    clips_gallery = [(str(clip[\"path\"]), \"Relevance: {:.2f}\".format(clip['score'])) for clip in clips]\n",
+        "    \n",
+        "    # Return both a text response and the clips\n",
+        "    return \"Here are the relevant video clips for: '{}'\".format(message), clips_gallery\n",
+        "\n",
+        "# Create a more flexible interface using Blocks\n",
+        "with gr.Blocks(theme=\"soft\") as demo:\n",
+        "    gr.Markdown(\"# Chat with your Video Library\")\n",
+        "    gr.Markdown(\"Ask questions about the video content and get relevant clips. You can continue the conversation with follow-up questions.\")\n",
+        "    \n",
+        "    # Create chatbot for conversation history\n",
+        "    chatbot = gr.Chatbot(height=300)\n",
+        "    \n",
+        "    # Create gallery to display video clips\n",
+        "    gallery = gr.Gallery(label=\"Relevant Video Clips\", show_label=True)\n",
+        "    \n",
+        "    # Create message input\n",
+        "    msg = gr.Textbox(\n",
+        "        placeholder=\"Ask about the video content...\", \n",
+        "        label=\"Your Question\",\n",
+        "        show_label=False\n",
+        "    )\n",
+        "    \n",
+        "    # Define clear button\n",
+        "    clear = gr.Button(\"Clear\")\n",
+        "    \n",
+        "    # Example questions\n",
+        "    gr.Examples(\n",
+        "        examples=[\n",
+        "            \"Using only the videos, explain how ResNets work.\",\n",
+        "            \"Using only the videos, explain the advantages of CNNs over fully connected networks.\",\n",
+        "            \"Using only the videos, explain the the binary cross entropy loss function.\"\n",
+        "        ],\n",
+        "        inputs=msg\n",
+        "    )\n",
+        "    \n",
+        "    # Define the chat function that updates both chatbot and gallery\n",
+        "    def respond(message, chat_history):\n",
+        "        # Get text response and clips\n",
+        "        response, clips = chat(message, chat_history)\n",
+        "        \n",
+        "        # Update chat history\n",
+        "        chat_history.append((message, response))\n",
+        "        \n",
+        "        # Return updated chat history and gallery\n",
+        "        return \"\", chat_history, clips\n",
+        "    \n",
+        "    # Set up the event handlers\n",
+        "    msg.submit(respond, [msg, chatbot], [msg, chatbot, gallery])\n",
+        "    clear.click(lambda: ([], [], None), None, [chatbot, gallery, msg])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "* Running on local URL:  http://127.0.0.1:7861\n",
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttp://127.0.0.1:7861/gradio_api/startup-events\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "\u001b[1;35mHTTP Request: HEAD \u001b[0m\u001b[34mhttp://127.0.0.1:7861/\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "\u001b[1;35mHTTP Request: GET \u001b[0m\u001b[34mhttps://api.gradio.app/v3/tunnel-request\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n",
+            "* Running on public URL: https://48d861a2319613eb9b.gradio.live\n",
+            "\n",
+            "This share link expires in 1 week. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n",
+            "\u001b[1;35mHTTP Request: HEAD \u001b[0m\u001b[34mhttps://48d861a2319613eb9b.gradio.live\u001b[1;35m \"HTTP/1.1 200 OK\"\u001b[0m\n"
+          ]
+        },
+        {
+          "data": {
+            "text/html": [
+              "<div><iframe src=\"https://48d861a2319613eb9b.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+            ],
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/plain": []
+          },
+          "execution_count": 10,
+          "metadata": {},
+          "output_type": "execute_result"
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\n",
+            "--- Processing query: 'Using only the videos, explain the the binary cross entropy loss function.' ---\n",
+            "Retrieving relevant video segments...\n",
+            "Encoding query with CLIP: 'Using only the videos, explain the the binary cros...'\n",
+            "Cleaned text for CLIP: Using only the videos, explain the the binary cros...\n",
+            "Query embedded successfully\n",
+            "Sending search request to Qdrant (attempt 1/5)\n",
+            "Creating fresh connection to Qdrant...\n",
+            "Search successful, found 3 results\n",
+            "Retrieval completed in 0.07 seconds\n",
+            "Found 3 relevant video segments\n",
+            "\n",
+            "Processing result 1/3:\n",
+            "  Video ID: eFgkZKhNUdM\n",
+            "  Timestamps: 1270.0s - 1302.0s\n",
+            "  Score: 0.8472\n",
+            "  Found alternative video path: /Users/yufeizhen/Desktop/project/videos/eFgkZKhNUdM/eFgkZKhNUdM.mp4\n",
+            "  Creating clip to: clips/clip_eFgkZKhNUdM_1270_0.847.mp4\n"
+          ]
+        }
+      ],
+      "source": [
+        "demo.launch(share=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "gFdZON-DKOlx",
+        "KVkt7HBUKTig",
+        "d9KkwKhjKXgr",
+        "6NQUu-rUKfln"
+      ],
+      "gpuType": "A100",
+      "machine_shape": "hm",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "llm-engineering---hH_lZm-py3.11",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.11.2"
+    },
+    "vscode": {
+      "interpreter": {
+        "hash": "b62b85bf5dbe6dfb396652aed826e60c089a2288076109f466cd63de941fd51e"
+      }
+    },
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "01c5ce2d579c4b7d8ba2607fc6d76328": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_15bf9c9d1cf44b5abee88dddc74a927b",
+              "IPY_MODEL_05fb5c50098f4e5997c07afed3b9383e",
+              "IPY_MODEL_cf25f66f1ca943be8af0c7ce14727f21"
+            ],
+            "layout": "IPY_MODEL_d3f93597cb6640a29656daa95153f07f"
+          }
+        },
+        "05fb5c50098f4e5997c07afed3b9383e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7d39d74185e549e483dd3bc9a5fe8c76",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_c66b78a449c6451fbe757d1082304de5",
+            "value": 1
+          }
+        },
+        "15bf9c9d1cf44b5abee88dddc74a927b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_2aa0d8f2c2bc4b3083edfcc5dc7ccc10",
+            "placeholder": "​",
+            "style": "IPY_MODEL_a0e715cd015d499ba015c54e810bf81d",
+            "value": "Batches: 100%"
+          }
+        },
+        "2aa0d8f2c2bc4b3083edfcc5dc7ccc10": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "5072655a4a724daa9d7b660e1709fa11": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "7d39d74185e549e483dd3bc9a5fe8c76": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "a0e715cd015d499ba015c54e810bf81d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "c66b78a449c6451fbe757d1082304de5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "cb41643480ca4834bcc03611c4783326": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "cf25f66f1ca943be8af0c7ce14727f21": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_cb41643480ca4834bcc03611c4783326",
+            "placeholder": "​",
+            "style": "IPY_MODEL_5072655a4a724daa9d7b660e1709fa11",
+            "value": " 1/1 [00:00&lt;00:00,  3.12it/s]"
+          }
+        },
+        "d3f93597cb6640a29656daa95153f07f": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

docker-compose.yml

@@ -0,0 +1,40 @@
+services:
+  mongo:
+    image: mongo:latest
+    container_name: "llm_engineering_mongo"
+    logging:
+      options:
+        max-size: 1g
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: "llm_engineering"
+      MONGO_INITDB_ROOT_PASSWORD: "llm_engineering"
+    ports:
+      - 27017:27017
+    volumes:
+      - mongo_data:/data/db
+    networks:
+      - local
+    restart: always
+
+  qdrant:
+    image: qdrant/qdrant:latest
+    container_name: "llm_engineering_qdrant"
+    ports:
+      - 6333:6333
+      - 6334:6334
+    expose:
+      - 6333
+      - 6334
+    volumes:
+      - qdrant_data:/qdrant/storage
+    networks:
+      - local
+    restart: always
+
+volumes:
+  mongo_data:
+  qdrant_data:
+
+networks:
+  local:
+    driver: bridge

en_core_web_sm-3.7.0-py3-none-any.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6215d71a3212690e9aec49408a27e3fe6ad7cd6c715476e93d70dc784041e93e
+size 12803377

llm_engineering/__init__.py

@@ -0,0 +1,4 @@
+from llm_engineering import application, domain, infrastructure
+from llm_engineering.settings import settings
+
+__all__ = ["settings", "application", "domain", "infrastructure"]

llm_engineering/application/__init__.py

@@ -0,0 +1,3 @@
+from . import utils
+
+__all__ = ["utils"]

llm_engineering/application/crawlers/__init__.py

@@ -0,0 +1,6 @@
+from .dispatcher import CrawlerDispatcher
+from .github import GithubCrawler
+from .linkedin import LinkedInCrawler
+from .medium import MediumCrawler
+
+__all__ = ["CrawlerDispatcher", "GithubCrawler", "LinkedInCrawler", "MediumCrawler"]

llm_engineering/application/crawlers/base.py

@@ -0,0 +1,66 @@
+import time
+from abc import ABC, abstractmethod
+from tempfile import mkdtemp
+
+import chromedriver_autoinstaller
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+from llm_engineering.domain.documents import NoSQLBaseDocument
+
+# Check if the current version of chromedriver exists
+# and if it doesn't exist, download it automatically,
+# then add chromedriver to path
+chromedriver_autoinstaller.install()
+
+
+class BaseCrawler(ABC):
+    model: type[NoSQLBaseDocument]
+
+    @abstractmethod
+    def extract(self, link: str, **kwargs) -> None: ...
+
+
+class BaseSeleniumCrawler(BaseCrawler, ABC):
+    def __init__(self, scroll_limit: int = 5) -> None:
+        options = webdriver.ChromeOptions()
+
+        options.add_argument("--no-sandbox")
+        options.add_argument("--headless=new")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--log-level=3")
+        options.add_argument("--disable-popup-blocking")
+        options.add_argument("--disable-notifications")
+        options.add_argument("--disable-extensions")
+        options.add_argument("--disable-background-networking")
+        options.add_argument("--ignore-certificate-errors")
+        options.add_argument(f"--user-data-dir={mkdtemp()}")
+        options.add_argument(f"--data-path={mkdtemp()}")
+        options.add_argument(f"--disk-cache-dir={mkdtemp()}")
+        options.add_argument("--remote-debugging-port=9226")
+
+        self.set_extra_driver_options(options)
+
+        self.scroll_limit = scroll_limit
+        self.driver = webdriver.Chrome(
+            options=options,
+        )
+
+    def set_extra_driver_options(self, options: Options) -> None:
+        pass
+
+    def login(self) -> None:
+        pass
+
+    def scroll_page(self) -> None:
+        """Scroll through the LinkedIn page based on the scroll limit."""
+        current_scroll = 0
+        last_height = self.driver.execute_script("return document.body.scrollHeight")
+        while True:
+            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(5)
+            new_height = self.driver.execute_script("return document.body.scrollHeight")
+            if new_height == last_height or (self.scroll_limit and current_scroll >= self.scroll_limit):
+                break
+            last_height = new_height
+            current_scroll += 1

llm_engineering/application/crawlers/custom_article.py

@@ -0,0 +1,54 @@
+from urllib.parse import urlparse
+
+from langchain_community.document_loaders import AsyncHtmlLoader
+from langchain_community.document_transformers.html2text import Html2TextTransformer
+from loguru import logger
+
+from llm_engineering.domain.documents import ArticleDocument
+
+from .base import BaseCrawler
+
+
+class CustomArticleCrawler(BaseCrawler):
+    model = ArticleDocument
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def extract(self, link: str, **kwargs) -> None:
+        old_model = self.model.find(link=link)
+        if old_model is not None:
+            logger.info(f"Article already exists in the database: {link}")
+
+            return
+
+        logger.info(f"Starting scrapping article: {link}")
+
+        loader = AsyncHtmlLoader([link])
+        docs = loader.load()
+
+        html2text = Html2TextTransformer()
+        docs_transformed = html2text.transform_documents(docs)
+        doc_transformed = docs_transformed[0]
+
+        content = {
+            "Title": doc_transformed.metadata.get("title"),
+            "Subtitle": doc_transformed.metadata.get("description"),
+            "Content": doc_transformed.page_content,
+            "language": doc_transformed.metadata.get("language"),
+        }
+
+        parsed_url = urlparse(link)
+        platform = parsed_url.netloc
+
+        user = kwargs["user"]
+        instance = self.model(
+            content=content,
+            link=link,
+            platform=platform,
+            author_id=user.id,
+            author_full_name=user.full_name,
+        )
+        instance.save()
+
+        logger.info(f"Finished scrapping custom article: {link}")

llm_engineering/application/crawlers/dispatcher.py

@@ -0,0 +1,51 @@
+import re
+from urllib.parse import urlparse
+
+from loguru import logger
+
+from .base import BaseCrawler
+from .custom_article import CustomArticleCrawler
+from .github import GithubCrawler
+from .linkedin import LinkedInCrawler
+from .medium import MediumCrawler
+
+
+class CrawlerDispatcher:
+    def __init__(self) -> None:
+        self._crawlers = {}
+
+    @classmethod
+    def build(cls) -> "CrawlerDispatcher":
+        dispatcher = cls()
+
+        return dispatcher
+
+    def register_medium(self) -> "CrawlerDispatcher":
+        self.register("https://medium.com", MediumCrawler)
+
+        return self
+
+    def register_linkedin(self) -> "CrawlerDispatcher":
+        self.register("https://linkedin.com", LinkedInCrawler)
+
+        return self
+
+    def register_github(self) -> "CrawlerDispatcher":
+        self.register("https://github.com", GithubCrawler)
+
+        return self
+
+    def register(self, domain: str, crawler: type[BaseCrawler]) -> None:
+        parsed_domain = urlparse(domain)
+        domain = parsed_domain.netloc
+
+        self._crawlers[r"https://(www\.)?{}/*".format(re.escape(domain))] = crawler
+
+    def get_crawler(self, url: str) -> BaseCrawler:
+        for pattern, crawler in self._crawlers.items():
+            if re.match(pattern, url):
+                return crawler()
+        else:
+            logger.warning(f"No crawler found for {url}. Defaulting to CustomArticleCrawler.")
+
+            return CustomArticleCrawler()

llm_engineering/application/crawlers/github.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+import subprocess
+import tempfile
+
+from loguru import logger
+
+from llm_engineering.domain.documents import RepositoryDocument
+
+from .base import BaseCrawler
+
+
+class GithubCrawler(BaseCrawler):
+    model = RepositoryDocument
+
+    def __init__(self, ignore=(".git", ".toml", ".lock", ".png")) -> None:
+        super().__init__()
+        self._ignore = ignore
+
+    def extract(self, link: str, **kwargs) -> None:
+        old_model = self.model.find(link=link)
+        if old_model is not None:
+            logger.info(f"Repository already exists in the database: {link}")
+
+            return
+
+        logger.info(f"Starting scrapping GitHub repository: {link}")
+
+        repo_name = link.rstrip("/").split("/")[-1]
+
+        local_temp = tempfile.mkdtemp()
+
+        try:
+            os.chdir(local_temp)
+            subprocess.run(["git", "clone", link])
+
+            repo_path = os.path.join(local_temp, os.listdir(local_temp)[0])  # noqa: PTH118
+
+            tree = {}
+            for root, _, files in os.walk(repo_path):
+                dir = root.replace(repo_path, "").lstrip("/")
+                if dir.startswith(self._ignore):
+                    continue
+
+                for file in files:
+                    if file.endswith(self._ignore):
+                        continue
+                    file_path = os.path.join(dir, file)  # noqa: PTH118
+                    with open(os.path.join(root, file), "r", errors="ignore") as f:  # noqa: PTH123, PTH118
+                        tree[file_path] = f.read().replace(" ", "")
+
+            user = kwargs["user"]
+            instance = self.model(
+                content=tree,
+                name=repo_name,
+                link=link,
+                platform="github",
+                author_id=user.id,
+                author_full_name=user.full_name,
+            )
+            instance.save()
+
+        except Exception:
+            raise
+        finally:
+            shutil.rmtree(local_temp)
+
+        logger.info(f"Finished scrapping GitHub repository: {link}")