original code by JeffLiang and Radames

Dockerfile

@@ -0,0 +1,48 @@
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONUNBUFFERED=1
+ENV NODE_MAJOR=20
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    build-essential \
+    python3.9 \
+    python3-pip \
+    python3-dev \
+    git \
+    ffmpeg \
+    google-perftools \
+    ca-certificates curl gnupg \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /code
+
+RUN mkdir -p /etc/apt/keyrings 
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list > /dev/null
+RUN apt-get update && apt-get install nodejs -y
+
+COPY ./requirements.txt /code/requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    SYSTEM=spaces
+
+RUN pip3 install --no-cache-dir --upgrade --pre -r /code/requirements.txt
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
+CMD ["./build-run.sh"]

LICENSE

@@ -0,0 +1,53 @@
+Stream V2V
+8417 MAR
+UT AUSTIN RESEARCH LICENSE
+(NONCONFIDENTIAL SOURCE CODE)
+
+The University of Texas at Austin has developed certain software and documentation that it desires to make available without charge to anyone for academic, research, experimental or personal use. If you wish to distribute or make other use of the software, you may purchase a license to do so from The University of Texas at Austin ([email protected]). 
+The accompanying source code is made available to you under the terms of this UT Research License (this “UTRL”). By installing or using the code, you are consenting to be bound by this UTRL. If you do not agree to the terms and conditions of this license, do not install or use any part of the code. 
+The terms and conditions in this UTRL not only apply to the source code made available by Licensor, but also to any improvements to, or derivative works of, that source code made by you and to any object code compiled from such source code, improvements or derivative works.
+
+1.  DEFINITIONS. 
+1.1  “Commercial Use” shall mean use of Software or Documentation by Licensee for direct or indirect financial, commercial or strategic gain or advantage, including without limitation: (a) bundling or integrating the Software with any hardware product or another software product for transfer, sale or license to a third party (even if distributing the Software on separate media and not charging for the Software); (b) providing customers with a link to the Software or a copy of the Software for use with hardware or another software product purchased by that customer; or (c) use in connection with the performance of services for which Licensee is compensated. 
+1.2  “Derivative Products” means any improvements to, or other derivative works of, the Software made by Licensee, and any computer software programs, and accompanying documentation, developed by Licensee which are a modification of, enhancement to, derived from or based upon the Licensed Software or documentation provided by Licensor for the Licensed Software, and any object code compiled from such computer software programs.
+1.3  “Documentation” shall mean all manuals, user documentation, and other related materials pertaining to the Software that are made available to Licensee in connection with the Software.
+1.4  “Licensor” shall mean The University of Texas at Austin, on behalf of the Board of Regents of the University of Texas System, an agency of the State of Texas, whose address is 3925 W. Braker Lane, Suite 1.9A (R3500), Austin, Texas 78759.
+1.5  “Licensee” or “you” shall mean the person or entity that has agreed to the terms hereof and is exercising rights granted hereunder.
+1.6  “Software” shall mean the computer program(s) referred to as: “Stream V2V” (UT Tech ID 8417 MAR), which is made available under this UTRL in source code form, including any error corrections, bug fixes, patches, updates or other modifications that Licensor may in its sole discretion make available to Licensee from time to time, and any object code compiled from such source code.
+
+2.  GRANT OF RIGHTS. 
+Subject to the terms and conditions hereunder, Licensor hereby grants to Licensee a worldwide, non-transferable, non-exclusive license to (a) install, use and reproduce the Software for academic, research, experimental and personal use (but specifically excluding Commercial Use); (b) use and modify the Software to create Derivative Products, subject to Section 3.2; (c) use the Documentation, if any, solely in connection with Licensee’s authorized use of the Software; and (d) a non-exclusive, royalty-free license for academic, research, experimental and personal use (but specifically excluding Commercial Use) to those patents, of which Diana Marculescu or Jeff Liang is a named inventor, that are licensable by Licensee and that are necessarily infringed by such authorized use of the Software, and solely in connection with Licensee’s authorized use of the Software.
+
+3. RESTRICTIONS; COVENANTS. 
+3.1 Licensee may not: (a) distribute, sub-license or otherwise transfer copies or rights to the Software (or any portion thereof) or the Documentation; (b) use the Software (or any portion thereof) or Documentation for Commercial Use, or for any other use except as described in Section 2; (c) copy the Software or Documentation other than for archival and backup purposes; or (d) remove any product identification, copyright, proprietary notices or labels from the Software and Documentation. This UTRL confers no rights upon Licensee except those expressly granted herein.
+3.2 Derivative Products. Licensee hereby agrees that it will provide a copy of all Derivative Products to Licensor and that its use of the Derivative Products will be subject to all of the same terms, conditions, restrictions and limitations on use imposed on the Software under this UTRL. Licensee hereby grants Licensor a worldwide, non-exclusive, royalty-free license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute Derivative Products. Licensee also hereby grants Licensor a worldwide, non-exclusive, royalty-free patent license to make, have made, use, offer to sell, sell, import and otherwise transfer the Derivative Products under those patent claims, from patents of which Diana Marculescu or Jeff Liang is a named inventor, that licensable by Licensee that are necessarily infringed by the Derivative Products.
+
+4. CONFIDENTIALITY; PROTECTION OF SOFTWARE. 
+4.1 Reserved.
+4.2 Proprietary Notices. Licensee shall maintain and place on any copy of Software or Documentation that it reproduces for internal use all notices as are authorized and/or required hereunder. Licensee shall include a copy of this UTRL and the following notice, on each copy of the Software and Documentation. Such license and notice shall be embedded in each copy of the Software, in the video screen display, on the physical medium embodying the Software copy and on any Documentation:
+Copyright © 2021, The University of Texas at Austin. All rights reserved.
+UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING THIS SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR ANY PARTICULAR PURPOSE, NON-INFRINGEMENT AND WARRANTIES OF PERFORMANCE, AND ANY WARRANTY THAT MIGHT OTHERWISE ARISE FROM COURSE OF DEALING OR USAGE OF TRADE. NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH RESPECT TO THE USE OF THE SOFTWARE OR DOCUMENTATION. Under no circumstances shall University be liable for incidental, special, indirect, direct or consequential damages or loss of profits, interruption of business, or related expenses which may arise from use of Software or Documentation, including but not limited to those resulting from defects in Software and/or Documentation, or loss or inaccuracy of data of any kind.
+
+5. WARRANTIES.
+5.1 Disclaimer of Warranties. TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE AND DOCUMENTATION ARE BEING PROVIDED ON AN “AS IS” BASIS WITHOUT ANY WARRANTIES OF ANY KIND RESPECTING THE SOFTWARE OR DOCUMENTATION, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY OF DESIGN, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
+5.2 Limitation of Liability. UNDER NO CIRCUMSTANCES UNLESS REQUIRED BY APPLICABLE LAW SHALL LICENSOR BE LIABLE FOR INCIDENTAL, SPECIAL, INDIRECT, DIRECT OR CONSEQUENTIAL DAMAGES OR LOSS OF PROFITS, INTERRUPTION OF BUSINESS, OR RELATED EXPENSES WHICH MAY ARISE AS A RESULT OF THIS LICENSE OR OUT OF THE USE OR ATTEMPT OF USE OF SOFTWARE OR DOCUMENTATION INCLUDING BUT NOT LIMITED TO THOSE RESULTING FROM DEFECTS IN SOFTWARE AND/OR DOCUMENTATION, OR LOSS OR INACCURACY OF DATA OF ANY KIND. THE FOREGOING EXCLUSIONS AND LIMITATIONS WILL APPLY TO ALL CLAIMS AND ACTIONS OF ANY KIND, WHETHER BASED ON CONTRACT, TORT (INCLUDING, WITHOUT LIMITATION, NEGLIGENCE), OR ANY OTHER GROUNDS.
+
+6. INDEMNIFICATION. 
+Licensee shall indemnify, defend and hold harmless Licensor, the University of Texas System, their Regents, and their officers, agents and employees from and against any claims, demands, or causes of action whatsoever caused by, or arising out of, or resulting from, the exercise or practice of the license granted hereunder by Licensee, its officers, employees, agents or representatives.
+
+7.  TERMINATION. 
+If Licensee breaches this UTRL, Licensee’s right to use the Software and Documentation will terminate immediately without notice, but all provisions of this UTRL except Section 2 will survive termination and continue in effect. Upon termination, Licensee must destroy all copies of the Software and Documentation.
+8.  GOVERNING LAW; JURISDICTION AND VENUE. 
+
+The validity, interpretation, construction and performance of this UTRL shall be governed by the laws of the State of Texas. The Texas state courts of Travis County, Texas (or, if there is exclusive federal jurisdiction, the United States District Court for the Western District of Texas) shall have exclusive jurisdiction and venue over any dispute arising out of this UTRL, and Licensee consents to the jurisdiction of such courts. Application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. 
+
+9.  EXPORT CONTROLS. 
+This license is subject to all applicable export restrictions. Licensee must comply with all export and import laws and restrictions and regulations of any United States or foreign agency or authority relating to the Software and its use. 
+
+10.  U.S. GOVERNMENT END-USERS. 
+The Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101, consisting of “commercial computer software” and “commercial computer software documentation,” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995) and 48 C.F.R. 227.7202 (June 1995). Consistent with 48 C.F.R. 12.212, 48 C.F.R. 27.405(b)(2) (June 1998) and 48 C.F.R. 227.7202, all U.S. Government End Users acquire the Software with only those rights as set forth herein. 
+
+11. MISCELLANEOUS 
+If any provision hereof shall be held illegal, invalid or unenforceable, in whole or in part, such provision shall be modified to the minimum extent necessary to make it legal, valid and enforceable, and the legality, validity and enforceability of all other provisions of this UTRL shall not be affected thereby. Licensee may not assign this UTRL in whole or in part, without Licensor’s prior written consent. Any attempt to assign this UTRL without such consent will be null and void. This UTRL is the complete and exclusive statement between Licensee and Licensor relating to the subject matter hereof and supersedes all prior oral and written and all contemporaneous oral negotiations, commitments and understandings of the parties, if any. Any waiver by either party of any default or breach hereunder shall not constitute a waiver of any provision of this UTRL or of any subsequent default or breach of the same or a different kind.
+
+END OF LICENSE

README.md

@@ -0,0 +1,17 @@
+---
+title: Stream V2V Demo
+emoji: 🖼️🖼️
+colorFrom: gray
+colorTo: indigo
+sdk: docker
+pinned: false
+suggested_hardware: a10g-small
+disable_embedding: true
+---
+
+Note: I am not the original author
+
+This is a deployment of this demo by Jeff Liang: https://github.com/Jeff-LiangF/streamv2v/tree/main/demo_w_camera 
+
+based on code from Radamés Ajna (https://github.com/radames/Real-Time-Latent-Consistency-Model/tree/main)
+

config.py

@@ -0,0 +1,109 @@
+from typing import NamedTuple
+import argparse
+import os
+
+
+class Args(NamedTuple):
+    host: str
+    port: int
+    reload: bool
+    mode: str
+    max_queue_size: int
+    timeout: float
+    safety_checker: bool
+    taesd: bool
+    ssl_certfile: str
+    ssl_keyfile: str
+    debug: bool
+    acceleration: str
+    engine_dir: str
+
+    def pretty_print(self):
+        print("\n")
+        for field, value in self._asdict().items():
+            print(f"{field}: {value}")
+        print("\n")
+
+
+MAX_QUEUE_SIZE = int(os.environ.get("MAX_QUEUE_SIZE", 0))
+TIMEOUT = float(os.environ.get("TIMEOUT", 0))
+SAFETY_CHECKER = os.environ.get("SAFETY_CHECKER", None) == "True"
+USE_TAESD = os.environ.get("USE_TAESD", "True") == "True"
+ENGINE_DIR = os.environ.get("ENGINE_DIR", "engines")
+ACCELERATION = os.environ.get("ACCELERATION", "xformers")
+
+default_host = os.getenv("HOST", "0.0.0.0")
+default_port = int(os.getenv("PORT", "7860"))
+default_mode = os.getenv("MODE", "default")
+
+parser = argparse.ArgumentParser(description="Run the app")
+parser.add_argument("--host", type=str, default=default_host, help="Host address")
+parser.add_argument("--port", type=int, default=default_port, help="Port number")
+parser.add_argument("--reload", action="store_true", help="Reload code on change")
+parser.add_argument(
+    "--mode", type=str, default=default_mode, help="App Inferece Mode: txt2img, img2img"
+)
+parser.add_argument(
+    "--max-queue-size",
+    dest="max_queue_size",
+    type=int,
+    default=MAX_QUEUE_SIZE,
+    help="Max Queue Size",
+)
+parser.add_argument("--timeout", type=float, default=TIMEOUT, help="Timeout")
+parser.add_argument(
+    "--safety-checker",
+    dest="safety_checker",
+    action="store_true",
+    default=SAFETY_CHECKER,
+    help="Safety Checker",
+)
+parser.add_argument(
+    "--taesd",
+    dest="taesd",
+    action="store_true",
+    help="Use Tiny Autoencoder",
+)
+parser.add_argument(
+    "--no-taesd",
+    dest="taesd",
+    action="store_false",
+    help="Use Tiny Autoencoder",
+)
+parser.add_argument(
+    "--ssl-certfile",
+    dest="ssl_certfile",
+    type=str,
+    default=None,
+    help="SSL certfile",
+)
+parser.add_argument(
+    "--ssl-keyfile",
+    dest="ssl_keyfile",
+    type=str,
+    default=None,
+    help="SSL keyfile",
+)
+parser.add_argument(
+    "--debug",
+    action="store_true",
+    default=False,
+    help="Debug",
+)
+parser.add_argument(
+    "--acceleration",
+    type=str,
+    default=ACCELERATION,
+    choices=["none", "xformers", "sfast", "tensorrt"],
+    help="Acceleration",
+)
+parser.add_argument(
+    "--engine-dir",
+    dest="engine_dir",
+    type=str,
+    default=ENGINE_DIR,
+    help="Engine Dir",
+)
+parser.set_defaults(taesd=USE_TAESD)
+config = Args(**vars(parser.parse_args()))
+config.pretty_print()

connection_manager.py

@@ -0,0 +1,116 @@
+from typing import Dict, Union
+from uuid import UUID
+import asyncio
+from fastapi import WebSocket
+from starlette.websockets import WebSocketState
+import logging
+from types import SimpleNamespace
+
+Connections = Dict[UUID, Dict[str, Union[WebSocket, asyncio.Queue]]]
+
+
+class ServerFullException(Exception):
+    """Exception raised when the server is full."""
+
+    pass
+
+
+class ConnectionManager:
+    def __init__(self):
+        self.active_connections: Connections = {}
+
+    async def connect(
+        self, user_id: UUID, websocket: WebSocket, max_queue_size: int = 0
+    ):
+        await websocket.accept()
+        user_count = self.get_user_count()
+        print(f"User count: {user_count}")
+        if max_queue_size > 0 and user_count >= max_queue_size:
+            print("Server is full")
+            await websocket.send_json({"status": "error", "message": "Server is full"})
+            await websocket.close()
+            raise ServerFullException("Server is full")
+        print(f"New user connected: {user_id}")
+        self.active_connections[user_id] = {
+            "websocket": websocket,
+            "queue": asyncio.Queue(),
+        }
+        await websocket.send_json(
+            {"status": "connected", "message": "Connected"},
+        )
+        await websocket.send_json({"status": "wait"})
+        await websocket.send_json({"status": "send_frame"})
+
+    def check_user(self, user_id: UUID) -> bool:
+        return user_id in self.active_connections
+
+    async def update_data(self, user_id: UUID, new_data: SimpleNamespace):
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            queue = user_session["queue"]
+            while not queue.empty():
+                try:
+                    queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    continue
+            await queue.put(new_data)
+
+    async def get_latest_data(self, user_id: UUID) -> SimpleNamespace:
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            queue = user_session["queue"]
+            try:
+                return await queue.get()
+            except asyncio.QueueEmpty:
+                return None
+
+    def delete_user(self, user_id: UUID):
+        user_session = self.active_connections.pop(user_id, None)
+        if user_session:
+            queue = user_session["queue"]
+            while not queue.empty():
+                try:
+                    queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    continue
+
+    def get_user_count(self) -> int:
+        return len(self.active_connections)
+
+    def get_websocket(self, user_id: UUID) -> WebSocket:
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            websocket = user_session["websocket"]
+            if websocket.client_state == WebSocketState.CONNECTED:
+                return user_session["websocket"]
+        return None
+
+    async def disconnect(self, user_id: UUID):
+        websocket = self.get_websocket(user_id)
+        if websocket:
+            await websocket.close()
+        self.delete_user(user_id)
+
+    async def send_json(self, user_id: UUID, data: Dict):
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                await websocket.send_json(data)
+        except Exception as e:
+            logging.error(f"Error: Send json: {e}")
+
+    async def receive_json(self, user_id: UUID) -> Dict:
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                return await websocket.receive_json()
+        except Exception as e:
+            logging.error(f"Error: Receive json: {e}")
+
+    async def receive_bytes(self, user_id: UUID) -> bytes:
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                return await websocket.receive_bytes()
+        except Exception as e:
+            logging.error(f"Error: Receive bytes: {e}")

frontend/.eslintignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

frontend/.eslintrc.cjs

@@ -0,0 +1,30 @@
+module.exports = {
+  root: true,
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:svelte/recommended',
+    'prettier'
+  ],
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 2020,
+    extraFileExtensions: ['.svelte']
+  },
+  env: {
+    browser: true,
+    es2017: true,
+    node: true
+  },
+  overrides: [
+    {
+      files: ['*.svelte'],
+      parser: 'svelte-eslint-parser',
+      parserOptions: {
+        parser: '@typescript-eslint/parser'
+      }
+    }
+  ]
+};

frontend/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*

frontend/.npmrc

@@ -0,0 +1 @@
+engine-strict=true

frontend/.prettierignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

frontend/.prettierrc

@@ -0,0 +1,19 @@
+{
+  "useTabs": false,
+  "singleQuote": true,
+  "trailingComma": "none",
+  "printWidth": 100,
+  "plugins": [
+    "prettier-plugin-svelte",
+    "prettier-plugin-organize-imports",
+    "prettier-plugin-tailwindcss"
+  ],
+  "overrides": [
+    {
+      "files": "*.svelte",
+      "options": {
+        "parser": "svelte"
+      }
+    }
+  ]
+}

frontend/README.md

@@ -0,0 +1,38 @@
+# create-svelte
+
+Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+
+## Creating a project
+
+If you're seeing this, you've probably already done this step. Congrats!
+
+```bash
+# create a new project in the current directory
+npm create svelte@latest
+
+# create a new project in my-app
+npm create svelte@latest my-app
+```
+
+## Developing
+
+Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+
+```bash
+npm run dev
+
+# or start the server and open the app in a new browser tab
+npm run dev -- --open
+```
+
+## Building
+
+To create a production version of your app:
+
+```bash
+npm run build
+```
+
+You can preview the production build with `npm run preview`.
+
+> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

frontend/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "frontend",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "dev": "vite dev",
+    "build": "vite build",
+    "preview": "vite preview",
+    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+    "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
+    "lint": "prettier --check . && eslint .",
+    "format": "prettier --write ."
+  },
+  "devDependencies": {
+    "@sveltejs/adapter-auto": "^2.0.0",
+    "@sveltejs/adapter-static": "^2.0.3",
+    "@sveltejs/kit": "^1.20.4",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "autoprefixer": "^10.4.16",
+    "eslint": "^8.28.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-svelte": "^2.30.0",
+    "postcss": "^8.4.31",
+    "prettier": "^3.1.0",
+    "prettier-plugin-organize-imports": "^3.2.4",
+    "prettier-plugin-svelte": "^3.1.0",
+    "prettier-plugin-tailwindcss": "^0.5.7",
+    "svelte": "^4.0.5",
+    "svelte-check": "^3.4.3",
+    "tailwindcss": "^3.3.5",
+    "tslib": "^2.4.1",
+    "typescript": "^5.0.0",
+    "vite": "^4.4.2"
+  },
+  "type": "module",
+  "dependencies": {
+    "piexifjs": "^1.0.6",
+    "rvfc-polyfill": "^1.0.7"
+  }
+}

frontend/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {}
+  }
+};

frontend/src/app.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/src/app.d.ts

@@ -0,0 +1,12 @@
+// See https://kit.svelte.dev/docs/types#app
+// for information about these interfaces
+declare global {
+  namespace App {
+    // interface Error {}
+    // interface Locals {}
+    // interface PageData {}
+    // interface Platform {}
+  }
+}
+
+export {};

frontend/src/app.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%sveltekit.assets%/favicon.png" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    %sveltekit.head%
+  </head>
+  <body data-sveltekit-preload-data="hover">
+    <div style="display: contents">%sveltekit.body%</div>
+  </body>
+</html>

frontend/src/lib/components/Button.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+  export let classList: string = 'p-2';
+  export let disabled: boolean = false;
+  export let title: string = '';
+</script>
+
+<button class="button {classList}" on:click {disabled} {title}>
+  <slot />
+</button>
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

frontend/src/lib/components/Checkbox.svelte

@@ -0,0 +1,14 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = false;
+  export let params: FieldProps;
+  onMount(() => {
+    value = Boolean(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input bind:checked={value} type="checkbox" id={params.id} class="cursor-pointer" />
+</div>

frontend/src/lib/components/ImagePlayer.svelte

@@ -0,0 +1,50 @@
+<script lang="ts">
+  import { lcmLiveStatus, LCMLiveStatus, streamId } from '$lib/lcmLive';
+  import { getPipelineValues } from '$lib/store';
+
+  import Button from '$lib/components/Button.svelte';
+  import Floppy from '$lib/icons/floppy.svelte';
+  import { snapImage } from '$lib/utils';
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: console.log('isLCMRunning', isLCMRunning);
+  let imageEl: HTMLImageElement;
+  async function takeSnapshot() {
+    if (isLCMRunning) {
+      await snapImage(imageEl, {
+        prompt: getPipelineValues()?.prompt,
+        negative_prompt: getPipelineValues()?.negative_prompt,
+        seed: getPipelineValues()?.seed,
+        guidance_scale: getPipelineValues()?.guidance_scale
+      });
+    }
+  }
+</script>
+
+<div
+  class="relative mx-auto aspect-square max-w-lg self-center overflow-hidden rounded-lg border border-slate-300"
+>
+  <!-- svelte-ignore a11y-missing-attribute -->
+  {#if isLCMRunning && $streamId}
+    <img
+      bind:this={imageEl}
+      class="aspect-square w-full rounded-lg"
+      src={'/api/stream/' + $streamId}
+    />
+    <div class="absolute bottom-1 right-1">
+      <Button
+        on:click={takeSnapshot}
+        disabled={!isLCMRunning}
+        title={'Take Snapshot'}
+        classList={'text-sm ml-auto text-white p-1 shadow-lg rounded-lg opacity-50'}
+      >
+        <Floppy classList={''} />
+      </Button>
+    </div>
+  {:else}
+    <img
+      class="aspect-square w-full rounded-lg"
+      src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+    />
+  {/if}
+</div>

frontend/src/lib/components/InputRange.svelte

@@ -0,0 +1,50 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = 8.0;
+  export let params: FieldProps;
+  onMount(() => {
+    value = Number(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input
+    class="col-span-2 h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-300 dark:bg-gray-500"
+    bind:value
+    type="range"
+    id={params.id}
+    name={params.id}
+    min={params?.min}
+    max={params?.max}
+    step={params?.step ?? 1}
+  />
+  <input
+    type="number"
+    step={params?.step ?? 1}
+    bind:value
+    class="rounded-md border px-1 py-1 text-center text-xs font-bold dark:text-black"
+  />
+</div>
+<!-- 
+<style lang="postcss" scoped>
+  input[type='range']::-webkit-slider-runnable-track {
+    @apply h-2 cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-webkit-slider-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+</style> -->

frontend/src/lib/components/MediaListSwitcher.svelte

@@ -0,0 +1,40 @@
+<script lang="ts">
+  import { mediaDevices, mediaStreamActions } from '$lib/mediaStream';
+  import Screen from '$lib/icons/screen.svelte';
+  import { onMount } from 'svelte';
+
+  let deviceId: string = '';
+  $: {
+    console.log($mediaDevices);
+  }
+  $: {
+    console.log(deviceId);
+  }
+  onMount(() => {
+    deviceId = $mediaDevices[0].deviceId;
+  });
+</script>
+
+<div class="flex items-center justify-center text-xs">
+  <button
+    title="Share your screen"
+    class="border-1 my-1 flex cursor-pointer gap-1 rounded-md border-gray-500 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    on:click={() => mediaStreamActions.startScreenCapture()}
+  >
+    <span>Share</span>
+
+    <Screen classList={''} />
+  </button>
+  {#if $mediaDevices}
+    <select
+      bind:value={deviceId}
+      on:change={() => mediaStreamActions.switchCamera(deviceId)}
+      id="devices-list"
+      class="border-1 block cursor-pointer rounded-md border-gray-800 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    >
+      {#each $mediaDevices as device, i}
+        <option value={device.deviceId}>{device.label}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

frontend/src/lib/components/PipelineOptions.svelte

@@ -0,0 +1,61 @@
+<script lang="ts">
+  import type { Fields } from '$lib/types';
+  import { FieldType } from '$lib/types';
+  import InputRange from './InputRange.svelte';
+  import SeedInput from './SeedInput.svelte';
+  import TextArea from './TextArea.svelte';
+  import Checkbox from './Checkbox.svelte';
+  import Selectlist from './Selectlist.svelte';
+  import { pipelineValues } from '$lib/store';
+
+  export let pipelineParams: Fields;
+
+  $: advanceOptions = Object.values(pipelineParams)?.filter(
+    (e) => e?.hide == true && e?.disabled !== true
+  );
+  $: featuredOptions = Object.values(pipelineParams)?.filter((e) => e?.hide !== true);
+</script>
+
+<div class="flex flex-col gap-3">
+  <div class="grid grid-cols-1 items-center gap-3">
+    {#if featuredOptions}
+      {#each featuredOptions as params}
+        {#if params.field === FieldType.RANGE}
+          <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
+        {:else if params.field === FieldType.SEED}
+          <SeedInput {params} bind:value={$pipelineValues[params.id]}></SeedInput>
+        {:else if params.field === FieldType.TEXTAREA}
+          <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
+        {:else if params.field === FieldType.CHECKBOX}
+          <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
+        {:else if params.field === FieldType.SELECT}
+          <Selectlist {params} bind:value={$pipelineValues[params.id]}></Selectlist>
+        {/if}
+      {/each}
+    {/if}
+  </div>
+  {#if advanceOptions && advanceOptions.length > 0}
+    <details>
+      <summary class="cursor-pointer font-medium">Advanced Options</summary>
+      <div
+        class="grid grid-cols-1 items-center gap-3 {Object.values(pipelineParams).length > 5
+          ? 'sm:grid-cols-2'
+          : ''}"
+      >
+        {#each advanceOptions as params}
+          {#if params.field === FieldType.RANGE}
+            <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
+          {:else if params.field === FieldType.SEED}
+            <SeedInput {params} bind:value={$pipelineValues[params.id]}></SeedInput>
+          {:else if params.field === FieldType.TEXTAREA}
+            <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
+          {:else if params.field === FieldType.CHECKBOX}
+            <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
+          {:else if params.field === FieldType.SELECT}
+            <Selectlist {params} bind:value={$pipelineValues[params.id]}></Selectlist>
+          {/if}
+        {/each}
+      </div>
+    </details>
+  {/if}
+</div>

frontend/src/lib/components/SeedInput.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  import Button from './Button.svelte';
+  export let value = 299792458;
+  export let params: FieldProps;
+
+  onMount(() => {
+    value = Number(params?.default ?? '');
+  });
+  function randomize() {
+    value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+  }
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for="seed">Seed</label>
+  <input
+    bind:value
+    type="number"
+    id="seed"
+    name="seed"
+    class="col-span-2 rounded-md border border-gray-700 p-2 text-right font-light dark:text-black"
+  />
+  <Button on:click={randomize}>Rand</Button>
+</div>

frontend/src/lib/components/Selectlist.svelte

@@ -0,0 +1,24 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = '';
+  export let params: FieldProps;
+  onMount(() => {
+    value = String(params?.default);
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label for="model-list" class="text-sm font-medium">{params?.title} </label>
+  {#if params?.values}
+    <select
+      bind:value
+      id="model-list"
+      class="cursor-pointer rounded-md border-2 border-gray-500 p-2 font-light dark:text-black"
+    >
+      {#each params.values as model, i}
+        <option value={model} selected={i === 0}>{model}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

frontend/src/lib/components/TextArea.svelte

@@ -0,0 +1,23 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value: string;
+  export let params: FieldProps;
+  onMount(() => {
+    value = String(params?.default ?? '');
+  });
+</script>
+
+<div class="">
+  <label class="text-sm font-medium" for={params?.title}>
+    {params?.title}
+  </label>
+  <div class="text-normal flex items-center rounded-md border border-gray-700">
+    <textarea
+      class="mx-1 w-full px-3 py-2 font-light outline-none dark:text-black"
+      title={params?.title}
+      placeholder="Add your prompt here..."
+      bind:value
+    ></textarea>
+  </div>
+</div>

frontend/src/lib/components/VideoInput.svelte

@@ -0,0 +1,109 @@
+<script lang="ts">
+  import 'rvfc-polyfill';
+
+  import { onDestroy, onMount } from 'svelte';
+  import {
+    mediaStreamStatus,
+    MediaStreamStatusEnum,
+    onFrameChangeStore,
+    mediaStream,
+    mediaDevices
+  } from '$lib/mediaStream';
+  import MediaListSwitcher from './MediaListSwitcher.svelte';
+  export let width = 512;
+  export let height = 512;
+  const size = { width, height };
+
+  let videoEl: HTMLVideoElement;
+  let canvasEl: HTMLCanvasElement;
+  let ctx: CanvasRenderingContext2D;
+  let videoFrameCallbackId: number;
+
+  // ajust the throttle time to your needs
+  const THROTTLE = 1000 / 120;
+  let selectedDevice: string = '';
+  let videoIsReady = false;
+
+  onMount(() => {
+    ctx = canvasEl.getContext('2d') as CanvasRenderingContext2D;
+    canvasEl.width = size.width;
+    canvasEl.height = size.height;
+  });
+  $: {
+    console.log(selectedDevice);
+  }
+  onDestroy(() => {
+    if (videoFrameCallbackId) videoEl.cancelVideoFrameCallback(videoFrameCallbackId);
+  });
+
+  $: if (videoEl) {
+    videoEl.srcObject = $mediaStream;
+  }
+  let lastMillis = 0;
+  async function onFrameChange(now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) {
+    if (now - lastMillis < THROTTLE) {
+      videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+      return;
+    }
+    const videoWidth = videoEl.videoWidth;
+    const videoHeight = videoEl.videoHeight;
+    let height0 = videoHeight;
+    let width0 = videoWidth;
+    let x0 = 0;
+    let y0 = 0;
+    if (videoWidth > videoHeight) {
+      width0 = videoHeight;
+      x0 = (videoWidth - videoHeight) / 2;
+    } else {
+      height0 = videoWidth;
+      y0 = (videoHeight - videoWidth) / 2;
+    }
+    ctx.drawImage(videoEl, x0, y0, width0, height0, 0, 0, size.width, size.height);
+    const blob = await new Promise<Blob>((resolve) => {
+      canvasEl.toBlob(
+        (blob) => {
+          resolve(blob as Blob);
+        },
+        'image/jpeg',
+        1
+      );
+    });
+    onFrameChangeStore.set({ blob });
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+
+  $: if ($mediaStreamStatus == MediaStreamStatusEnum.CONNECTED && videoIsReady) {
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+</script>
+
+<div class="relative mx-auto max-w-lg overflow-hidden rounded-lg border border-slate-300">
+  <div class="relative z-10 aspect-square w-full object-cover">
+    {#if $mediaDevices.length > 0}
+      <div class="absolute bottom-0 right-0 z-10">
+        <MediaListSwitcher />
+      </div>
+    {/if}
+    <video
+      class="pointer-events-none aspect-square w-full object-cover"
+      bind:this={videoEl}
+      on:loadeddata={() => {
+        videoIsReady = true;
+      }}
+      playsinline
+      autoplay
+      muted
+      loop
+    ></video>
+    <canvas bind:this={canvasEl} class="absolute left-0 top-0 aspect-square w-full object-cover"
+    ></canvas>
+  </div>
+  <div class="absolute left-0 top-0 flex aspect-square w-full items-center justify-center">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" class="w-40 p-5 opacity-20">
+      <path
+        fill="currentColor"
+        d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z"
+      />
+    </svg>
+  </div>
+</div>

frontend/src/lib/components/Warning.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+  export let message: string = '';
+
+  let timeout = 0;
+  $: if (message !== '') {
+    console.log('message', message);
+    clearTimeout(timeout);
+    timeout = setTimeout(() => {
+      message = '';
+    }, 5000);
+  }
+</script>
+
+{#if message}
+  <div class="fixed right-0 top-0 m-4 cursor-pointer" on:click={() => (message = '')}>
+    <div class="rounded bg-red-800 p-4 text-white">
+      {message}
+    </div>
+    <div class="bar transition-all duration-500" style="width: 0;"></div>
+  </div>
+{/if}
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

frontend/src/lib/icons/floppy.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 448 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M48 96v320a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V170.5a16 16 0 0 0-4.7-11.3l33.9-33.9a64 64 0 0 1 18.7 45.3V416a64 64 0 0 1-64 64H64a64 64 0 0 1-64-64V96a64 64 0 0 1 64-64h245.5a64 64 0 0 1 45.3 18.7l74.5 74.5-33.9 33.9-74.6-74.4-.8-.8V184a24 24 0 0 1-24 24H104a24 24 0 0 1-24-24V80H64a16 16 0 0 0-16 16zm80-16v80h144V80H128zm32 240a64 64 0 1 1 128 0 64 64 0 1 1-128 0z"
+  />
+</svg>

frontend/src/lib/icons/screen.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string = '';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -32 576 576" height="16px" class={classList}>
+  <path
+    fill="currentColor"
+    d="M64 0A64 64 0 0 0 0 64v288a64 64 0 0 0 64 64h176l-10.7 32H160a32 32 0 1 0 0 64h256a32 32 0 1 0 0-64h-69.3L336 416h176a64 64 0 0 0 64-64V64a64 64 0 0 0-64-64H64zm448 64v288H64V64h448z"
+  />
+</svg>

frontend/src/lib/icons/spinner.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 512 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M304 48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zm0 416a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM48 304a48 48 0 1 0 0-96 48 48 0 1 0 0 96zm464-48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM142.9 437A48 48 0 1 0 75 369.1 48 48 0 1 0 142.9 437zm0-294.2A48 48 0 1 0 75 75a48 48 0 1 0 67.9 67.9zM369.1 437A48 48 0 1 0 437 369.1 48 48 0 1 0 369.1 437z"
+  />
+</svg>

frontend/src/lib/index.ts

@@ -0,0 +1 @@
+// place files you want to import through the `$lib` alias in this folder.

frontend/src/lib/lcmLive.ts

@@ -0,0 +1,99 @@
+import { writable } from 'svelte/store';
+
+
+export enum LCMLiveStatus {
+    CONNECTED = "connected",
+    DISCONNECTED = "disconnected",
+    WAIT = "wait",
+    SEND_FRAME = "send_frame",
+    TIMEOUT = "timeout",
+}
+
+const initStatus: LCMLiveStatus = LCMLiveStatus.DISCONNECTED;
+
+export const lcmLiveStatus = writable<LCMLiveStatus>(initStatus);
+export const streamId = writable<string | null>(null);
+
+let websocket: WebSocket | null = null;
+export const lcmLiveActions = {
+    async start(getSreamdata: () => any[]) {
+        return new Promise((resolve, reject) => {
+
+            try {
+                const userId = crypto.randomUUID();
+                const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws"
+                    }:${window.location.host}/api/ws/${userId}`;
+
+                websocket = new WebSocket(websocketURL);
+                websocket.onopen = () => {
+                    console.log("Connected to websocket");
+                };
+                websocket.onclose = () => {
+                    lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                    console.log("Disconnected from websocket");
+                };
+                websocket.onerror = (err) => {
+                    console.error(err);
+                };
+                websocket.onmessage = (event) => {
+                    const data = JSON.parse(event.data);
+                    switch (data.status) {
+                        case "connected":
+                            lcmLiveStatus.set(LCMLiveStatus.CONNECTED);
+                            streamId.set(userId);
+                            resolve({ status: "connected", userId });
+                            break;
+                        case "send_frame":
+                            lcmLiveStatus.set(LCMLiveStatus.SEND_FRAME);
+                            const streamData = getSreamdata();
+                            websocket?.send(JSON.stringify({ status: "next_frame" }));
+                            for (const d of streamData) {
+                                this.send(d);
+                            }
+                            break;
+                        case "wait":
+                            lcmLiveStatus.set(LCMLiveStatus.WAIT);
+                            break;
+                        case "timeout":
+                            console.log("timeout");
+                            lcmLiveStatus.set(LCMLiveStatus.TIMEOUT);
+                            streamId.set(null);
+                            reject(new Error("timeout"));
+                            break;
+                        case "error":
+                            console.log(data.message);
+                            lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                            streamId.set(null);
+                            reject(new Error(data.message));
+                            break;
+                    }
+                };
+
+            } catch (err) {
+                console.error(err);
+                lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                streamId.set(null);
+                reject(err);
+            }
+        });
+    },
+    send(data: Blob | { [key: string]: any }) {
+        if (websocket && websocket.readyState === WebSocket.OPEN) {
+            if (data instanceof Blob) {
+                websocket.send(data);
+            } else {
+                websocket.send(JSON.stringify(data));
+            }
+        } else {
+            console.log("WebSocket not connected");
+        }
+    },
+    async stop() {
+        lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+        if (websocket) {
+            websocket.close();
+        }
+        websocket = null;
+        streamId.set(null);
+    },
+};

frontend/src/lib/mediaStream.ts

@@ -0,0 +1,98 @@
+import { writable, type Writable, get } from 'svelte/store';
+
+export enum MediaStreamStatusEnum {
+    INIT = "init",
+    CONNECTED = "connected",
+    DISCONNECTED = "disconnected",
+}
+export const onFrameChangeStore: Writable<{ blob: Blob }> = writable({ blob: new Blob() });
+
+export const mediaDevices = writable<MediaDeviceInfo[]>([]);
+export const mediaStreamStatus = writable(MediaStreamStatusEnum.INIT);
+export const mediaStream = writable<MediaStream | null>(null);
+
+export const mediaStreamActions = {
+    async enumerateDevices() {
+        // console.log("Enumerating devices");
+        await navigator.mediaDevices.enumerateDevices()
+            .then(devices => {
+                const cameras = devices.filter(device => device.kind === 'videoinput');
+                mediaDevices.set(cameras);
+            })
+            .catch(err => {
+                console.error(err);
+            });
+    },
+    async start(mediaDevicedID?: string) {
+        const constraints = {
+            audio: false,
+            video: {
+                width: 1024, height: 1024, deviceId: mediaDevicedID
+            }
+        };
+
+        await navigator.mediaDevices
+            .getUserMedia(constraints)
+            .then((stream) => {
+                mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+                mediaStream.set(stream);
+            })
+            .catch((err) => {
+                console.error(`${err.name}: ${err.message}`);
+                mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+                mediaStream.set(null);
+            });
+    },
+    async startScreenCapture() {
+        const displayMediaOptions = {
+            video: {
+                displaySurface: "window",
+            },
+            audio: false,
+            surfaceSwitching: "include"
+        };
+
+
+        let captureStream = null;
+
+        try {
+            captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+            const videoTrack = captureStream.getVideoTracks()[0];
+
+            console.log("Track settings:");
+            console.log(JSON.stringify(videoTrack.getSettings(), null, 2));
+            console.log("Track constraints:");
+            console.log(JSON.stringify(videoTrack.getConstraints(), null, 2));
+            mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+            mediaStream.set(captureStream)
+        } catch (err) {
+            console.error(err);
+        }
+
+    },
+    async switchCamera(mediaDevicedID: string) {
+        if (get(mediaStreamStatus) !== MediaStreamStatusEnum.CONNECTED) {
+            return;
+        }
+        const constraints = {
+            audio: false,
+            video: { width: 1024, height: 1024, deviceId: mediaDevicedID }
+        };
+        await navigator.mediaDevices
+            .getUserMedia(constraints)
+            .then((stream) => {
+                mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+                mediaStream.set(stream)
+            })
+            .catch((err) => {
+                console.error(`${err.name}: ${err.message}`);
+            });
+    },
+    async stop() {
+        navigator.mediaDevices.getUserMedia({ video: true }).then((stream) => {
+            stream.getTracks().forEach((track) => track.stop());
+        });
+        mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+        mediaStream.set(null);
+    },
+};

frontend/src/lib/store.ts

@@ -0,0 +1,15 @@
+
+import { derived, writable, get, type Writable, type Readable } from 'svelte/store';
+
+export const pipelineValues: Writable<Record<string, any>> = writable({});
+export const deboucedPipelineValues: Readable<Record<string, any>>
+    = derived(pipelineValues, ($pipelineValues, set) => {
+        const debounced = setTimeout(() => {
+            set($pipelineValues);
+        }, 100);
+        return () => clearTimeout(debounced);
+    });
+
+
+
+export const getPipelineValues = () => get(pipelineValues);

frontend/src/lib/types.ts

@@ -0,0 +1,40 @@
+export const enum FieldType {
+    RANGE = "range",
+    SEED = "seed",
+    TEXTAREA = "textarea",
+    CHECKBOX = "checkbox",
+    SELECT = "select",
+}
+export const enum PipelineMode {
+    IMAGE = "image",
+    VIDEO = "video",
+    TEXT = "text",
+}
+
+
+export interface Fields {
+    [key: string]: FieldProps;
+}
+
+export interface FieldProps {
+    default: number | string;
+    max?: number;
+    min?: number;
+    title: string;
+    field: FieldType;
+    step?: number;
+    disabled?: boolean;
+    hide?: boolean;
+    id: string;
+    values?: string[];
+}
+export interface PipelineInfo {
+    title: {
+        default: string;
+    }
+    name: string;
+    description: string;
+    input_mode: {
+        default: PipelineMode;
+    }
+}

frontend/src/lib/utils.ts

@@ -0,0 +1,38 @@
+import * as piexif from "piexifjs";
+
+interface IImageInfo {
+    prompt?: string;
+    negative_prompt?: string;
+    seed?: number;
+    guidance_scale?: number;
+}
+
+export function snapImage(imageEl: HTMLImageElement, info: IImageInfo) {
+    try {
+        const zeroth: { [key: string]: any } = {};
+        const exif: { [key: string]: any } = {};
+        const gps: { [key: string]: any } = {};
+        zeroth[piexif.ImageIFD.Make] = "LCM Image-to-Image ControNet";
+        zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${info?.prompt} | negative_prompt: ${info?.negative_prompt} | seed: ${info?.seed} | guidance_scale: ${info?.guidance_scale}`;
+        zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model";
+        exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString();
+
+        const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps };
+        const exifBytes = piexif.dump(exifObj);
+
+        const canvas = document.createElement("canvas");
+        canvas.width = imageEl.naturalWidth;
+        canvas.height = imageEl.naturalHeight;
+        const ctx = canvas.getContext("2d") as CanvasRenderingContext2D;
+        ctx.drawImage(imageEl, 0, 0);
+        const dataURL = canvas.toDataURL("image/jpeg");
+        const withExif = piexif.insert(exifBytes, dataURL);
+
+        const a = document.createElement("a");
+        a.href = withExif;
+        a.download = `lcm_txt_2_img${Date.now()}.png`;
+        a.click();
+    } catch (err) {
+        console.log(err);
+    }
+}

frontend/src/routes/+layout.svelte

@@ -0,0 +1,5 @@
+<script>
+  import '../app.css';
+</script>
+
+<slot />

frontend/src/routes/+page.svelte

@@ -0,0 +1,152 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import type { Fields, PipelineInfo } from '$lib/types';
+  import { PipelineMode } from '$lib/types';
+  import ImagePlayer from '$lib/components/ImagePlayer.svelte';
+  import VideoInput from '$lib/components/VideoInput.svelte';
+  import Button from '$lib/components/Button.svelte';
+  import PipelineOptions from '$lib/components/PipelineOptions.svelte';
+  import Spinner from '$lib/icons/spinner.svelte';
+  import Warning from '$lib/components/Warning.svelte';
+  import { lcmLiveStatus, lcmLiveActions, LCMLiveStatus } from '$lib/lcmLive';
+  import { mediaStreamActions, onFrameChangeStore } from '$lib/mediaStream';
+  import { getPipelineValues, deboucedPipelineValues } from '$lib/store';
+
+  let pipelineParams: Fields;
+  let pipelineInfo: PipelineInfo;
+  let pageContent: string;
+  let isImageMode: boolean = false;
+  let maxQueueSize: number = 0;
+  let currentQueueSize: number = 0;
+  let queueCheckerRunning: boolean = false;
+  let warningMessage: string = '';
+  onMount(() => {
+    getSettings();
+  });
+
+  async function getSettings() {
+    const settings = await fetch('/api/settings').then((r) => r.json());
+    pipelineParams = settings.input_params.properties;
+    pipelineInfo = settings.info.properties;
+    isImageMode = pipelineInfo.input_mode.default === PipelineMode.IMAGE;
+    maxQueueSize = settings.max_queue_size;
+    pageContent = settings.page_content;
+    console.log(pipelineParams);
+    toggleQueueChecker(true);
+  }
+  function toggleQueueChecker(start: boolean) {
+    queueCheckerRunning = start && maxQueueSize > 0;
+    if (start) {
+      getQueueSize();
+    }
+  }
+  async function getQueueSize() {
+    if (!queueCheckerRunning) {
+      return;
+    }
+    const data = await fetch('/api/queue').then((r) => r.json());
+    currentQueueSize = data.queue_size;
+    setTimeout(getQueueSize, 10000);
+  }
+
+  function getSreamdata() {
+    if (isImageMode) {
+      return [getPipelineValues(), $onFrameChangeStore?.blob];
+    } else {
+      return [$deboucedPipelineValues];
+    }
+  }
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: if ($lcmLiveStatus === LCMLiveStatus.TIMEOUT) {
+    warningMessage = 'Session timed out. Please try again.';
+  }
+  let disabled = false;
+  async function toggleLcmLive() {
+    try {
+      if (!isLCMRunning) {
+        if (isImageMode) {
+          await mediaStreamActions.enumerateDevices();
+          await mediaStreamActions.start();
+        }
+        disabled = true;
+        await lcmLiveActions.start(getSreamdata);
+        disabled = false;
+        toggleQueueChecker(false);
+      } else {
+        if (isImageMode) {
+          mediaStreamActions.stop();
+        }
+        lcmLiveActions.stop();
+        toggleQueueChecker(true);
+      }
+    } catch (e) {
+      warningMessage = e instanceof Error ? e.message : '';
+      disabled = false;
+      toggleQueueChecker(true);
+    }
+  }
+</script>
+
+<svelte:head>
+  <script
+    src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.9/iframeResizer.contentWindow.min.js"
+  ></script>
+</svelte:head>
+
+<main class="container mx-auto flex max-w-5xl flex-col gap-3 px-4 py-4">
+  <Warning bind:message={warningMessage}></Warning>
+  <article class="text-center">
+    {#if pageContent}
+      {@html pageContent}
+    {/if}
+    {#if maxQueueSize > 0}
+      <p class="text-sm">
+        There are <span id="queue_size" class="font-bold">{currentQueueSize}</span>
+        user(s) sharing the same GPU, affecting real-time performance. Maximum queue size is {maxQueueSize}.
+        <a
+          href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
+          target="_blank"
+          class="text-blue-500 underline hover:no-underline">Duplicate</a
+        > and run it on your own GPU.
+      </p>
+    {/if}
+  </article>
+  {#if pipelineParams}
+    <article class="my-3 grid grid-cols-1 gap-3 sm:grid-cols-2">
+      {#if isImageMode}
+        <div class="sm:col-start-1">
+          <VideoInput
+            width={Number(pipelineParams.width.default)}
+            height={Number(pipelineParams.height.default)}
+          ></VideoInput>
+        </div>
+      {/if}
+      <div class={isImageMode ? 'sm:col-start-2' : 'col-span-2'}>
+        <ImagePlayer />
+      </div>
+      <div class="sm:col-span-2">
+        <Button on:click={toggleLcmLive} {disabled} classList={'text-lg my-1 p-2'}>
+          {#if isLCMRunning}
+            Stop
+          {:else}
+            Start
+          {/if}
+        </Button>
+        <PipelineOptions {pipelineParams}></PipelineOptions>
+      </div>
+    </article>
+  {:else}
+    <!-- loading -->
+    <div class="flex items-center justify-center gap-3 py-48 text-2xl">
+      <Spinner classList={'animate-spin opacity-50'}></Spinner>
+      <p>Loading...</p>
+    </div>
+  {/if}
+</main>
+
+<style lang="postcss">
+  :global(html) {
+    @apply text-black dark:bg-gray-900 dark:text-white;
+  }
+</style>

frontend/src/routes/+page.ts

@@ -0,0 +1 @@
+export const prerender = true

frontend/svelte.config.js

@@ -0,0 +1,17 @@
+import adapter from '@sveltejs/adapter-static';
+import { vitePreprocess } from '@sveltejs/kit/vite';
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+  preprocess: vitePreprocess({ postcss: true }),
+  kit: {
+    adapter: adapter({
+      pages: 'public',
+      assets: 'public',
+      fallback: undefined,
+      precompress: false,
+      strict: true
+    })
+  }
+};
+
+export default config;

frontend/tailwind.config.js

@@ -0,0 +1,8 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: ['./src/**/*.{html,js,svelte,ts}', '../**/*.py'],
+  theme: {
+    extend: {}
+  },
+  plugins: []
+};

frontend/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "./.svelte-kit/tsconfig.json",
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "strict": true
+  }
+  // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
+  //
+  // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
+  // from the referenced tsconfig.json - TypeScript does not merge them in
+}

frontend/vite.config.ts

@@ -0,0 +1,15 @@
+import { sveltekit } from '@sveltejs/kit/vite';
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+  plugins: [sveltekit()],
+  server: {
+    proxy: {
+      '/api': 'http://localhost:7860',
+      '/api/ws': {
+        target: 'ws://localhost:7860',
+        ws: true
+      }
+    },
+  }
+});

main.py

@@ -0,0 +1,193 @@
+from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect
+from fastapi.responses import StreamingResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi import Request
+
+import markdown2
+
+import logging
+import uuid
+import time
+from types import SimpleNamespace
+import asyncio
+import os
+import time
+import mimetypes
+import torch
+
+from config import config, Args
+from util import pil_to_frame, bytes_to_pil, is_firefox
+from connection_manager import ConnectionManager, ServerFullException
+from vid2vid import Pipeline
+
+# fix mime error on windows
+mimetypes.add_type("application/javascript", ".js")
+
+THROTTLE = 1.0 / 120
+# logging.basicConfig(level=logging.DEBUG)
+
+
+class App:
+    def __init__(self, config: Args, pipeline):
+        self.args = config
+        self.pipeline = pipeline
+        self.app = FastAPI()
+        self.conn_manager = ConnectionManager()
+        self.init_app()
+
+    def init_app(self):
+        self.app.add_middleware(
+            CORSMiddleware,
+            allow_origins=["*"],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+        @self.app.websocket("/api/ws/{user_id}")
+        async def websocket_endpoint(user_id: uuid.UUID, websocket: WebSocket):
+            try:
+                await self.conn_manager.connect(
+                    user_id, websocket, self.args.max_queue_size
+                )
+                await handle_websocket_data(user_id)
+            except ServerFullException as e:
+                logging.error(f"Server Full: {e}")
+            finally:
+                await self.conn_manager.disconnect(user_id)
+                logging.info(f"User disconnected: {user_id}")
+
+        async def handle_websocket_data(user_id: uuid.UUID):
+            if not self.conn_manager.check_user(user_id):
+                return HTTPException(status_code=404, detail="User not found")
+            last_time = time.time()
+            try:
+                while True:
+                    if (
+                        self.args.timeout > 0
+                        and time.time() - last_time > self.args.timeout
+                    ):
+                        await self.conn_manager.send_json(
+                            user_id,
+                            {
+                                "status": "timeout",
+                                "message": "Your session has ended",
+                            },
+                        )
+                        await self.conn_manager.disconnect(user_id)
+                        return
+                    data = await self.conn_manager.receive_json(user_id)
+                    if data["status"] != "next_frame":
+                        asyncio.sleep(THROTTLE)
+                        continue
+
+                    params = await self.conn_manager.receive_json(user_id)
+                    params = pipeline.InputParams(**params)
+                    info = pipeline.Info()
+                    params = SimpleNamespace(**params.dict())
+                    if info.input_mode == "image":
+                        image_data = await self.conn_manager.receive_bytes(user_id)
+                        if len(image_data) == 0:
+                            await self.conn_manager.send_json(
+                                user_id, {"status": "send_frame"}
+                            )
+                            await asyncio.sleep(THROTTLE)
+                            continue
+                        params.image = bytes_to_pil(image_data)
+                    await self.conn_manager.update_data(user_id, params)
+                    await self.conn_manager.send_json(user_id, {"status": "wait"})
+
+            except Exception as e:
+                logging.error(f"Websocket Error: {e}, {user_id} ")
+                await self.conn_manager.disconnect(user_id)
+
+        @self.app.get("/api/queue")
+        async def get_queue_size():
+            queue_size = self.conn_manager.get_user_count()
+            return JSONResponse({"queue_size": queue_size})
+
+        @self.app.get("/api/stream/{user_id}")
+        async def stream(user_id: uuid.UUID, request: Request):
+            try:
+
+                async def generate():
+                    last_params = SimpleNamespace()
+                    while True:
+                        last_time = time.time()
+                        params = await self.conn_manager.get_latest_data(user_id)
+                        if not vars(params) or params.__dict__ == last_params.__dict__:
+                            await self.conn_manager.send_json(
+                                user_id, {"status": "send_frame"}
+                            )
+                            continue
+
+                        last_params = params
+                        image = pipeline.predict(params)
+                        if image is None:
+                            await self.conn_manager.send_json(
+                                user_id, {"status": "send_frame"}
+                            )
+                            continue
+                        frame = pil_to_frame(image)
+                        yield frame
+                        # https://bugs.chromium.org/p/chromium/issues/detail?id=1250396
+                        if not is_firefox(request.headers["user-agent"]):
+                            yield frame
+                        await self.conn_manager.send_json(
+                            user_id, {"status": "send_frame"}
+                        )
+                        if self.args.debug:
+                            print(f"Time taken: {time.time() - last_time}")
+
+                return StreamingResponse(
+                    generate(),
+                    media_type="multipart/x-mixed-replace;boundary=frame",
+                    headers={"Cache-Control": "no-cache"},
+                )
+            except Exception as e:
+                logging.error(f"Streaming Error: {e}, {user_id} ")
+                return HTTPException(status_code=404, detail="User not found")
+
+        # route to setup frontend
+        @self.app.get("/api/settings")
+        async def settings():
+            info_schema = pipeline.Info.schema()
+            info = pipeline.Info()
+            if info.page_content:
+                page_content = markdown2.markdown(info.page_content)
+
+            input_params = pipeline.InputParams.schema()
+            return JSONResponse(
+                {
+                    "info": info_schema,
+                    "input_params": input_params,
+                    "max_queue_size": self.args.max_queue_size,
+                    "page_content": page_content if info.page_content else "",
+                }
+            )
+
+        if not os.path.exists("./frontend/public"):
+            os.makedirs("./frontend/public")
+
+        self.app.mount(
+            "/", StaticFiles(directory="./frontend/public", html=True), name="public"
+        )
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+torch_dtype = torch.float16
+pipeline = Pipeline(config, device, torch_dtype)
+app = App(config, pipeline).app
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "main:app",
+        host=config.host,
+        port=config.port,
+        reload=config.reload,
+        ssl_certfile=config.ssl_certfile,
+        ssl_keyfile=config.ssl_keyfile,
+    )

requirements.txt

@@ -0,0 +1,14 @@
+diffusers==0.27.0
+transformers==4.35.2
+--extra-index-url https://download.pytorch.org/whl/cu121;
+torch
+fastapi==0.104.1
+uvicorn[standard]==0.24.0.post1
+Pillow==10.1.0
+accelerate==0.24.0
+compel==2.0.2
+controlnet-aux==0.0.7
+peft==0.6.0
+xformers; sys_platform != 'darwin' or platform_machine != 'arm64'
+markdown2
+stable_fast @ https://github.com/chengzeyi/stable-fast/releases/download/v0.0.15.post1/stable_fast-0.0.15.post1+torch211cu121-cp310-cp310-manylinux2014_x86_64.whl; sys_platform=='linux'

start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+cd frontend
+npm install
+npm run build
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32m\nfrontend build success \033[0m"
+else
+    echo -e "\033[1;31m\nfrontend build failed\n\033[0m" >&2  exit 1
+fi
+cd ../
+python3 main.py --port 7860 --host 0.0.0.0