use PuLIB for the turbo mode

src/production/renderImage.mts

@@ -1,8 +1,7 @@
-import { generateImageLCMAsBase64 } from "../providers/image-generation/generateImageLCMGradio.mts"
-import { generateImageSDXLTurboAsBase64 } from "../providers/image-generation/generateImageSDXLTurbo.mts"
 import { generateImageSDXLAsBase64 } from "../providers/image-generation/generateImageSDXLGradio.mts"
 import { generateImageSDXL360AsBase64 } from "../providers/image-generation/generateImageSDXL360.mts"
 import { RenderedScene, RenderRequest } from "../types.mts"
+import { generateImagePulibAsBase64 } from "../providers/image-generation/generateImagePulib.mts"
 
 export async function renderImage(
   request: RenderRequest,
@@ -22,7 +21,7 @@ export async function renderImage(
     // but much, much faster to run
     // for the moment we use SDXL + LCM, as it offers better scene coherence,
     // but we might switch to SDXL Turbo in the future if its quality improves
-    ? generateImageLCMAsBase64 // generateImageSDXLTurboAsBase64 
+    ? generateImagePulibAsBase64 // generateImageSDXLTurboAsBase64 
 
     : generateImageSDXLAsBase64
 
@@ -31,6 +30,7 @@ export async function renderImage(
   const params = {
     positivePrompt: request.prompt,
     negativePrompt: request.negativePrompt,
+    identityImage: request.identityImage,
     seed: request.seed,
     nbSteps: request.nbSteps,
     width: request.width,

src/providers/image-generation/generateImagePulib.mts

@@ -0,0 +1,125 @@
+
+import { client } from "@gradio/client"
+
+import { generateSeed } from "../../utils/misc/generateSeed.mts"
+import { getValidNumber } from "../../utils/validators/getValidNumber.mts"
+import { convertToWebp } from "../../utils/image/convertToWebp.mts"
+import { addBase64HeaderToPng } from "../../utils/image/addBase64HeaderToPng.mts"
+
+// TODO add a system to mark failed instances as "unavailable" for a couple of minutes
+// console.log("process.env:", process.env)
+
+// note: to reduce costs I use the small A10s (not the large)
+// anyway, we will soon not need to use this cloud anymore 
+// since we will be able to leverage the Inference API
+const gradioSpaceApiUrl = `https://jbilcke-hf-ai-tube-model-pulid.hf.space`
+const gradioSpace = `jbilcke-hf/ai-tube-model-pulid`
+const secretToken = `${process.env.VC_MICROSERVICE_SECRET_TOKEN || ""}`
+
+// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
+
+export async function generateImagePulibAsBase64(options: {
+  positivePrompt: string;
+  negativePrompt?: string;
+  identityImage?: string;
+  seed?: number;
+  width?: number;
+  height?: number;
+  nbSteps?: number;
+}): Promise<string> {
+
+  const positivePrompt = options?.positivePrompt || ""
+  if (!positivePrompt) {
+    throw new Error("missing prompt")
+  }
+
+  // the negative prompt CAN be missing, since we use a trick
+  // where we make the interface mandatory in the TS doc,
+  // but browsers might send something partial
+  const negativePrompt = options?.negativePrompt || ""
+  
+  // we treat 0 as meaning "random seed"
+  const seed = (options?.seed ? options.seed : 0) || generateSeed()
+
+  const width = getValidNumber(options?.width, 256, 1024, 512)
+  const height = getValidNumber(options?.height, 256, 1024, 512)
+  const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
+  // console.log("SEED:", seed)
+
+  const identityImage = `${options.identityImage || ""}`
+
+  const positive = [
+    positivePrompt,
+  ].filter(word => word)
+  .join(", ")
+
+  const negative =  [
+    negativePrompt,
+    "watermark",
+    "copyright",
+    "blurry",
+    // "artificial",
+    // "cropped",
+    "low quality",
+    "ugly",
+    'flaws in the eyes',
+    'flaws in the face',
+    'flaws',
+    'lowres',
+    'non-HDRi',
+    'low quality',
+    'worst quality',
+    'artifacts noise',
+    'text',
+    'glitch',
+    'deformed',
+    'mutated',
+    'disfigured hands',
+    'low resolution',
+    'partially rendered objects',
+    'deformed or partially rendered eyes',
+    'ddeformed eyeballs',
+    'cross-eyed',
+  ].filter(word => word)
+  .join(", ")
+
+  const api = await client(gradioSpaceApiUrl, {
+    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+  })
+
+  // we hardcode the number of steps to 4
+  const steps = 4
+
+  // console.log("querying " + gradioSpaceApiUrl + " with tons of params")
+
+  const rawResponse = (await api.predict("/run", [
+    secretToken, // # str in 'parameter_4' Textbox component
+    identityImage || "", // 'ID image (main)' Image component
+    "", // 'Additional ID image (auxiliary)' Image component
+    "", // 'Additional ID image (auxiliary)' Image component
+    "", // 'Additional ID image (auxiliary)' Image component
+    positive, //  # str in 'Prompt' Textbox component
+    negative, //   # str in 'Negative Prompt' Textbox component
+    1.2, //  # int | float (numeric value between 1 and 1.5) in 'CFG, recommend value range [1, 1.5], 1 will be faster ' Slider component
+    generateSeed(), //,  # int | float (numeric value between 0 and 4294967295) in 'Seed' Slider component
+    steps, // # int | float (numeric value between 1 and 100) in 'Steps' Slider component
+    height, // # int | float (numeric value between 512 and 1280) in 'Height' Slider component
+    width, // # int | float (numeric value between 512 and 1280) in 'Width' Slider component
+    0.8, // # int | float (numeric value between 0 and 5) in 'ID scale' Slider component
+    "fidelity", // # str (Option from: ['fidelity', 'extremely style']) in 'mode' Dropdown component
+    false, // 'ID Mix (if you want to mix two ID image, please turn this on, otherwise, turn this off)' Checkbox component
+  ])) as any
+
+  const result = rawResponse?.data?.[0] as string
+  if (!result?.length) {
+    throw new Error(`the returned image was empty`)
+  }
+
+  try {
+    const finalImage = await convertToWebp(addBase64HeaderToPng(result))
+    return finalImage
+  } catch (err) {
+    // console.log("err:", err)
+    throw new Error(err)
+  }
+}

src/providers/video-generation/generateVideoWithAnimateDiffLightning.mts

@@ -3,8 +3,11 @@ import { generateSeed } from "../../utils/misc/generateSeed.mts"
 import { tryApiCalls } from "../../utils/misc/tryApiCall.mts"
 import { getValidNumber } from "../../utils/validators/getValidNumber.mts"
 
-// const gradioApi = `${process.env.AI_TUBE_MODEL_ANIMATELCM_GRADIO_URL || ""}`
-const gradioApi = "https://jbilcke-hf-ai-tube-model-animatediff-lightning.hf.space"
+const replicas = [
+  "https://jbilcke-hf-ai-tube-model-adl-1.hf.space",
+  "https://jbilcke-hf-ai-tube-model-adl-2.hf.space",
+]
+
 const accessToken = `${process.env.VC_MICROSERVICE_SECRET_TOKEN || ""}`
 
 export const generateVideoWithAnimateDiffLightning = async (
@@ -14,6 +17,8 @@ export const generateVideoWithAnimateDiffLightning = async (
   
   const debug = false
 
+  let replica = replicas[0] || ""
+
   const actualFunction = async (): Promise<RenderedScene> => {
 
     const prompt = request.prompt || ""
@@ -61,7 +66,9 @@ export const generateVideoWithAnimateDiffLightning = async (
         })
       }
 
-      const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
+      replicas.push(replica = replicas.shift())
+
+      const res = await fetch(replica + (replica.endsWith("/") ? "" : "/") + "api/predict", {
         method: "POST",
         headers: {
           "Content-Type": "application/json",

src/providers/voice-generation/generateVoiceWithOpenVoice.mts

@@ -0,0 +1,2 @@
+// TODO
+export const todo = "todo"

src/types.mts

@@ -279,6 +279,9 @@ export type RenderRequest = {
   // unused for now
   negativePrompt: string
 
+  // image used for the consistent identity of the main entity (optional)
+  identityImage: string
+
   // whether to use video segmentation
   // disabled (default)
   // firstframe: we only analyze the first frame

src/utils/image/addBase64HeaderToPng.mts

@@ -0,0 +1,14 @@
+export function addBase64HeaderToPng(base64Data: string) {
+  if (typeof base64Data !== "string" || !base64Data) {
+    return ""
+  }
+  if (base64Data.startsWith('data:')) {
+    if (base64Data.startsWith('data:image/png;base64,')) {
+      return base64Data
+    } else {
+      throw new Error("fatal: the input string is NOT a PNG!")
+    }
+  } else {
+    return `data:image/png;base64,${base64Data}`
+  }
+}

src/utils/misc/makeSureSpaceIsRunning.mts

@@ -9,7 +9,7 @@ export async function makeSureSpaceIsRunning({
   // userName,
   // spaceName,
 }: {
-  space: string // a joined "user_name/space_name"
+  space?: string // a joined "user_name/space_name"
 
   maxWaitTimeInSec?: number
 

src/utils/requests/hashRequest.mts

@@ -8,6 +8,7 @@ export function hashRequest(request: RenderRequest) {
     version: 1,
     prompt: request.prompt,
     negativePrompt: request.negativePrompt,
+    identityImage: request.identityImage,
     segmentation: request.segmentation,
     actionnables: request.actionnables,
     nbFrames: request.nbFrames,