adding captionning step

src/index.mts

@@ -182,6 +182,7 @@ app.post("/render", async (req, res) => {
     renderId: "",
     status: "pending",
     assetUrl: "",
+    alt: request.prompt || "",
     maskUrl: "",
     error: "",
     segments: []
@@ -246,6 +247,7 @@ app.get("/render/:renderId", async (req, res) => {
     renderId: "",
     status: "pending",
     assetUrl: "",
+    alt: "",
     error: "",
     maskUrl: "",
     segments: []

src/production/renderImageAnalysis.mts

@@ -0,0 +1,29 @@
+import { analyzeImage } from "../analysis/analyzeImageWithIDEFICSAndNastyHack.mts"
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { upscaleImage } from "../utils/upscaleImage.mts"
+
+export async function renderImageAnalysis(
+  request: RenderRequest,
+  response: RenderedScene,
+): Promise<RenderedScene> {
+  response.alt = request.prompt
+  
+  try {
+    // note: this converts a base64 PNG to a base64 JPG (which is good, actually!)
+    response.assetUrl = await analyzeImage(response.assetUrl, response.assetUrl)
+    console.log(`analysis worked on the first try!`)
+  } catch (err) {
+    console.error(`analysis failed the first time.. let's try again..`)
+    try {
+      response.assetUrl = await upscaleImage(response.assetUrl, request.upscalingFactor)
+      console.log(`analysis worked on the second try!`)
+    } catch (err) {
+      console.error(`analysis failed on the second attempt.. let's keep the prompt as a fallback, then :|`)
+      // no need to log a catastrophic failure here, since we still have the original (low-res image)
+      // to work with
+      response.alt = request.prompt
+    }
+  }
+
+  return response
+}

src/production/renderPipeline.mts

@@ -5,9 +5,9 @@ import { renderImage } from "./renderImage.mts"
 import { renderVideo } from "./renderVideo.mts"
 import { renderImageSegmentation } from "./renderImageSegmentation.mts"
 import { renderVideoSegmentation } from "./renderVideoSegmentation.mts"
-import { upscaleImage } from "../utils/upscaleImage.mts"
 import { renderImageUpscaling } from "./renderImageUpscaling.mts"
 import { saveRenderedSceneToCache } from "../utils/saveRenderedSceneToCache.mts"
+import { renderImageAnalysis } from "./renderImageAnalysis.mts"
 
 export async function renderPipeline(request: RenderRequest, response: RenderedScene) {
   const isVideo = request?.nbFrames > 1
@@ -40,8 +40,13 @@ export async function renderPipeline(request: RenderRequest, response: RenderedS
     ? Promise.resolve()
     : renderImageUpscaling(request, response)
 
+  const optionalAnalysisStep = request.analyze
+    ? renderImageAnalysis(request, response)
+    : Promise.resolve()
+
   await Promise.all([
     renderSegmentation(request, response),
+    optionalAnalysisStep,
     optionalUpscalingStep
   ])
 

src/production/renderScene.mts

@@ -1,10 +1,7 @@
 import { v4 as uuidv4 } from "uuid"
 
 import { RenderedScene, RenderRequest } from "../types.mts"
-import { generateSeed } from "../utils/generateSeed.mts"
-import { getValidNumber } from "../utils/getValidNumber.mts"
 import { renderPipeline } from "./renderPipeline.mts"
-import { getValidBoolean } from "../utils/getValidBoolean.mts"
 
 const cache: Record<string, RenderedScene> = {}
 const cacheQueue: string[] = []
@@ -19,6 +16,7 @@ export async function renderScene(request: RenderRequest): Promise<RenderedScene
     renderId,
     status: "pending",
     assetUrl: "",
+    alt: request.prompt || "",
     error: "",
     maskUrl: "",
     segments: []

src/types.mts

@@ -316,6 +316,8 @@ export interface RenderRequest {
   cache: CacheMode
 
   wait: boolean // wait until the job is completed
+
+  analyze: boolean // analyze the image to generate a caption (optional)
 }
 
 export interface ImageAnalysisRequest {
@@ -358,6 +360,7 @@ export interface RenderedScene {
   renderId: string
   status: RenderedSceneStatus
   assetUrl: string 
+  alt: string
   error: string
   maskUrl: string
   segments: ImageSegment[]

src/utils/parseRenderRequest.mts

@@ -24,6 +24,8 @@ export function parseRenderRequest(request: RenderRequest) {
 
     request.nbSteps = getValidNumber(request.nbSteps, 5, 50, 10)
 
+    request.analyze = request?.analyze ? true : false
+
     if (isVideo) {
       request.width = getValidNumber(request.width, 256, 1024, 1024)
       request.height = getValidNumber(request.height, 256, 1024, 512)