Spaces:
Sleeping
Sleeping
File size: 3,413 Bytes
8919651 e864e26 8919651 6419aeb 8e3d721 f24ad59 8e3d721 f24ad59 db70195 8e3d721 f24ad59 db70195 8e3d721 6419aeb 58b1ffb 6419aeb f24ad59 8e3d721 e864e26 8e3d721 e864e26 8e3d721 e864e26 8e3d721 f24ad59 8e3d721 f24ad59 8e3d721 3b780fb 8e3d721 f24ad59 8e3d721 f24ad59 58b1ffb 8e3d721 3b780fb 8e3d721 f24ad59 db70195 8e3d721 3b780fb 8e3d721 f24ad59 8919651 f24ad59 8e3d721 3b780fb 8e3d721 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
import {
ClapProject,
ClapSegment,
getClapAssetSourceType,
newSegment,
filterSegments,
ClapSegmentFilteringMode,
ClapSegmentCategory,
ClapOutputType
} from "@aitube/clap"
import { ClapCompletionMode } from "@aitube/client"
import { getVideoPrompt } from "@aitube/engine"
import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
import { generateStoryboard } from "./generateStoryboard"
export async function processShot({
shotSegment,
existingClap,
newerClap,
mode,
turbo,
}: {
shotSegment: ClapSegment
existingClap: ClapProject
newerClap: ClapProject
mode: ClapCompletionMode
turbo: boolean
}): Promise<void> {
const shotSegments: ClapSegment[] = filterSegments(
ClapSegmentFilteringMode.BOTH,
shotSegment,
existingClap.segments
)
const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
s.category === ClapSegmentCategory.STORYBOARD
)
let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
// TASK 1: GENERATE MISSING STORYBOARD SEGMENT
if (!shotStoryboardSegment) {
shotStoryboardSegment = newSegment({
track: 1,
startTimeInMs: shotSegment.startTimeInMs,
endTimeInMs: shotSegment.endTimeInMs,
assetDurationInMs: shotSegment.assetDurationInMs,
category: ClapSegmentCategory.STORYBOARD,
prompt: "",
assetUrl: "",
outputType: ClapOutputType.IMAGE,
})
// we fix the existing clap
if (shotStoryboardSegment) {
existingClap.segments.push(shotStoryboardSegment)
}
console.log(`[api/v1/edit/storyboards] processShot: generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
}
if (!shotStoryboardSegment) { throw new Error(`failed to generate a newSegment`) }
// TASK 2: GENERATE MISSING STORYBOARD PROMPT
if (!shotStoryboardSegment?.prompt) {
// storyboard is missing, let's generate it
shotStoryboardSegment.prompt = getVideoPrompt(
shotSegments,
existingClap.entityIndex,
["high quality", "crisp", "detailed"]
)
// console.log(`[api/v1/edit/storyboards] processShot: generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
}
// TASK 3: GENERATE MISSING STORYBOARD BITMAP
if (!shotStoryboardSegment.assetUrl) {
// console.log(`[api/v1/edit/storyboards] generating image..`)
try {
shotStoryboardSegment.assetUrl = await generateStoryboard({
prompt: getPositivePrompt(shotStoryboardSegment.prompt),
width: existingClap.meta.width,
height: existingClap.meta.height,
turbo,
})
shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
} catch (err) {
console.log(`[api/v1/edit/storyboards] processShot: failed to generate an image: ${err}`)
throw err
}
console.log(`[api/v1/edit/storyboards] processShot: generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
// if mode is full, newerClap already contains the ference to shotStoryboardSegment
// but if it's partial, we need to manually add it
if (mode !== ClapCompletionMode.FULL) {
newerClap.segments.push(shotStoryboardSegment)
}
} else {
console.log(`[api/v1/edit/storyboards] processShot: there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
}
}
|