CharacterAI improvements

App/Generate/Story/Story.py

@@ -1,6 +1,6 @@
 import json
 from pydantic import BaseModel
-from typing import List
+from typing import List, Optional
 
 
 class Scene(BaseModel):
@@ -13,6 +13,7 @@ class Scene(BaseModel):
 
 class Story(BaseModel):
     scenes: List[Scene]
+    voice: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data):

App/Generate/database/CharacterAPI.py

@@ -0,0 +1,123 @@
+import aiohttp
+import asyncio
+import os
+import uuid
+import tempfile
+from typing import List, Dict, Any
+
+from pydantic import BaseModel
+
+
+class AlignmentData(BaseModel):
+    word: str
+    start: float
+    end: float
+
+    def to_dict(self) -> dict:
+        return {
+            "word": self.word,
+            "alignedWord": self.word,
+            "startTime": self.start,
+            "endTime": self.end,
+            "hasFailedAlignment": False,
+        }
+
+
+class CharacterAITTS:
+    def __init__(self):
+        self.api_url = "https://yakova-embedding.hf.space"
+        self.dir = str(tempfile.mkdtemp())
+        self.descript = "https://yakova-embedding.hf.space"
+        self.headers = {"Connection": "keep-alive", "Content-Type": "application/json"}
+
+    async def _make_transcript(self, links, text):
+
+        data = {"audio_url": links, "text": text, "file_extenstion": ".mp3"}
+        response_data = await self._make_request(
+            "post", "descript_transcript", json=data, external=self.descript
+        )
+        if not response_data:
+            data["audio_url"] = data["audio_url"][0]
+            print(data)
+            response_data = await self.aligner(
+                "post",
+                "align/url",
+                json=data,
+            )
+            print(response_data)
+            response_data = self.process_alignments(
+                data=response_data["alignment"], offset=0
+            )
+        return response_data
+
+    def process_alignments(
+        self, data: List[Dict[str, Any]], offset: float = 0
+    ) -> List[Dict[str, Any]]:
+        alignments = [AlignmentData(**item) for item in data]
+        return [alignment.to_dict() for alignment in alignments]
+
+    async def aligner(
+        self,
+        method,
+        endpoint,
+        json=None,
+        external="https://yakova-aligner.hf.space/align/url",
+    ):
+        async with aiohttp.ClientSession() as session:
+            if external:
+                url = f"{external}"
+            else:
+                url = f"{self.api_url}/{endpoint}"
+            async with getattr(session, method)(url=url, json=json) as response:
+                return await response.json()
+
+    async def _make_request(self, method, endpoint, json=None, external=None):
+        async with aiohttp.ClientSession() as session:
+            if external:
+                url = f"{external}/{endpoint}"
+            else:
+                url = f"{self.api_url}/{endpoint}"
+            async with getattr(session, method)(url=url, json=json) as response:
+                return await response.json()
+
+    async def say(self, text, speaker=None):
+
+        data = {"text": text, "voice": speaker}
+
+        response_data = await self._make_request("post", "cai_tts", json=data)
+        # print(response_data)
+        audio_url = response_data["audio"]
+        temp = await self.download_file(audio_url)
+        return audio_url, temp
+
+    async def download_file(self, url):
+        filename = str(uuid.uuid4()) + ".mp3"
+        os.makedirs(self.dir, exist_ok=True)
+        save_path = os.path.join(self.dir, filename)
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    with open(save_path, "wb") as file:
+                        while True:
+                            chunk = await response.content.read(1024)
+                            if not chunk:
+                                break
+                            file.write(chunk)
+
+        return save_path
+
+
+# # Usage
+# async def main():
+#     tts = CharacterAITTS()
+#     url, temp = await tts.say(
+#         "Did you know that you don't have the balls to talk to me"
+#     )
+#     tranny = await tts._make_transcript(
+#         links=[url], text="Did you know that you don't have the balls to talk to me"
+#     )
+#     print(tranny)
+
+
+# # Run the main function
+# asyncio.run(main())

App/Generate/database/Model.py

@@ -4,7 +4,9 @@ import asyncio, os
 import uuid, random
 from pydub import AudioSegment
 from .DescriptAPI import Speak
-from .ElevenLab import ElevenLab
+
+# from .ElevenLab import ElevenLab
+from .CharacterAPI import CharacterAITTS
 from .Vercel import AsyncImageGenerator
 from .Video3d import VideoGenerator
 import aiohttp
@@ -196,12 +198,13 @@ class Project(orm.Model):
 
 
 class Scene(orm.Model):
-    tts = ElevenLab()
+    tts = CharacterAITTS()
     # eleven = ElevenLab()
     tablename = "scenes"
     registry = models
     fields = {
         "id": orm.Integer(primary_key=True),
+        "voice": orm.String(max_length=100, allow_null=True, default=""),
         "project": orm.ForeignKey(Project),
         "images": orm.JSON(default=None),
         "narration": orm.String(max_length=10_000, allow_null=True, default=""),
@@ -237,7 +240,7 @@ class Scene(orm.Model):
         while retry_count < 3:
             try:
                 return await self.tts.say(
-                    text=self.narration + " master"
+                    text=self.narration + " master", speaker=self.voice
                 )  ### The blanks help to even stuff up.
             except Exception as e:
                 print(f"Failed to generate narration: {e}")

App/Generate/database/Vercel.py

@@ -44,15 +44,15 @@ class AsyncImageGenerator:
                         "cfg": 2.5,
                         # "seed": 42,
                         "image": "https://image.lexica.art/full_webp/e41b87fb-4cc3-43cd-a6e6-f3dbb08c2399",
-                        "steps": 28,
-                        "width": 1024,
-                        "height": 1536,
-                        "aspect_ratio": "16:9",
+                        # "steps": 28,
+                        # "width": 1024,
+                        # "height": 1536,
+                        # "aspect_ratio": "16:9",
                         "prompt": payload,
                         # "sampler": "dpmpp_2m_sde_gpu",
                         # "scheduler": "karras",
                         # "output_format": "png",
-                        "output_quality": 95,
+                        # "output_quality": 95,
                         # "negative_prompt": "clouds, people, buildings",
                         # "number_of_images": 1,
                         # "ip_adapter_weight": 1,
@@ -62,8 +62,8 @@ class AsyncImageGenerator:
                     # "path": "models/playgroundai/playground-v2.5-1024px-aesthetic/versions/a45f82a1382bed5c7aeb861dac7c7d191b0fdf74d8d57c4a0e6ed7d4d0bf7d24",
                     # "path": "models/fofr/any-comfyui-workflow/versions/cd385285ba75685a040afbded7b79814a971f3febf46c5eab7c716e200c784e1",
                     # "path": "models/fofr/sd3-explorer/versions/a9f4aebd943ad7db13de8e34debea359d5578d08f128e968f9a36c3e9b0148d4",
-                    # "path": "models/stability-ai/stable-diffusion-3",
-                    "path": "models/black-forest-labs/flux-schnell",
+                    "path": "models/bingbangboom-lab/flux-new-whimscape/versions/2e8de10f217bc56da163a0204cf09f89995eaf643459014803fae79753183682",
+                    # "path": "models/black-forest-labs/flux-schnell",
                     # "path": "models/lucataco/proteus-v0.4/versions/34a427535a3c45552b94369280b823fcd0e5c9710e97af020bf445c033d4569e",
                     # "path": "models/lucataco/juggernaut-xl-v9/versions/bea09cf018e513cef0841719559ea86d2299e05448633ac8fe270b5d5cd6777e",
                 }
@@ -140,18 +140,18 @@ async def process_images(payloads):
         return results
 
 
-# # # Example payloads
-# payloads = [
-#     """
-# comicbook illustration artistic, beautiful Awsome cat
-#      """
-# ]
+# # Example payloads
+payloads = [
+    """
+comicbook illustration artistic, beautiful Awsome cat
+     """
+]
 
 
-# # # Run the asyncio event loop
-# async def main():
-#     results = await process_images(payloads)
-#     pprint.pprint(results)
+# # Run the asyncio event loop
+async def main():
+    results = await process_images(payloads)
+    pprint.pprint(results)
 
 
-# asyncio.run(main())
+asyncio.run(main())

App/Generate/generatorRoutes.py

@@ -49,6 +49,7 @@ async def generate_assets(generated_story: Story, batch_size=4, threeD=True):
                 model_scene = await Scene.objects.create(project=x)
                 model_scene.image_prompts = story_scene.image_prompts
                 model_scene.narration = story_scene.narration
+                model_scene.voice = story_scene.voice
                 await model_scene.update(**model_scene.__dict__)
                 all_scenes.append(model_scene)
                 batch_updates.append(