minor changes
Browse files- App/Embedding/EmbeddingRoutes.py +7 -2
- App/Embedding/Schemas.py +4 -0
- App/Embedding/utils/Initialize.py +4 -0
- App/Embedding/utils/__init__.py +1 -4
- App/TTS/utils/Picsart.py +33 -0
- App/TTS/utils/Podcastle.py +22 -58
App/Embedding/EmbeddingRoutes.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from fastapi import APIRouter, BackgroundTasks
|
| 2 |
|
| 3 |
-
from .utils.Initialize import TextSearch, IdSearch
|
| 4 |
-
from .Schemas import SearchRequest, AddDocumentRequest
|
| 5 |
import redis, os, json
|
| 6 |
|
| 7 |
REDIS = os.environ.get("REDIS")
|
|
@@ -17,6 +17,11 @@ async def create_embeddings(req: AddDocumentRequest):
|
|
| 17 |
pass
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
@embeddigs_router.post("/search_id")
|
| 21 |
async def search_id(
|
| 22 |
req: SearchRequest,
|
|
|
|
| 1 |
from fastapi import APIRouter, BackgroundTasks
|
| 2 |
|
| 3 |
+
from .utils.Initialize import TextSearch, IdSearch, LookUpIds
|
| 4 |
+
from .Schemas import SearchRequest, AddDocumentRequest, TrendingRequest
|
| 5 |
import redis, os, json
|
| 6 |
|
| 7 |
REDIS = os.environ.get("REDIS")
|
|
|
|
| 17 |
pass
|
| 18 |
|
| 19 |
|
| 20 |
+
@embeddigs_router.get("/Trending")
|
| 21 |
+
async def getTrending(req: TrendingRequest):
|
| 22 |
+
LookUpIds(req.imdb_ids)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
@embeddigs_router.post("/search_id")
|
| 26 |
async def search_id(
|
| 27 |
req: SearchRequest,
|
App/Embedding/Schemas.py
CHANGED
|
@@ -10,5 +10,9 @@ class SearchRequest(BaseModel):
|
|
| 10 |
query: str
|
| 11 |
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
class GetTranscriptions(BaseModel):
|
| 14 |
userId: int
|
|
|
|
| 10 |
query: str
|
| 11 |
|
| 12 |
|
| 13 |
+
class TrendingRequest(BaseModel):
|
| 14 |
+
imdb_ids: list[str]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
class GetTranscriptions(BaseModel):
|
| 18 |
userId: int
|
App/Embedding/utils/Initialize.py
CHANGED
|
@@ -71,6 +71,10 @@ def IdSearch(query: str, background_task: BackgroundTasks):
|
|
| 71 |
return TextSearch(text, filter={"key": {"$ne": query}})
|
| 72 |
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
def TextSearch(query: str, filter=None):
|
| 75 |
docs = docsearch.similarity_search(query, k=10, filter=filter)
|
| 76 |
keys = [doc.metadata["key"] for doc in docs]
|
|
|
|
| 71 |
return TextSearch(text, filter={"key": {"$ne": query}})
|
| 72 |
|
| 73 |
|
| 74 |
+
def LookUpIds(imdb_ids: list[str]):
|
| 75 |
+
return FetchDocuments(imdb_ids)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
def TextSearch(query: str, filter=None):
|
| 79 |
docs = docsearch.similarity_search(query, k=10, filter=filter)
|
| 80 |
keys = [doc.metadata["key"] for doc in docs]
|
App/Embedding/utils/__init__.py
CHANGED
|
@@ -1,10 +1,7 @@
|
|
| 1 |
from elasticsearch import Elasticsearch
|
| 2 |
import os
|
| 3 |
|
| 4 |
-
elastic_host = os.environ.get(
|
| 5 |
-
"ELASTIC_HOST",
|
| 6 |
-
"https://u46hxt12c:[email protected]:443",
|
| 7 |
-
)
|
| 8 |
|
| 9 |
# initialize elasticSearch
|
| 10 |
es = Elasticsearch([elastic_host])
|
|
|
|
| 1 |
from elasticsearch import Elasticsearch
|
| 2 |
import os
|
| 3 |
|
| 4 |
+
elastic_host = os.environ.get("ELASTIC_HOST")
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
# initialize elasticSearch
|
| 7 |
es = Elasticsearch([elastic_host])
|
App/TTS/utils/Picsart.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from playwright.async_api import async_playwright
|
| 3 |
+
|
| 4 |
+
async def extract_bootstrap_data():
|
| 5 |
+
async with async_playwright() as p:
|
| 6 |
+
browser = await p.chromium.launch()
|
| 7 |
+
page = await browser.new_page()
|
| 8 |
+
|
| 9 |
+
# Navigate to the Pixabay Sound Effects page
|
| 10 |
+
await page.goto('https://pixabay.com/sound-effects/search/door%20creaking/')
|
| 11 |
+
|
| 12 |
+
# Wait for the content to load (you can adjust the timeout as needed)
|
| 13 |
+
# await page.wait_for_selector('.js-media-list-wrapper')
|
| 14 |
+
# await page.wait_for_selector('.js-media-item')
|
| 15 |
+
|
| 16 |
+
# Get the content of the 5th script tag
|
| 17 |
+
# script_content = await page.evaluate('''() => {
|
| 18 |
+
# const scripty=document.querySelectorAll('script')[0];
|
| 19 |
+
# return scripty.content
|
| 20 |
+
# }''')
|
| 21 |
+
# print(script_content)
|
| 22 |
+
# await page.evaluate(f'''{script_content}(''')
|
| 23 |
+
|
| 24 |
+
page_content = await page.content()
|
| 25 |
+
# Print the content of the 5th script tag
|
| 26 |
+
# print(page_content)
|
| 27 |
+
|
| 28 |
+
# Close the browser
|
| 29 |
+
await browser.close()
|
| 30 |
+
|
| 31 |
+
# Run the extraction function
|
| 32 |
+
if __name__ == '__main__':
|
| 33 |
+
asyncio.run(extract_bootstrap_data())
|
App/TTS/utils/Podcastle.py
CHANGED
|
@@ -1,19 +1,20 @@
|
|
| 1 |
import aiohttp
|
| 2 |
import asyncio
|
| 3 |
-
from App.TTS.Schemas import TTSGenerateRequest,StatusRequest
|
| 4 |
from pydantic import BaseModel
|
| 5 |
|
|
|
|
| 6 |
class PodcastleAPI:
|
| 7 |
def __init__(self, username, password):
|
| 8 |
self.base_url = "https://podcastle.ai/api"
|
| 9 |
self.username = username
|
| 10 |
self.password = password
|
| 11 |
self.headers = {
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
# Add your other headers here
|
| 18 |
}
|
| 19 |
self.session = None # Initialize the session in the constructor
|
|
@@ -28,17 +29,14 @@ class PodcastleAPI:
|
|
| 28 |
|
| 29 |
async def signin(self):
|
| 30 |
url = f"{self.base_url}/auth/signin"
|
| 31 |
-
payload = {
|
| 32 |
-
"username": self.username,
|
| 33 |
-
"password": self.password
|
| 34 |
-
}
|
| 35 |
|
| 36 |
if not self.session:
|
| 37 |
await self.create_session()
|
| 38 |
|
| 39 |
async with self.session.post(url, json=payload) as response:
|
| 40 |
response_data = await response.json()
|
| 41 |
-
self.access_token = response_data[
|
| 42 |
return response_data
|
| 43 |
|
| 44 |
async def make_request(self, tts_request: TTSGenerateRequest):
|
|
@@ -49,17 +47,21 @@ class PodcastleAPI:
|
|
| 49 |
await self.signin()
|
| 50 |
|
| 51 |
headers_with_auth = self.headers.copy()
|
| 52 |
-
headers_with_auth[
|
| 53 |
|
| 54 |
url = f"{self.base_url}/speech/text-to-speech"
|
| 55 |
|
| 56 |
-
async with self.session.post(
|
|
|
|
|
|
|
| 57 |
if response.status == 401:
|
| 58 |
# If a 401 error is encountered, sign in again to update the access token
|
| 59 |
await self.signin()
|
| 60 |
# Retry the request with the updated access token
|
| 61 |
-
headers_with_auth[
|
| 62 |
-
async with self.session.post(
|
|
|
|
|
|
|
| 63 |
response_text = await retry_response.json()
|
| 64 |
return response_text
|
| 65 |
else:
|
|
@@ -74,7 +76,7 @@ class PodcastleAPI:
|
|
| 74 |
await self.signin()
|
| 75 |
|
| 76 |
headers_with_auth = self.headers.copy()
|
| 77 |
-
headers_with_auth[
|
| 78 |
|
| 79 |
url = f"{self.base_url}/speech/text-to-speech/{tts_status.requestId}"
|
| 80 |
|
|
@@ -83,17 +85,16 @@ class PodcastleAPI:
|
|
| 83 |
# If a 401 error is encountered, sign in again to update the access token
|
| 84 |
await self.signin()
|
| 85 |
# Retry the request with the updated access token
|
| 86 |
-
headers_with_auth[
|
| 87 |
-
async with self.session.get(
|
|
|
|
|
|
|
| 88 |
response_text = await retry_response.json()
|
| 89 |
return response_text
|
| 90 |
else:
|
| 91 |
response_text = await response.json()
|
| 92 |
return response_text
|
| 93 |
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
async def __aenter__(self):
|
| 98 |
if not self.session:
|
| 99 |
await self.create_session()
|
|
@@ -101,40 +102,3 @@ class PodcastleAPI:
|
|
| 101 |
|
| 102 |
async def __aexit__(self, exc_type, exc_value, traceback):
|
| 103 |
await self.close_session()
|
| 104 |
-
|
| 105 |
-
# Example usage:
|
| 106 |
-
if __name__ == "__main__":
|
| 107 |
-
class Speak(BaseModel):
|
| 108 |
-
paragraphId: str
|
| 109 |
-
text: str
|
| 110 |
-
speaker: str
|
| 111 |
-
|
| 112 |
-
class TTSGenerateRequest(BaseModel):
|
| 113 |
-
paragraphs: [Speak]
|
| 114 |
-
requestId: str
|
| 115 |
-
workspaceId: str
|
| 116 |
-
|
| 117 |
-
async def main():
|
| 118 |
-
username = "[email protected]"
|
| 119 |
-
password = "k7bNvgmJUda3yEG"
|
| 120 |
-
|
| 121 |
-
# Create a TTSGenerateRequest object
|
| 122 |
-
tts_request = TTSGenerateRequest(
|
| 123 |
-
paragraphs=[
|
| 124 |
-
Speak(
|
| 125 |
-
paragraphId="6f05p",
|
| 126 |
-
text="<speak>Hey Daniel. Are you ok?. Manchester United almost lost yesterday </speak>",
|
| 127 |
-
speaker="c60166365edf46589657770d"
|
| 128 |
-
)
|
| 129 |
-
],
|
| 130 |
-
requestId="7d6018ae-9617-4d22-879f-5e67283fa140",
|
| 131 |
-
workspaceId="f84fd58e-2899-4531-9f51-77c155c1e294"
|
| 132 |
-
)
|
| 133 |
-
|
| 134 |
-
async with PodcastleAPI(username, password) as podcastle_api:
|
| 135 |
-
# Make the TTS request using the TTSGenerateRequest object
|
| 136 |
-
response_text = await podcastle_api.make_request(tts_request)
|
| 137 |
-
print(response_text)
|
| 138 |
-
|
| 139 |
-
loop = asyncio.get_event_loop()
|
| 140 |
-
loop.run_until_complete(main())
|
|
|
|
| 1 |
import aiohttp
|
| 2 |
import asyncio
|
| 3 |
+
from App.TTS.Schemas import TTSGenerateRequest, StatusRequest
|
| 4 |
from pydantic import BaseModel
|
| 5 |
|
| 6 |
+
|
| 7 |
class PodcastleAPI:
|
| 8 |
def __init__(self, username, password):
|
| 9 |
self.base_url = "https://podcastle.ai/api"
|
| 10 |
self.username = username
|
| 11 |
self.password = password
|
| 12 |
self.headers = {
|
| 13 |
+
"authority": "podcastle.ai",
|
| 14 |
+
"accept": "*/*",
|
| 15 |
+
"accept-language": "en-US,en;q=0.9",
|
| 16 |
+
"cache-control": "no-cache",
|
| 17 |
+
"content-type": "application/json",
|
| 18 |
# Add your other headers here
|
| 19 |
}
|
| 20 |
self.session = None # Initialize the session in the constructor
|
|
|
|
| 29 |
|
| 30 |
async def signin(self):
|
| 31 |
url = f"{self.base_url}/auth/signin"
|
| 32 |
+
payload = {"username": self.username, "password": self.password}
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
if not self.session:
|
| 35 |
await self.create_session()
|
| 36 |
|
| 37 |
async with self.session.post(url, json=payload) as response:
|
| 38 |
response_data = await response.json()
|
| 39 |
+
self.access_token = response_data["auth"]["accessToken"]
|
| 40 |
return response_data
|
| 41 |
|
| 42 |
async def make_request(self, tts_request: TTSGenerateRequest):
|
|
|
|
| 47 |
await self.signin()
|
| 48 |
|
| 49 |
headers_with_auth = self.headers.copy()
|
| 50 |
+
headers_with_auth["authorization"] = f"Bearer {self.access_token}"
|
| 51 |
|
| 52 |
url = f"{self.base_url}/speech/text-to-speech"
|
| 53 |
|
| 54 |
+
async with self.session.post(
|
| 55 |
+
url, json=tts_request.dict(), headers=headers_with_auth
|
| 56 |
+
) as response:
|
| 57 |
if response.status == 401:
|
| 58 |
# If a 401 error is encountered, sign in again to update the access token
|
| 59 |
await self.signin()
|
| 60 |
# Retry the request with the updated access token
|
| 61 |
+
headers_with_auth["authorization"] = f"Bearer {self.access_token}"
|
| 62 |
+
async with self.session.post(
|
| 63 |
+
url, json=tts_request.dict(), headers=headers_with_auth
|
| 64 |
+
) as retry_response:
|
| 65 |
response_text = await retry_response.json()
|
| 66 |
return response_text
|
| 67 |
else:
|
|
|
|
| 76 |
await self.signin()
|
| 77 |
|
| 78 |
headers_with_auth = self.headers.copy()
|
| 79 |
+
headers_with_auth["authorization"] = f"Bearer {self.access_token}"
|
| 80 |
|
| 81 |
url = f"{self.base_url}/speech/text-to-speech/{tts_status.requestId}"
|
| 82 |
|
|
|
|
| 85 |
# If a 401 error is encountered, sign in again to update the access token
|
| 86 |
await self.signin()
|
| 87 |
# Retry the request with the updated access token
|
| 88 |
+
headers_with_auth["authorization"] = f"Bearer {self.access_token}"
|
| 89 |
+
async with self.session.get(
|
| 90 |
+
url, headers=headers_with_auth
|
| 91 |
+
) as retry_response:
|
| 92 |
response_text = await retry_response.json()
|
| 93 |
return response_text
|
| 94 |
else:
|
| 95 |
response_text = await response.json()
|
| 96 |
return response_text
|
| 97 |
|
|
|
|
|
|
|
|
|
|
| 98 |
async def __aenter__(self):
|
| 99 |
if not self.session:
|
| 100 |
await self.create_session()
|
|
|
|
| 102 |
|
| 103 |
async def __aexit__(self, exc_type, exc_value, traceback):
|
| 104 |
await self.close_session()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|