Hugging Face's logo

Spaces:
Yakova
/
Embedding
Running

App Files Community
Mbonea commited on
Commit
95a7280
·
0 Parent(s):

initial commit

Browse files
Files changed (16) hide show
  1. .gitignore +4 -0
  2. App/Analytics/AnalyticsRoutes.py +7 -0
  3. App/Analytics/Model.py +23 -0
  4. App/Analytics/Schemas.py +13 -0
  5. App/Embedding/EmbeddingRoutes.py +22 -0
  6. App/Embedding/Schemas.py +14 -0
  7. App/Embedding/text.py +793 -0
  8. App/Embedding/utils/Elastic.py +29 -0
  9. App/Embedding/utils/Initialize.py +48 -0
  10. App/Embedding/utils/__init__.py +31 -0
  11. App/Worker.py +46 -0
  12. App/__main__.py +8 -0
  13. App/app.py +41 -0
  14. Dockerfile +25 -0
  15. README.md +8 -0
  16. requirements.txt +7 -0
.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ *.pyc
2
+ *.session
3
+ *.session-journal
4
+ .env
App/Analytics/AnalyticsRoutes.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, status
2
+ from .Schemas import BaseRequest, editRequest
3
+ from .Model import Comments
4
+ from App.Users.Model import User
5
+ from App.Post.Model import Post
6
+
7
+ analytics_router = APIRouter(tags=["Analytics"])
App/Analytics/Model.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import orm
3
+ import psycopg2
4
+ import datetime
5
+ import pydantic
6
+ from App.modelInit import database, models
7
+ from App.Users.Model import User
8
+
9
+
10
+ class Analytics(orm.Model):
11
+ tablename = "Analytics"
12
+ registry = models
13
+ fields = {
14
+ "id": orm.Integer(primary_key=True),
15
+ "user": orm.ForeignKey(
16
+ User, on_delete=orm.CASCADE, allow_null=True
17
+ ), # Optional for unknown users.
18
+ "post": orm.ForeignKey(User, on_delete=orm.CASCADE),
19
+ "ip": orm.String(max_length=100),
20
+ "device": orm.String(max_length=100),
21
+ "country": orm.String(max_length=100),
22
+ "createdAt": orm.DateTime(index=True, default=datetime.datetime.now),
23
+ }
App/Analytics/Schemas.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List, Optional
2
+ from pydantic import EmailStr, BaseModel
3
+ from datetime import date, datetime, time, timedelta
4
+
5
+
6
+ class BaseRequest(BaseModel):
7
+ user: Optional[int]
8
+ id: Optional[int]
9
+ content: Optional[str]
10
+
11
+
12
+ class editRequest(BaseRequest):
13
+ updatedAt: datetime = datetime.now()
App/Embedding/EmbeddingRoutes.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+
3
+ from .utils.Initialize import TextSearch, IdSearch
4
+ from .Schemas import SearchRequest, AddDocumentRequest
5
+
6
+ embeddigs_router = APIRouter(tags=["embeddings"])
7
+
8
+
9
+ # create
10
+ @embeddigs_router.post("/add_document")
11
+ async def create_embeddings(req: AddDocumentRequest):
12
+ pass
13
+
14
+
15
+ @embeddigs_router.post("/search_id")
16
+ async def search_id(req: SearchRequest):
17
+ return IdSearch(query=req.query)
18
+
19
+
20
+ @embeddigs_router.post("/search_text")
21
+ async def search_text(req: SearchRequest):
22
+ return TextSearch(query=req.query)
App/Embedding/Schemas.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+
3
+
4
+ class AddDocumentRequest(BaseModel):
5
+ content: str
6
+ metadata: dict
7
+
8
+
9
+ class SearchRequest(BaseModel):
10
+ query: str
11
+
12
+
13
+ class GetTranscriptions(BaseModel):
14
+ userId: int
App/Embedding/text.py ADDED
@@ -0,0 +1,793 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "took": 8,
3
+ "timed_out": false,
4
+ "_shards": {
5
+ "total": 1,
6
+ "successful": 1,
7
+ "skipped": 0,
8
+ "failed": 0
9
+ },
10
+ "hits": {
11
+ "total": {
12
+ "value": 171,
13
+ "relation": "eq"
14
+ },
15
+ "max_score": null,
16
+ "hits": [
17
+ {
18
+ "_index": "telegram_media",
19
+ "_type": "_doc",
20
+ "_id": "11546",
21
+ "_score": 1,
22
+ "_source": {
23
+ "imdb_id": "tt0938283",
24
+ "year": 2010,
25
+ "genres": [
26
+ "action",
27
+ "adventure",
28
+ "family",
29
+ "fantasy"
30
+ ],
31
+ "imageUrl": "http://telegra.ph/file/4b595f25be6d4802ed366.jpg",
32
+ "file_id": "BAADBAADrQkAApRJeFGQj0Yiqy2QBRYE",
33
+ "rating": 4.1,
34
+ "caption": "The Last Airbender",
35
+ "title": "The Last Airbender",
36
+ "type": "movie",
37
+ "poster": "/zgwRTYWEEPivTwjB9S03HtmMcbM.jpg",
38
+ "filecode": "xpf2ra1pbe9e"
39
+ },
40
+ "fields": {
41
+ "imdb_id.keyword": [
42
+ "tt0938283"
43
+ ]
44
+ },
45
+ "inner_hits": {
46
+ "simple": {
47
+ "hits": {
48
+ "total": {
49
+ "value": 1,
50
+ "relation": "eq"
51
+ },
52
+ "max_score": null,
53
+ "hits": [
54
+ {
55
+ "_index": "telegram_media",
56
+ "_type": "_doc",
57
+ "_id": "11546",
58
+ "_score": null,
59
+ "_source": {
60
+ "imdb_id": "tt0938283",
61
+ "year": 2010,
62
+ "genres": [
63
+ "action",
64
+ "adventure",
65
+ "family",
66
+ "fantasy"
67
+ ],
68
+ "imageUrl": "http://telegra.ph/file/4b595f25be6d4802ed366.jpg",
69
+ "file_id": "BAADBAADrQkAApRJeFGQj0Yiqy2QBRYE",
70
+ "rating": 4.1,
71
+ "caption": "The Last Airbender",
72
+ "title": "The Last Airbender",
73
+ "type": "movie",
74
+ "poster": "/zgwRTYWEEPivTwjB9S03HtmMcbM.jpg",
75
+ "filecode": "xpf2ra1pbe9e"
76
+ },
77
+ "fields": {
78
+ "caption": [
79
+ "The Last Airbender"
80
+ ]
81
+ },
82
+ "sort": [
83
+ 9223372036854776000
84
+ ]
85
+ }
86
+ ]
87
+ }
88
+ }
89
+ }
90
+ },
91
+ {
92
+ "_index": "telegram_media",
93
+ "_type": "_doc",
94
+ "_id": "12113",
95
+ "_score": 1,
96
+ "_source": {
97
+ "imdb_id": "tt10011298",
98
+ "year": 2020,
99
+ "genres": [
100
+ "animation",
101
+ "fantasy"
102
+ ],
103
+ "imageUrl": "http://telegra.ph/file/52df5f37e4730af07cba6.jpg",
104
+ "file_id": "BAADBAADSAcAAiX1OVMhjJ-3dGM2eBYE",
105
+ "rating": 6.7,
106
+ "season_number": 1001,
107
+ "caption": "Dragons Dogma S01E01",
108
+ "title": "Dragon's Dogma",
109
+ "type": "tvseries",
110
+ "poster": "/m8VA5fudLl3jXywxxSsE4g5ltYS.jpg",
111
+ "filecode": "autlcdo64kzi"
112
+ },
113
+ "fields": {
114
+ "imdb_id.keyword": [
115
+ "tt10011298"
116
+ ]
117
+ },
118
+ "inner_hits": {
119
+ "simple": {
120
+ "hits": {
121
+ "total": {
122
+ "value": 7,
123
+ "relation": "eq"
124
+ },
125
+ "max_score": null,
126
+ "hits": [
127
+ {
128
+ "_index": "telegram_media",
129
+ "_type": "_doc",
130
+ "_id": "12113",
131
+ "_score": null,
132
+ "_source": {
133
+ "imdb_id": "tt10011298",
134
+ "year": 2020,
135
+ "genres": [
136
+ "animation",
137
+ "fantasy"
138
+ ],
139
+ "imageUrl": "http://telegra.ph/file/52df5f37e4730af07cba6.jpg",
140
+ "file_id": "BAADBAADSAcAAiX1OVMhjJ-3dGM2eBYE",
141
+ "rating": 6.7,
142
+ "season_number": 1001,
143
+ "caption": "Dragons Dogma S01E01",
144
+ "title": "Dragon's Dogma",
145
+ "type": "tvseries",
146
+ "poster": "/m8VA5fudLl3jXywxxSsE4g5ltYS.jpg",
147
+ "filecode": "autlcdo64kzi"
148
+ },
149
+ "fields": {
150
+ "caption": [
151
+ "Dragons Dogma S01E01"
152
+ ]
153
+ },
154
+ "sort": [
155
+ 1001
156
+ ]
157
+ }
158
+ ]
159
+ }
160
+ }
161
+ }
162
+ },
163
+ {
164
+ "_index": "telegram_media",
165
+ "_type": "_doc",
166
+ "_id": "14083",
167
+ "_score": 1,
168
+ "_source": {
169
+ "image": "/app/src/Temp/DOTA Dragons BloodJPEG.jpeg",
170
+ "imdb_id": "tt14069590",
171
+ "year": 2021,
172
+ "genres": [
173
+ "animation",
174
+ "action",
175
+ "adventure",
176
+ "fantasy",
177
+ "sci-fi",
178
+ "thriller"
179
+ ],
180
+ "imageUrl": "http://telegra.ph/file/9fa86be18c7bc157ec420.jpg",
181
+ "season_number": 1001,
182
+ "rating": 8.9,
183
+ "caption": "DOTA Dragons Blood S01E01",
184
+ "type": "tvseries",
185
+ "title": "Dota: Dragon's Blood",
186
+ "poster": "/s1wP1YeQS9fgSHiXZ3yJb2ufB2D.jpg",
187
+ "filecode": "netl7jsra9i7"
188
+ },
189
+ "fields": {
190
+ "imdb_id.keyword": [
191
+ "tt14069590"
192
+ ]
193
+ },
194
+ "inner_hits": {
195
+ "simple": {
196
+ "hits": {
197
+ "total": {
198
+ "value": 24,
199
+ "relation": "eq"
200
+ },
201
+ "max_score": null,
202
+ "hits": [
203
+ {
204
+ "_index": "telegram_media",
205
+ "_type": "_doc",
206
+ "_id": "14083",
207
+ "_score": null,
208
+ "_source": {
209
+ "image": "/app/src/Temp/DOTA Dragons BloodJPEG.jpeg",
210
+ "imdb_id": "tt14069590",
211
+ "year": 2021,
212
+ "genres": [
213
+ "animation",
214
+ "action",
215
+ "adventure",
216
+ "fantasy",
217
+ "sci-fi",
218
+ "thriller"
219
+ ],
220
+ "imageUrl": "http://telegra.ph/file/9fa86be18c7bc157ec420.jpg",
221
+ "season_number": 1001,
222
+ "rating": 8.9,
223
+ "caption": "DOTA Dragons Blood S01E01",
224
+ "type": "tvseries",
225
+ "title": "Dota: Dragon's Blood",
226
+ "poster": "/s1wP1YeQS9fgSHiXZ3yJb2ufB2D.jpg",
227
+ "filecode": "netl7jsra9i7"
228
+ },
229
+ "fields": {
230
+ "caption": [
231
+ "DOTA Dragons Blood S01E01"
232
+ ]
233
+ },
234
+ "sort": [
235
+ 1001
236
+ ]
237
+ }
238
+ ]
239
+ }
240
+ }
241
+ }
242
+ },
243
+ {
244
+ "_index": "telegram_media",
245
+ "_type": "_doc",
246
+ "_id": "1357",
247
+ "_score": 1,
248
+ "_source": {
249
+ "year": 2012,
250
+ "imdb_id": "tt1695360",
251
+ "genres": [
252
+ "animation",
253
+ "action",
254
+ "adventure",
255
+ "comedy",
256
+ "drama",
257
+ "family",
258
+ "fantasy",
259
+ "sci-fi"
260
+ ],
261
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BNTRmM2E5OWYtYmYzMi00MDRiLTg2ZTYtZGQyZTNhZDMxYTgzXkEyXkFqcGdeQXVyODUwNjEzMzg@._V1_.jpg",
262
+ "file_id": "BAADBAADmAYAAvIleVK5TCg7gH9TlBYE",
263
+ "rating": 8.4,
264
+ "season_number": 1001,
265
+ "caption": "The Legend of Korra S01E01",
266
+ "type": "tvseries",
267
+ "title": "The Legend of Korra",
268
+ "poster": "/rvaKYJk4oMN1qN6vbwUQNExW7WH.jpg",
269
+ "filecode": "nmbekmxid4pv"
270
+ },
271
+ "fields": {
272
+ "imdb_id.keyword": [
273
+ "tt1695360"
274
+ ]
275
+ },
276
+ "inner_hits": {
277
+ "simple": {
278
+ "hits": {
279
+ "total": {
280
+ "value": 50,
281
+ "relation": "eq"
282
+ },
283
+ "max_score": null,
284
+ "hits": [
285
+ {
286
+ "_index": "telegram_media",
287
+ "_type": "_doc",
288
+ "_id": "1357",
289
+ "_score": null,
290
+ "_source": {
291
+ "year": 2012,
292
+ "imdb_id": "tt1695360",
293
+ "genres": [
294
+ "animation",
295
+ "action",
296
+ "adventure",
297
+ "comedy",
298
+ "drama",
299
+ "family",
300
+ "fantasy",
301
+ "sci-fi"
302
+ ],
303
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BNTRmM2E5OWYtYmYzMi00MDRiLTg2ZTYtZGQyZTNhZDMxYTgzXkEyXkFqcGdeQXVyODUwNjEzMzg@._V1_.jpg",
304
+ "file_id": "BAADBAADmAYAAvIleVK5TCg7gH9TlBYE",
305
+ "rating": 8.4,
306
+ "season_number": 1001,
307
+ "caption": "The Legend of Korra S01E01",
308
+ "type": "tvseries",
309
+ "title": "The Legend of Korra",
310
+ "poster": "/rvaKYJk4oMN1qN6vbwUQNExW7WH.jpg",
311
+ "filecode": "nmbekmxid4pv"
312
+ },
313
+ "fields": {
314
+ "caption": [
315
+ "The Legend of Korra S01E01"
316
+ ]
317
+ },
318
+ "sort": [
319
+ 1001
320
+ ]
321
+ }
322
+ ]
323
+ }
324
+ }
325
+ }
326
+ },
327
+ {
328
+ "_index": "telegram_media",
329
+ "_type": "_doc",
330
+ "_id": "1890",
331
+ "_score": 1,
332
+ "_source": {
333
+ "year": 2018,
334
+ "imdb_id": "tt8688814",
335
+ "genres": [
336
+ "animation",
337
+ "adventure",
338
+ "family",
339
+ "fantasy"
340
+ ],
341
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BMjA5MjEwODU1MV5BMl5BanBnXkFtZTgwNzk0MzA5NTM@._V1_.jpg",
342
+ "file_id": "BAADBAADHwYAAjgpaVPzOpIuqqJPDBYE",
343
+ "rating": 8.4,
344
+ "season_number": 1006,
345
+ "caption": "The Dragon Prince S01E06",
346
+ "type": "tvseries",
347
+ "title": "The Dragon Prince",
348
+ "poster": "/d7PIRa6ez7ZEl9D4JUrnSsmcnVD.jpg",
349
+ "filecode": "8ihleaphvfjz"
350
+ },
351
+ "fields": {
352
+ "imdb_id.keyword": [
353
+ "tt8688814"
354
+ ]
355
+ },
356
+ "inner_hits": {
357
+ "simple": {
358
+ "hits": {
359
+ "total": {
360
+ "value": 27,
361
+ "relation": "eq"
362
+ },
363
+ "max_score": null,
364
+ "hits": [
365
+ {
366
+ "_index": "telegram_media",
367
+ "_type": "_doc",
368
+ "_id": "1885",
369
+ "_score": null,
370
+ "_source": {
371
+ "year": 2018,
372
+ "imdb_id": "tt8688814",
373
+ "genres": [
374
+ "animation",
375
+ "adventure",
376
+ "family",
377
+ "fantasy"
378
+ ],
379
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BMjA5MjEwODU1MV5BMl5BanBnXkFtZTgwNzk0MzA5NTM@._V1_.jpg",
380
+ "file_id": "BAADBAADFgYAAjgpaVNN70U9JtoIXhYE",
381
+ "rating": 8.4,
382
+ "season_number": 1001,
383
+ "caption": "The Dragon Prince S01E01",
384
+ "type": "tvseries",
385
+ "title": "The Dragon Prince",
386
+ "poster": "/d7PIRa6ez7ZEl9D4JUrnSsmcnVD.jpg",
387
+ "filecode": "5sqed1bbcykq"
388
+ },
389
+ "fields": {
390
+ "caption": [
391
+ "The Dragon Prince S01E01"
392
+ ]
393
+ },
394
+ "sort": [
395
+ 1001
396
+ ]
397
+ }
398
+ ]
399
+ }
400
+ }
401
+ }
402
+ },
403
+ {
404
+ "_index": "telegram_media",
405
+ "_type": "_doc",
406
+ "_id": "4231",
407
+ "_score": 1,
408
+ "_source": {
409
+ "year": 2019,
410
+ "imdb_id": "tt9679542",
411
+ "genres": [
412
+ "animation",
413
+ "action",
414
+ "adventure",
415
+ "drama",
416
+ "fantasy",
417
+ "sci-fi"
418
+ ],
419
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BZTU1ODAyN2UtZjdlOC00ODUwLWE3NjEtYjE3NmViNTAwMzMyXkEyXkFqcGdeQXVyMzgxODM4NjM@._V1_.jpg",
420
+ "file_id": "BAADBAAD6wcAAgcVQFFoZbSLjYJe_hYE",
421
+ "rating": 8.2,
422
+ "season_number": 1021,
423
+ "caption": "Dr Stone S1 21",
424
+ "type": "tvseries",
425
+ "title": "Dokutaa Sutoon",
426
+ "poster": "/dLlnzbDCblBXcJqFLXyvN43NIwp.jpg",
427
+ "filecode": "7vr4zp6f1tf6"
428
+ },
429
+ "fields": {
430
+ "imdb_id.keyword": [
431
+ "tt9679542"
432
+ ]
433
+ },
434
+ "inner_hits": {
435
+ "simple": {
436
+ "hits": {
437
+ "total": {
438
+ "value": 35,
439
+ "relation": "eq"
440
+ },
441
+ "max_score": null,
442
+ "hits": [
443
+ {
444
+ "_index": "telegram_media",
445
+ "_type": "_doc",
446
+ "_id": "4211",
447
+ "_score": null,
448
+ "_source": {
449
+ "year": 2019,
450
+ "imdb_id": "tt9679542",
451
+ "genres": [
452
+ "animation",
453
+ "action",
454
+ "adventure",
455
+ "drama",
456
+ "fantasy",
457
+ "sci-fi"
458
+ ],
459
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BZTU1ODAyN2UtZjdlOC00ODUwLWE3NjEtYjE3NmViNTAwMzMyXkEyXkFqcGdeQXVyMzgxODM4NjM@._V1_.jpg",
460
+ "file_id": "BAADBAAD1AcAAgcVQFEt94NYS5EcyRYE",
461
+ "rating": 8.2,
462
+ "season_number": 1001,
463
+ "caption": "Dr Stone S1 01",
464
+ "type": "tvseries",
465
+ "title": "Dokutaa Sutoon",
466
+ "poster": "/dLlnzbDCblBXcJqFLXyvN43NIwp.jpg",
467
+ "filecode": "d8cg2yiei6zi"
468
+ },
469
+ "fields": {
470
+ "caption": [
471
+ "Dr Stone S1 01"
472
+ ]
473
+ },
474
+ "sort": [
475
+ 1001
476
+ ]
477
+ }
478
+ ]
479
+ }
480
+ }
481
+ }
482
+ },
483
+ {
484
+ "_index": "telegram_media",
485
+ "_type": "_doc",
486
+ "_id": "4282",
487
+ "_score": 1,
488
+ "_source": {
489
+ "year": 2019,
490
+ "imdb_id": "tt9348718",
491
+ "genres": [
492
+ "animation",
493
+ "action",
494
+ "adventure",
495
+ "drama",
496
+ "horror",
497
+ "mystery",
498
+ "romance",
499
+ "sci-fi"
500
+ ],
501
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BODNhNGI1MGUtNTRlMy00ZGI2LWI2NTYtMjE1Nzc1ZDk5MjM1XkEyXkFqcGdeQXVyMTMxODk2OTU@._V1_.jpg",
502
+ "file_id": "BAADBAAD3wkAAgcVSFFB528nZsumyhYE",
503
+ "rating": 6.2,
504
+ "season_number": 1010,
505
+ "caption": "7SEEDS S01E10",
506
+ "type": "tvseries",
507
+ "title": "7Seeds",
508
+ "poster": "/iQMkaPbDKwThXTnpVUAWhtgnQVp.jpg",
509
+ "filecode": "sett835pzl01"
510
+ },
511
+ "fields": {
512
+ "imdb_id.keyword": [
513
+ "tt9348718"
514
+ ]
515
+ },
516
+ "inner_hits": {
517
+ "simple": {
518
+ "hits": {
519
+ "total": {
520
+ "value": 12,
521
+ "relation": "eq"
522
+ },
523
+ "max_score": null,
524
+ "hits": [
525
+ {
526
+ "_index": "telegram_media",
527
+ "_type": "_doc",
528
+ "_id": "4273",
529
+ "_score": null,
530
+ "_source": {
531
+ "year": 2019,
532
+ "imdb_id": "tt9348718",
533
+ "genres": [
534
+ "animation",
535
+ "action",
536
+ "adventure",
537
+ "drama",
538
+ "horror",
539
+ "mystery",
540
+ "romance",
541
+ "sci-fi"
542
+ ],
543
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BODNhNGI1MGUtNTRlMy00ZGI2LWI2NTYtMjE1Nzc1ZDk5MjM1XkEyXkFqcGdeQXVyMTMxODk2OTU@._V1_.jpg",
544
+ "file_id": "BAADBAAD2QkAAgcVSFFeMSNI8mk1EhYE",
545
+ "rating": 6.2,
546
+ "season_number": 1001,
547
+ "caption": "7SEEDS S01E01",
548
+ "type": "tvseries",
549
+ "title": "7Seeds",
550
+ "poster": "/iQMkaPbDKwThXTnpVUAWhtgnQVp.jpg",
551
+ "filecode": "7plswnwvm1l0"
552
+ },
553
+ "fields": {
554
+ "caption": [
555
+ "7SEEDS S01E01"
556
+ ]
557
+ },
558
+ "sort": [
559
+ 1001
560
+ ]
561
+ }
562
+ ]
563
+ }
564
+ }
565
+ }
566
+ },
567
+ {
568
+ "_index": "telegram_media",
569
+ "_type": "_doc",
570
+ "_id": "6623",
571
+ "_score": 1,
572
+ "_source": {
573
+ "year": 2020,
574
+ "imdb_id": "tt8741648",
575
+ "genres": [
576
+ "action",
577
+ "adventure",
578
+ "family",
579
+ "fantasy"
580
+ ],
581
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BOTE3ZDcxODktMmMyNy00NmFjLThhMGQtZDgyNmQ4YmJmYjAwXkEyXkFqcGdeQXVyMTMxODk2OTU@._V1_.jpg",
582
+ "file_id": "BAADBAADRwYAAoyHyFMTBqk9HmNZPxYE",
583
+ "rating": 5.9,
584
+ "season_number": 1006,
585
+ "caption": "The Letter for the King S01E06",
586
+ "type": "tvseries",
587
+ "title": "The Letter for the King",
588
+ "poster": "/4Pwp6tqn0bKRQsbisuJ0t5rBub0.jpg",
589
+ "filecode": "hxwryuznfzzg"
590
+ },
591
+ "fields": {
592
+ "imdb_id.keyword": [
593
+ "tt8741648"
594
+ ]
595
+ },
596
+ "inner_hits": {
597
+ "simple": {
598
+ "hits": {
599
+ "total": {
600
+ "value": 6,
601
+ "relation": "eq"
602
+ },
603
+ "max_score": null,
604
+ "hits": [
605
+ {
606
+ "_index": "telegram_media",
607
+ "_type": "_doc",
608
+ "_id": "6618",
609
+ "_score": null,
610
+ "_source": {
611
+ "year": 2020,
612
+ "imdb_id": "tt8741648",
613
+ "genres": [
614
+ "action",
615
+ "adventure",
616
+ "family",
617
+ "fantasy"
618
+ ],
619
+ "imageUrl": "https://m.media-amazon.com/images/M/MV5BOTE3ZDcxODktMmMyNy00NmFjLThhMGQtZDgyNmQ4YmJmYjAwXkEyXkFqcGdeQXVyMTMxODk2OTU@._V1_.jpg",
620
+ "file_id": "BAADBAADRgYAAoyHyFNMYq4-eZjx_hYE",
621
+ "rating": 5.9,
622
+ "season_number": 1001,
623
+ "caption": "The Letter for the King S01E01",
624
+ "type": "tvseries",
625
+ "title": "The Letter for the King",
626
+ "poster": "/4Pwp6tqn0bKRQsbisuJ0t5rBub0.jpg",
627
+ "filecode": "uzvw0jdjvatn"
628
+ },
629
+ "fields": {
630
+ "caption": [
631
+ "The Letter for the King S01E01"
632
+ ]
633
+ },
634
+ "sort": [
635
+ 1001
636
+ ]
637
+ }
638
+ ]
639
+ }
640
+ }
641
+ }
642
+ },
643
+ {
644
+ "_index": "telegram_media",
645
+ "_type": "_doc",
646
+ "_id": "18370",
647
+ "_score": 1,
648
+ "_source": {
649
+ "imageUrl": "http://telegra.ph/file/c8ea745914a2001c2bd0a.jpg",
650
+ "year": 2022,
651
+ "title": "The Quest",
652
+ "season_number": 1003,
653
+ "imdb_id": "tt11852810",
654
+ "genres": [
655
+ "adventure",
656
+ "fantasy",
657
+ "reality-tv"
658
+ ],
659
+ "rating": 3.7,
660
+ "type": "tvseries",
661
+ "image": "/app/src/Temp/tt11852810JPEG.jpeg",
662
+ "poster": "/nNQvnKK6TZNF6ibEzuCo6OLNdKz.jpg",
663
+ "caption": "The Quest 2022 S01E03",
664
+ "filecode": "u0j6pkhxk4cg"
665
+ },
666
+ "fields": {
667
+ "imdb_id.keyword": [
668
+ "tt11852810"
669
+ ]
670
+ },
671
+ "inner_hits": {
672
+ "simple": {
673
+ "hits": {
674
+ "total": {
675
+ "value": 8,
676
+ "relation": "eq"
677
+ },
678
+ "max_score": null,
679
+ "hits": [
680
+ {
681
+ "_index": "telegram_media",
682
+ "_type": "_doc",
683
+ "_id": "18368",
684
+ "_score": null,
685
+ "_source": {
686
+ "title": "The Quest",
687
+ "season_number": 1001,
688
+ "imdb_id": "tt11852810",
689
+ "type": "tvseries",
690
+ "image": "/app/src/Temp/tt11852810JPEG.jpeg",
691
+ "imageUrl": "http://telegra.ph/file/c8ea745914a2001c2bd0a.jpg",
692
+ "poster": "/nNQvnKK6TZNF6ibEzuCo6OLNdKz.jpg",
693
+ "rating": 3.7,
694
+ "genres": [
695
+ "adventure",
696
+ "fantasy",
697
+ "reality-tv"
698
+ ],
699
+ "year": 2022,
700
+ "caption": "The Quest 2022 S01E01",
701
+ "filecode": "2ji2bhdklwiu"
702
+ },
703
+ "fields": {
704
+ "caption": [
705
+ "The Quest 2022 S01E01"
706
+ ]
707
+ },
708
+ "sort": [
709
+ 1001
710
+ ]
711
+ }
712
+ ]
713
+ }
714
+ }
715
+ }
716
+ },
717
+ {
718
+ "_index": "telegram_media",
719
+ "_type": "_doc",
720
+ "_id": "18777",
721
+ "_score": 1,
722
+ "_source": {
723
+ "rating": 7.8,
724
+ "genres": [
725
+ "action",
726
+ "adventure",
727
+ "fantasy",
728
+ "sci-fi"
729
+ ],
730
+ "poster": "/jRXYjXNq0Cs2TcJjLkki24MLp7u.jpg",
731
+ "type": "movie",
732
+ "imageUrl": "http://telegra.ph/file/af29523b9678b3f754520.jpg",
733
+ "year": 2009,
734
+ "title": "Avatar",
735
+ "imdb_id": "tt0499549",
736
+ "image": "/app/src/Temp/tt0499549JPEG.jpeg",
737
+ "caption": "Avatar",
738
+ "filecode": "h3b15rrf8dvr"
739
+ },
740
+ "fields": {
741
+ "imdb_id.keyword": [
742
+ "tt0499549"
743
+ ]
744
+ },
745
+ "inner_hits": {
746
+ "simple": {
747
+ "hits": {
748
+ "total": {
749
+ "value": 1,
750
+ "relation": "eq"
751
+ },
752
+ "max_score": null,
753
+ "hits": [
754
+ {
755
+ "_index": "telegram_media",
756
+ "_type": "_doc",
757
+ "_id": "18777",
758
+ "_score": null,
759
+ "_source": {
760
+ "rating": 7.8,
761
+ "genres": [
762
+ "action",
763
+ "adventure",
764
+ "fantasy",
765
+ "sci-fi"
766
+ ],
767
+ "poster": "/jRXYjXNq0Cs2TcJjLkki24MLp7u.jpg",
768
+ "type": "movie",
769
+ "imageUrl": "http://telegra.ph/file/af29523b9678b3f754520.jpg",
770
+ "year": 2009,
771
+ "title": "Avatar",
772
+ "imdb_id": "tt0499549",
773
+ "image": "/app/src/Temp/tt0499549JPEG.jpeg",
774
+ "caption": "Avatar",
775
+ "filecode": "h3b15rrf8dvr"
776
+ },
777
+ "fields": {
778
+ "caption": [
779
+ "Avatar"
780
+ ]
781
+ },
782
+ "sort": [
783
+ 9223372036854776000
784
+ ]
785
+ }
786
+ ]
787
+ }
788
+ }
789
+ }
790
+ }
791
+ ]
792
+ }
793
+ }
App/Embedding/utils/Elastic.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from elasticsearch import Elasticsearch
2
+
3
+
4
+ # initialize elasticSearch
5
+ es = Elasticsearch(
6
+ [
7
+ "https://u46hxt12c:[email protected]:443"
8
+ ]
9
+ )
10
+
11
+
12
+ def FetchDocuments(ids):
13
+ es_index_name = "telegram_media"
14
+ res = es.search(
15
+ index=es_index_name,
16
+ body={
17
+ "query": {"terms": {"imdb_id.keyword": ids, "boost": 1.0}},
18
+ "collapse": {
19
+ "field": "imdb_id.keyword",
20
+ "inner_hits": {
21
+ "name": "simple",
22
+ "collapse": {"field": "caption"},
23
+ "sort": [{"season_number": {"order": "asc"}}],
24
+ "size": 1,
25
+ },
26
+ },
27
+ },
28
+ )
29
+ return res
App/Embedding/utils/Initialize.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.embeddings import HuggingFaceEmbeddings
2
+ from langchain.docstore.document import Document
3
+ from langchain.vectorstores import Pinecone
4
+ import os,requests
5
+ import pinecone,pprint
6
+ from .Elastic import FetchDocuments
7
+
8
+
9
+ index_name = 'movie-recommender-fast'
10
+ model_name = "thenlper/gte-base"
11
+ embeddings = HuggingFaceEmbeddings(model_name=model_name)
12
+
13
+ TMDB_API=os.environ.get('TMDB_API')
14
+
15
+ # get api key from app.pinecone.io
16
+ PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY')
17
+ # find your environment next to the api key in pinecone console
18
+ PINECONE_ENV = os.environ.get('PINECONE_ENVIRONMENT')
19
+
20
+ pinecone.init(
21
+ api_key=PINECONE_API_KEY,
22
+ environment=PINECONE_ENV
23
+ )
24
+
25
+ docsearch = Pinecone.from_existing_index(index_name, embeddings)
26
+
27
+ def generate_text(doc):
28
+ if doc['tv_results']:
29
+ return pprint.pformat(doc['tv_results'][0]),doc['tv_results'][0]
30
+ return pprint.pformat(doc['movie_results'][0]),doc['movie_results'][0]
31
+
32
+
33
+
34
+ def IdSearch(query:str):
35
+ doc=requests.get(f'https://api.themoviedb.org/3/find/{query}?external_source=imdb_id&language=en&api_key={TMDB_API}').json()
36
+ text,_=generate_text(doc)
37
+ return TextSearch(text,filter={"key": {"$ne":query}})
38
+
39
+
40
+
41
+ def TextSearch(query: str,filter=None):
42
+ docs = docsearch.similarity_search(query,k=10,filter=filter)
43
+ keys= [ doc.metadata['key'] for doc in docs ]
44
+ return FetchDocuments(keys)
45
+
46
+
47
+
48
+
App/Embedding/utils/__init__.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from elasticsearch import Elasticsearch
2
+ import os
3
+
4
+ elastic_host=os.environ.get('ELASTIC_HOST')
5
+
6
+ # initialize elasticSearch
7
+ es = Elasticsearch(
8
+ [
9
+ elastic_host
10
+ ]
11
+ )
12
+
13
+
14
+ def FetchDocuments(ids):
15
+ es_index_name = "telegram_media"
16
+ res = es.search(
17
+ index=es_index_name,
18
+ body={
19
+ "size": 10,
20
+ "query": {"terms": {"imdb_id.keyword": ids, "boost": 1.0}},
21
+ "collapse": {
22
+ "field": "imdb_id.keyword",
23
+ "inner_hits": {
24
+ "name": "simple",
25
+ "collapse": {"field": "caption"},
26
+ "sort": [{"season_number": {"order": "asc"}}],
27
+ "size": 1,
28
+ },
29
+ },
30
+ },
31
+ )
App/Worker.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from celery import Celery, chain
2
+ import os
3
+ import time
4
+ from App import celery_config
5
+ import yt_dlp
6
+ import tempfile
7
+ from App.Transcription.Utils.audio_transcription import transcribe_file
8
+ from App.Embedding.utils.Initialize import encode, generateChunks
9
+
10
+ celery = Celery()
11
+ celery.config_from_object(celery_config)
12
+
13
+
14
+ @celery.task(name="embbeding", bind=True)
15
+ def generate_store(self, data, task_id):
16
+ chunks = generateChunks(data, task_id)
17
+ encode(chunks)
18
+ print("hellooo")
19
+
20
+
21
+ @celery.task(name="transcription", bind=True)
22
+ def transcription_task(self, file_path, model_size="tiny"):
23
+ data = transcribe_file(state=self, file_path=file_path, model_size=model_size)
24
+ generate_store.delay(data["content"], self.request.id)
25
+ return data
26
+
27
+
28
+ @celery.task(name="download", bind=True)
29
+ def downloadfile(self, url, ydl_opts, model_size="base"):
30
+ # updated
31
+ self.update_state(state="Downloading File..", meta={})
32
+
33
+ ####
34
+ with yt_dlp.YoutubeDL(ydl_opts) as ydl:
35
+ ydl.download([url])
36
+
37
+ # updated
38
+ self.update_state(state="Downloading complete", meta={})
39
+ audio_file = ydl_opts["outtmpl"]
40
+ print(model_size, "worker after")
41
+ # print(audio_file["default"])
42
+ data = transcribe_file(
43
+ state=self, file_path=audio_file["default"], model_size=model_size
44
+ )
45
+ generate_store.delay(data["content"], self.request.id)
46
+ return data
App/__main__.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ from App.app import app
2
+
3
+ import uvicorn
4
+ import asyncio
5
+
6
+
7
+ if __name__ == "__main__":
8
+ uvicorn.run(app, host="0.0.0.0", port=7860)
App/app.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+
3
+ from fastapi.middleware.gzip import GZipMiddleware
4
+
5
+
6
+ from .Embedding.EmbeddingRoutes import embeddigs_router
7
+
8
+
9
+ from fastapi.middleware.cors import CORSMiddleware
10
+
11
+
12
+ import logging
13
+
14
+
15
+ # Configure logging
16
+ logging.basicConfig(
17
+ level=logging.DEBUG,
18
+ format="%(asctime)s - %(levelname)s - %(message)s",
19
+ datefmt="%Y-%m-%d %H:%M:%S",
20
+ )
21
+
22
+
23
+ app = FastAPI()
24
+ origins = ["*"]
25
+
26
+ app.add_middleware(
27
+ CORSMiddleware,
28
+ allow_origins=origins,
29
+ allow_credentials=True,
30
+ allow_methods=["*"],
31
+ allow_headers=["*"],
32
+ )
33
+ app.add_middleware(GZipMiddleware, minimum_size=1000)
34
+
35
+
36
+ @app.get("/")
37
+ async def landing_page():
38
+ return {"code": 200, "message": "we are back!"}
39
+
40
+
41
+ app.include_router(embeddigs_router)
Dockerfile ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Builder stage
2
+ FROM python:latest as builder
3
+
4
+ RUN useradd -ms /bin/bash admin
5
+
6
+ WORKDIR /srv
7
+ RUN chown -R admin:admin /srv
8
+ RUN chmod 755 /srv
9
+
10
+
11
+
12
+
13
+
14
+ COPY requirements.txt .
15
+ RUN pip install --no-cache-dir -r requirements.txt
16
+
17
+ # Copy the application code
18
+ USER admin
19
+
20
+ COPY --chown=admin . /srv
21
+
22
+ # Command to run the application
23
+ CMD uvicorn App.app:app --host 0.0.0.0 --port 7860 --workers 8
24
+ # Expose the server port
25
+ EXPOSE 7860
README.md ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Embedding
3
+ emoji: 🐳
4
+ colorFrom: purple
5
+ colorTo: gray
6
+ sdk: docker
7
+ app_port: 7860
8
+ ---
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fastapi
2
+ elasticsearch==7.7.0
3
+ pinecone-client[grpc]
4
+ sentence-transformers
5
+ langchain
6
+ uvicorn[standard]
7
+ pydantic