Upload 9 files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+auto-sub.gif filter=lfs diff=lfs merge=lfs -text

01_🎥_Input_YouTube_Link.py

@@ -0,0 +1,256 @@
+import whisper
+from pytube import YouTube
+import requests
+import time
+import streamlit as st
+from streamlit_lottie import st_lottie
+import numpy as np
+import os
+from typing import Iterator
+from io import StringIO
+from utils import write_vtt, write_srt
+import ffmpeg
+from languages import LANGUAGES
+import torch
+from zipfile import ZipFile
+from io import BytesIO
+import base64
+import pathlib
+import re
+
+st.set_page_config(page_title="Auto Subtitled Video Generator", page_icon=":movie_camera:", layout="wide")
+
+torch.cuda.is_available()
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+# Model options: tiny, base, small, medium, large
+loaded_model = whisper.load_model("small", device=DEVICE)
+
+
+
+# Define a function that we can use to load lottie files from a link.
+@st.cache(allow_output_mutation=True)
+def load_lottieurl(url: str):
+    r = requests.get(url)
+    if r.status_code != 200:
+        return None
+    return r.json()
+
+APP_DIR = pathlib.Path(__file__).parent.absolute()
+
+LOCAL_DIR = APP_DIR / "local_youtube"
+LOCAL_DIR.mkdir(exist_ok=True)
+save_dir = LOCAL_DIR / "output"
+save_dir.mkdir(exist_ok=True)
+
+
+
+col1, col2 = st.columns([1, 3])
+with col1:
+    lottie = load_lottieurl("https://assets8.lottiefiles.com/packages/lf20_jh9gfdye.json")
+    st_lottie(lottie)
+
+with col2:
+    st.write("""
+    ## Auto Subtitled Video Generator 
+    ##### Input a YouTube video link and get a video with subtitles.
+    ###### ➠ If you want to transcribe the video in its original language, select the task as "Transcribe"
+    ###### ➠ If you want to translate the subtitles to English, select the task as "Translate" 
+    ###### I recommend starting with the base model and then experimenting with the larger models, the small and medium models often work well. """)
+    
+
+def populate_metadata(link):
+    yt = YouTube(link)
+    author = yt.author
+    title = yt.title
+    description = yt.description
+    thumbnail = yt.thumbnail_url
+    length = yt.length
+    views = yt.views
+    return author, title, description, thumbnail, length, views
+
+
+def download_video(link):
+    yt = YouTube(link)
+    video = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first().download()
+    return video
+
+
+def convert(seconds):
+    return time.strftime("%H:%M:%S", time.gmtime(seconds))
+
+
+def inference(link, loaded_model, task):
+    yt = YouTube(link)
+    path = yt.streams.filter(only_audio=True)[0].download(filename=f"{save_dir}/audio.mp3")
+    if task == "Transcribe":
+        options = dict(task="transcribe", best_of=5)
+        results = loaded_model.transcribe(path, **options)
+        vtt = getSubs(results["segments"], "vtt", 80)
+        srt = getSubs(results["segments"], "srt", 80)
+        lang = results["language"]
+        return results["text"], vtt, srt, lang
+    elif task == "Translate":
+        options = dict(task="translate", best_of=5)
+        results = loaded_model.transcribe(path, **options)
+        vtt = getSubs(results["segments"], "vtt", 80)
+        srt = getSubs(results["segments"], "srt", 80)
+        lang = results["language"]
+        return results["text"], vtt, srt, lang
+    else:
+        raise ValueError("Task not supported")
+
+
+def getSubs(segments: Iterator[dict], format: str, maxLineWidth: int) -> str:
+    segmentStream = StringIO()
+
+    if format == 'vtt':
+        write_vtt(segments, file=segmentStream, maxLineWidth=maxLineWidth)
+    elif format == 'srt':
+        write_srt(segments, file=segmentStream, maxLineWidth=maxLineWidth)
+    else:
+        raise Exception("Unknown format " + format)
+
+    segmentStream.seek(0)
+    return segmentStream.read()
+
+
+def get_language_code(language):
+    if language in LANGUAGES.keys():
+        detected_language = LANGUAGES[language]
+        return detected_language
+    else:
+        raise ValueError("Language not supported")
+
+
+def generate_subtitled_video(video, audio, transcript):
+    video_file = ffmpeg.input(video)
+    audio_file = ffmpeg.input(audio)
+    ffmpeg.concat(video_file.filter("subtitles", transcript), audio_file, v=1, a=1).output("youtube_sub.mp4").run(quiet=True, overwrite_output=True)
+    video_with_subs = open("youtube_sub.mp4", "rb")
+    return video_with_subs        
+    
+
+def main():
+    link = st.text_input("YouTube Link (The longer the video, the longer the processing time)", placeholder="Input YouTube link and press enter")
+    task = st.selectbox("Select Task", ["Transcribe", "Translate"], index=0)
+    if task == "Transcribe":
+        if st.button("Transcribe"):
+            author, title, description, thumbnail, length, views = populate_metadata(link)
+            with st.spinner("Transcribing the video..."):
+                results = inference(link, loaded_model, task)
+            video = download_video(link)
+            lang = results[3]
+            detected_language = get_language_code(lang)
+                
+            col3, col4 = st.columns(2)
+            with col3:
+                st.video(video)
+            
+            # Split result["text"]  on !,? and . , but save the punctuation
+            sentences = re.split("([!?.])", results[0])
+            # Join the punctuation back to the sentences
+            sentences = ["".join(i) for i in zip(sentences[0::2], sentences[1::2])]
+            text = "\n\n".join(sentences)
+            with open("transcript.txt", "w+", encoding='utf8') as f:
+                f.writelines(text)
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.txt"), "rb") as f:
+                datatxt = f.read()
+                
+            with open("transcript.vtt", "w+",encoding='utf8') as f:
+                f.writelines(results[1])
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.vtt"), "rb") as f:
+                datavtt = f.read()
+                
+            with open("transcript.srt", "w+",encoding='utf8') as f:
+                f.writelines(results[2])
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.srt"), "rb") as f:
+                datasrt = f.read()
+  
+            with col4:
+                with st.spinner("Generating Subtitled Video"):
+                    video_with_subs = generate_subtitled_video(video, f"{save_dir}/audio.mp3", "transcript.srt")
+                st.video(video_with_subs)
+                st.balloons()
+
+            zipObj = ZipFile("YouTube_transcripts_and_video.zip", "w")
+            zipObj.write("transcript.txt")
+            zipObj.write("transcript.vtt")
+            zipObj.write("transcript.srt")
+            zipObj.write("youtube_sub.mp4")
+            zipObj.close()
+            ZipfileDotZip = "YouTube_transcripts_and_video.zip"
+            with open(ZipfileDotZip, "rb") as f:
+                datazip = f.read()
+                b64 = base64.b64encode(datazip).decode()
+                href = f"<a href=\"data:file/zip;base64,{b64}\" download='{ZipfileDotZip}'>\
+        Download Transcripts and Video\
+    </a>"
+            st.markdown(href, unsafe_allow_html=True)
+            
+    elif task == "Translate":
+        if st.button("Translate to English"):
+            author, title, description, thumbnail, length, views = populate_metadata(link)
+            with st.spinner("Translating to English..."):
+                results = inference(link, loaded_model, task)
+            video = download_video(link)
+            lang = results[3]
+            detected_language = get_language_code(lang)
+                
+            col3, col4 = st.columns(2)
+            with col3:
+                st.video(video)
+                
+            # Split result["text"]  on !,? and . , but save the punctuation
+            sentences = re.split("([!?.])", results[0])
+            # Join the punctuation back to the sentences
+            sentences = ["".join(i) for i in zip(sentences[0::2], sentences[1::2])]
+            text = "\n\n".join(sentences)
+            with open("transcript.txt", "w+", encoding='utf8') as f:
+                f.writelines(text)
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.txt"), "rb") as f:
+                datatxt = f.read()
+                
+            with open("transcript.vtt", "w+",encoding='utf8') as f:
+                f.writelines(results[1])
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.vtt"), "rb") as f:
+                datavtt = f.read()
+                
+            with open("transcript.srt", "w+",encoding='utf8') as f:
+                f.writelines(results[2])
+                f.close()
+            with open(os.path.join(os.getcwd(), "transcript.srt"), "rb") as f:
+                datasrt = f.read()
+                       
+            with col4:
+                with st.spinner("Generating Subtitled Video"):
+                    video_with_subs = generate_subtitled_video(video, f"{save_dir}/audio.mp3", "transcript.srt")
+                st.video(video_with_subs)
+                st.balloons()
+            
+            zipObj = ZipFile("YouTube_transcripts_and_video.zip", "w")
+            zipObj.write("transcript.txt")
+            zipObj.write("transcript.vtt")
+            zipObj.write("transcript.srt")
+            zipObj.write("youtube_sub.mp4")
+            zipObj.close()
+            ZipfileDotZip = "YouTube_transcripts_and_video.zip"
+            with open(ZipfileDotZip, "rb") as f:
+                datazip = f.read()
+                b64 = base64.b64encode(datazip).decode()
+                href = f"<a href=\"data:file/zip;base64,{b64}\" download='{ZipfileDotZip}'>\
+        Download Transcripts and Video\
+    </a>"
+            st.markdown(href, unsafe_allow_html=True)
+            
+    else:
+        st.info("Please select a task.")
+
+
+if __name__ == "__main__":
+    main()
+    

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.10
+
+RUN pip install virtualenv
+ENV VIRTUAL_ENV=/venv
+RUN virtualenv venv -p python3
+ENV PATH="VIRTUAL_ENV/bin:$PATH"
+
+WORKDIR /app
+ADD . /app
+
+# Install dependencies
+RUN pip install -r requirements.txt
+RUN apt-get update
+RUN apt-get install -y ffmpeg
+
+# Expose port 
+ENV PORT 8501
+
+# Run the application:
+CMD ["streamlit","run","01_🎥_Input_YouTube_Link.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Batuhan Yılmaz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,22 @@
----
-title: Auto Subtitled Video Generator
-emoji: 🐢
-colorFrom: yellow
-colorTo: red
-sdk: streamlit
-sdk_version: 1.15.2
-app_file: app.py
-pinned: false
-license: other
----
+## Auto-Subtitled-Video-Generator
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+![Python](https://img.shields.io/badge/Python-FFD43B?style=for-the-badge&logo=python&logoColor=blue)
+![Streamlit](https://img.shields.io/badge/Streamlit-FF4B4B?style=for-the-badge&logo=Streamlit&logoColor=white)
+![OpenAI](https://camo.githubusercontent.com/ea872adb9aba9cf6b4e976262f6d4b83b97972d0d5a7abccfde68eb2ae55325f/68747470733a2f2f696d672e736869656c64732e696f2f7374617469632f76313f7374796c653d666f722d7468652d6261646765266d6573736167653d4f70656e414926636f6c6f723d343132393931266c6f676f3d4f70656e4149266c6f676f436f6c6f723d464646464646266c6162656c3d)
+
+#### About this project
+- This project is an automatic speech recognition application that takes a YouTube video link or a video file as input to generate a video with subtitles.
+- You can also upload an audio file to generate a transcript as .txt, .vtt, .srt files.
+- The application performs 2 tasks:
+  - Detects the language, transcribes the input video in its original language.
+  - Detects the language, translates it into English and then transcribes.
+- Downloaded the video of the input link using [pytube](https://github.com/pytube/pytube).
+- Generated a transcription of the video using the [OpenAI Whisper](https://openai.com/blog/whisper) model.
+- Saved the transcriptions as .txt, .vtt and .srt files.
+- Generated a subtitled version of the input video using [ffmpeg](https://github.com/FFmpeg).
+- Displayed the original video and the subtitled video side by side.
+- Built a multipage web app using [Streamlit](https://streamlit.io) and hosted on [HuggingFace Spaces](https://huggingface.co/spaces).
+- You can download the generated .txt, .vtt, .srt files and the subtitled video.
+- You can use the app via this [link](https://huggingface.co/spaces/BatuhanYilmaz/Auto-Subtitled-Video-Generator).
+
+![](auto-sub.gif)

languages.py

@@ -0,0 +1,101 @@
+LANGUAGES = {
+    "en": "eng",
+    "zh": "zho",
+    "de": "deu",
+    "es": "spa",
+    "ru": "rus",
+    "ko": "kor",
+    "fr": "fra",
+    "ja": "jpn",
+    "pt": "por",
+    "tr": "tur",
+    "pl": "pol",
+    "ca": "cat",
+    "nl": "nld",
+    "ar": "ara",
+    "sv": "swe",
+    "it": "ita",
+    "id": "ind",
+    "hi": "hin",
+    "fi": "fin",
+    "vi": "vie",
+    "iw": "heb",
+    "uk": "ukr",
+    "el": "ell",
+    "ms": "msa",
+    "cs": "ces",
+    "ro": "ron",
+    "da": "dan",
+    "hu": "hun",
+    "ta": "tam",
+    "no": "nor",
+    "th": "tha",
+    "ur": "urd",
+    "hr": "hrv",
+    "bg": "bul",
+    "lt": "lit",
+    "la": "lat",
+    "mi": "mri",
+    "ml": "mal",
+    "cy": "cym",
+    "sk": "slk",
+    "te": "tel",
+    "fa": "fas",
+    "lv": "lav",
+    "bn": "ben",
+    "sr": "srp",
+    "az": "aze",
+    "sl": "slv",
+    "kn": "kan",
+    "et": "est",
+    "mk": "mkd",
+    "br": "bre",
+    "eu": "eus",
+    "is": "isl",
+    "hy": "hye",
+    "ne": "nep",
+    "mn": "mon",
+    "bs": "bos",
+    "kk": "kaz",
+    "sq": "sqi",
+    "sw": "swa",
+    "gl": "glg",
+    "mr": "mar",
+    "pa": "pan",
+    "si": "sin",
+    "km": "khm",
+    "sn": "sna",
+    "yo": "yor",
+    "so": "som",
+    "af": "afr",
+    "oc": "oci",
+    "ka": "kat",
+    "be": "bel",
+    "tg": "tgk",
+    "sd": "snd",
+    "gu": "guj",
+    "am": "amh",
+    "yi": "yid",
+    "lo": "lao",
+    "uz": "uzb",
+    "fo": "fao",
+    "ht": "hat",
+    "ps": "pus",
+    "tk": "tuk",
+    "nn": "nno",
+    "mt": "mlt",
+    "sa": "san",
+    "lb": "ltz",
+    "my": "mya",
+    "bo": "bod",
+    "tl": "tgl",
+    "mg": "mlg",
+    "as": "asm",
+    "tt": "tat",
+    "haw": "haw",
+    "ln": "lin",
+    "ha": "hau",
+    "ba": "bak",
+    "jw": "jav",
+    "su": "sun",
+}

packages.txt

@@ -0,0 +1 @@
+ffmpeg

requirements.txt

@@ -0,0 +1,9 @@
+git+https://github.com/openai/whisper.git
+ffmpeg==1.4
+ffmpeg_python==0.2.0
+numpy==1.23.3
+pytube==12.1.0
+requests==2.28.1
+streamlit==1.13.0
+streamlit_lottie==0.0.3
+whisper

utils.py

@@ -0,0 +1,96 @@
+import textwrap
+import zlib
+from typing import Iterator, TextIO
+
+
+def exact_div(x, y):
+    assert x % y == 0
+    return x // y
+
+
+def str2bool(string):
+    str2val = {"True": True, "False": False}
+    if string in str2val:
+        return str2val[string]
+    else:
+        raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}")
+
+
+def optional_int(string):
+    return None if string == "None" else int(string)
+
+
+def optional_float(string):
+    return None if string == "None" else float(string)
+
+
+def compression_ratio(text) -> float:
+    return len(text) / len(zlib.compress(text.encode("utf-8")))
+
+
+def format_timestamp(seconds: float, always_include_hours: bool = False, fractionalSeperator: str = '.'):
+    assert seconds >= 0, "non-negative timestamp expected"
+    milliseconds = round(seconds * 1000.0)
+
+    hours = milliseconds // 3_600_000
+    milliseconds -= hours * 3_600_000
+
+    minutes = milliseconds // 60_000
+    milliseconds -= minutes * 60_000
+
+    seconds = milliseconds // 1_000
+    milliseconds -= seconds * 1_000
+
+    hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""
+    return f"{hours_marker}{minutes:02d}:{seconds:02d}{fractionalSeperator}{milliseconds:03d}"
+
+
+def write_txt(transcript: Iterator[dict], file: TextIO):
+    for segment in transcript:
+        print(segment['text'].strip(), file=file, flush=True)
+
+
+def write_vtt(transcript: Iterator[dict], file: TextIO, maxLineWidth=None):
+    print("WEBVTT\n", file=file)
+    for segment in transcript:
+        text = processText(segment['text'], maxLineWidth).replace('-->', '->')
+
+        print(
+            f"{format_timestamp(segment['start'])} --> {format_timestamp(segment['end'])}\n"
+            f"{text}\n",
+            file=file,
+            flush=True,
+        )
+
+
+def write_srt(transcript: Iterator[dict], file: TextIO, maxLineWidth=None):
+    """
+    Write a transcript to a file in SRT format.
+    Example usage:
+        from pathlib import Path
+        from whisper.utils import write_srt
+        result = transcribe(model, audio_path, temperature=temperature, **args)
+        # save SRT
+        audio_basename = Path(audio_path).stem
+        with open(Path(output_dir) / (audio_basename + ".srt"), "w", encoding="utf-8") as srt:
+            write_srt(result["segments"], file=srt)
+    """
+    for i, segment in enumerate(transcript, start=1):
+        text = processText(segment['text'].strip(), maxLineWidth).replace('-->', '->')
+
+        # write srt lines
+        print(
+            f"{i}\n"
+            f"{format_timestamp(segment['start'], always_include_hours=True, fractionalSeperator=',')} --> "
+            f"{format_timestamp(segment['end'], always_include_hours=True, fractionalSeperator=',')}\n"
+            f"{text}\n",
+            file=file,
+            flush=True,
+        )
+
+def processText(text: str, maxLineWidth=None):
+    if (maxLineWidth is None or maxLineWidth < 0):
+        return text
+
+    lines = textwrap.wrap(text, width=maxLineWidth, tabsize=4)
+    return '\n'.join(lines)