Multipage

home.py

@@ -0,0 +1,33 @@
+import logging
+
+import streamlit as st
+
+logger = logging.getLogger()
+
+st.title("streamlit-webrtc demo!")
+st.info(
+    """👈 Select the demo
+"""
+)
+
+
+if __name__ == "__main__":
+    import os
+
+    DEBUG = os.environ.get("DEBUG", "false").lower() not in ["false", "no", "0"]
+
+    logging.basicConfig(
+        format="[%(asctime)s] %(levelname)7s from %(name)s in %(pathname)s:%(lineno)d: "
+        "%(message)s",
+        level=logging.DEBUG if DEBUG else logging.INFO,
+        force=True,
+    )
+
+    fsevents_logger = logging.getLogger("fsevents")
+    fsevents_logger.setLevel(logging.WARNING)
+
+    aiortc_logger = logging.getLogger("aiortc")
+    aiortc_logger.setLevel(logging.INFO)
+
+    aioice_logger = logging.getLogger("aioice")
+    aioice_logger.setLevel(logging.INFO)

pages/.isort.cfg

@@ -0,0 +1,3 @@
+[settings]
+profile=black
+known_third_party=streamlit_webrtc

pages/10_sendonly_audio.py

@@ -0,0 +1,85 @@
+"""A sample to use WebRTC in sendonly mode to transfer audio frames
+from the browser to the server and visualize them with matplotlib
+and `st.pyplot`."""
+
+import logging
+import queue
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pydub
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+logger = logging.getLogger(__name__)
+
+
+webrtc_ctx = webrtc_streamer(
+    key="sendonly-audio",
+    mode=WebRtcMode.SENDONLY,
+    audio_receiver_size=256,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    media_stream_constraints={"audio": True},
+)
+
+fig_place = st.empty()
+
+fig, [ax_time, ax_freq] = plt.subplots(2, 1, gridspec_kw={"top": 1.5, "bottom": 0.2})
+
+sound_window_len = 5000  # 5s
+sound_window_buffer = None
+while True:
+    if webrtc_ctx.audio_receiver:
+        try:
+            audio_frames = webrtc_ctx.audio_receiver.get_frames(timeout=1)
+        except queue.Empty:
+            logger.warning("Queue is empty. Abort.")
+            break
+
+        sound_chunk = pydub.AudioSegment.empty()
+        for audio_frame in audio_frames:
+            sound = pydub.AudioSegment(
+                data=audio_frame.to_ndarray().tobytes(),
+                sample_width=audio_frame.format.bytes,
+                frame_rate=audio_frame.sample_rate,
+                channels=len(audio_frame.layout.channels),
+            )
+            sound_chunk += sound
+
+        if len(sound_chunk) > 0:
+            if sound_window_buffer is None:
+                sound_window_buffer = pydub.AudioSegment.silent(
+                    duration=sound_window_len
+                )
+
+            sound_window_buffer += sound_chunk
+            if len(sound_window_buffer) > sound_window_len:
+                sound_window_buffer = sound_window_buffer[-sound_window_len:]
+
+        if sound_window_buffer:
+            # Ref: https://own-search-and-study.xyz/2017/10/27/python%E3%82%92%E4%BD%BF%E3%81%A3%E3%81%A6%E9%9F%B3%E5%A3%B0%E3%83%87%E3%83%BC%E3%82%BF%E3%81%8B%E3%82%89%E3%82%B9%E3%83%9A%E3%82%AF%E3%83%88%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%A0%E3%82%92%E4%BD%9C/  # noqa
+            sound_window_buffer = sound_window_buffer.set_channels(1)  # Stereo to mono
+            sample = np.array(sound_window_buffer.get_array_of_samples())
+
+            ax_time.cla()
+            times = (np.arange(-len(sample), 0)) / sound_window_buffer.frame_rate
+            ax_time.plot(times, sample)
+            ax_time.set_xlabel("Time")
+            ax_time.set_ylabel("Magnitude")
+
+            spec = np.fft.fft(sample)
+            freq = np.fft.fftfreq(sample.shape[0], 1.0 / sound_chunk.frame_rate)
+            freq = freq[: int(freq.shape[0] / 2)]
+            spec = spec[: int(spec.shape[0] / 2)]
+            spec[0] = spec[0] / 2
+
+            ax_freq.cla()
+            ax_freq.plot(freq, np.abs(spec))
+            ax_freq.set_xlabel("Frequency")
+            ax_freq.set_yscale("log")
+            ax_freq.set_ylabel("Magnitude")
+
+            fig_place.pyplot(fig)
+    else:
+        logger.warning("AudioReciver is not set. Abort.")
+        break

pages/11_programatic_control_playing.py

@@ -0,0 +1,13 @@
+"""A sample of controlling the playing state from Python."""
+
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+playing = st.checkbox("Playing", value=True)
+
+webrtc_streamer(
+    key="programatic_control",
+    desired_playing_state=playing,
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+)

pages/12_media_constraints_configs.py

@@ -0,0 +1,20 @@
+"""A sample to configure MediaStreamConstraints object"""
+
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+frame_rate = 5
+webrtc_streamer(
+    key="media-constraints",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    media_stream_constraints={
+        "video": {"frameRate": {"ideal": frame_rate}},
+    },
+    video_html_attrs={
+        "style": {"width": "50%", "margin": "0 auto", "border": "5px yellow solid"},
+        "controls": False,
+        "autoPlay": True,
+    },
+)
+st.write(f"The frame rate is set as {frame_rate}. Video style is changed.")

pages/13_ui_texts_customization.py

@@ -0,0 +1,15 @@
+from streamlit_webrtc import webrtc_streamer
+
+webrtc_streamer(
+    key="custom_ui_texts",
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    translations={
+        "start": "開始",
+        "stop": "停止",
+        "select_device": "デバイス選択",
+        "media_api_not_available": "Media APIが利用できない環境です",
+        "device_ask_permission": "メディアデバイスへのアクセスを許可してください",
+        "device_not_available": "メディアデバイスを利用できません",
+        "device_access_denied": "メディアデバイスへのアクセスが拒否されました",
+    },
+)

pages/1_object_detection.py

@@ -0,0 +1,172 @@
+"""Object detection demo with MobileNet SSD.
+This model and code are based on
+https://github.com/robmarkcole/object-detection-app
+"""
+
+import logging
+import queue
+from pathlib import Path
+from typing import List, NamedTuple
+
+import av
+import cv2
+import numpy as np
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+from sample_utils.download import download_file
+
+HERE = Path(__file__).parent
+ROOT = HERE.parent
+
+logger = logging.getLogger(__name__)
+
+
+MODEL_URL = "https://github.com/robmarkcole/object-detection-app/raw/master/model/MobileNetSSD_deploy.caffemodel"  # noqa: E501
+MODEL_LOCAL_PATH = ROOT / "./models/MobileNetSSD_deploy.caffemodel"
+PROTOTXT_URL = "https://github.com/robmarkcole/object-detection-app/raw/master/model/MobileNetSSD_deploy.prototxt.txt"  # noqa: E501
+PROTOTXT_LOCAL_PATH = ROOT / "./models/MobileNetSSD_deploy.prototxt.txt"
+
+CLASSES = [
+    "background",
+    "aeroplane",
+    "bicycle",
+    "bird",
+    "boat",
+    "bottle",
+    "bus",
+    "car",
+    "cat",
+    "chair",
+    "cow",
+    "diningtable",
+    "dog",
+    "horse",
+    "motorbike",
+    "person",
+    "pottedplant",
+    "sheep",
+    "sofa",
+    "train",
+    "tvmonitor",
+]
+
+
+@st.experimental_singleton  # type: ignore # See https://github.com/python/mypy/issues/7781, https://github.com/python/mypy/issues/12566  # noqa: E501
+def generate_label_colors():
+    return np.random.uniform(0, 255, size=(len(CLASSES), 3))
+
+
+COLORS = generate_label_colors()
+
+download_file(MODEL_URL, MODEL_LOCAL_PATH, expected_size=23147564)
+download_file(PROTOTXT_URL, PROTOTXT_LOCAL_PATH, expected_size=29353)
+
+DEFAULT_CONFIDENCE_THRESHOLD = 0.5
+
+
+class Detection(NamedTuple):
+    name: str
+    prob: float
+
+
+# Session-specific caching
+cache_key = "object_detection_dnn"
+if cache_key in st.session_state:
+    net = st.session_state[cache_key]
+else:
+    net = cv2.dnn.readNetFromCaffe(str(PROTOTXT_LOCAL_PATH), str(MODEL_LOCAL_PATH))
+    st.session_state[cache_key] = net
+
+streaming_placeholder = st.empty()
+
+confidence_threshold = st.slider(
+    "Confidence threshold", 0.0, 1.0, DEFAULT_CONFIDENCE_THRESHOLD, 0.05
+)
+
+
+def _annotate_image(image, detections):
+    # loop over the detections
+    (h, w) = image.shape[:2]
+    result: List[Detection] = []
+    for i in np.arange(0, detections.shape[2]):
+        confidence = detections[0, 0, i, 2]
+
+        if confidence > confidence_threshold:
+            # extract the index of the class label from the `detections`,
+            # then compute the (x, y)-coordinates of the bounding box for
+            # the object
+            idx = int(detections[0, 0, i, 1])
+            box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
+            (startX, startY, endX, endY) = box.astype("int")
+
+            name = CLASSES[idx]
+            result.append(Detection(name=name, prob=float(confidence)))
+
+            # display the prediction
+            label = f"{name}: {round(confidence * 100, 2)}%"
+            cv2.rectangle(image, (startX, startY), (endX, endY), COLORS[idx], 2)
+            y = startY - 15 if startY - 15 > 15 else startY + 15
+            cv2.putText(
+                image,
+                label,
+                (startX, y),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                0.5,
+                COLORS[idx],
+                2,
+            )
+    return image, result
+
+
+result_queue: queue.Queue = (
+    queue.Queue()
+)  # TODO: A general-purpose shared state object may be more useful.
+
+
+def callback(frame: av.VideoFrame) -> av.VideoFrame:
+    image = frame.to_ndarray(format="bgr24")
+    blob = cv2.dnn.blobFromImage(
+        cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
+    )
+    net.setInput(blob)
+    detections = net.forward()
+    annotated_image, result = _annotate_image(image, detections)
+
+    # NOTE: This `recv` method is called in another thread,
+    # so it must be thread-safe.
+    result_queue.put(result)  # TODO:
+
+    return av.VideoFrame.from_ndarray(annotated_image, format="bgr24")
+
+
+with streaming_placeholder.container():
+    webrtc_ctx = webrtc_streamer(
+        key="object-detection",
+        mode=WebRtcMode.SENDRECV,
+        rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+        video_frame_callback=callback,
+        media_stream_constraints={"video": True, "audio": False},
+        async_processing=True,
+    )
+
+if st.checkbox("Show the detected labels", value=True):
+    if webrtc_ctx.state.playing:
+        labels_placeholder = st.empty()
+        # NOTE: The video transformation with object detection and
+        # this loop displaying the result labels are running
+        # in different threads asynchronously.
+        # Then the rendered video frames and the labels displayed here
+        # are not strictly synchronized.
+        while True:
+            try:
+                result = result_queue.get(timeout=1.0)
+            except queue.Empty:
+                result = None
+            labels_placeholder.table(result)
+
+st.markdown(
+    "This demo uses a model and code from "
+    "https://github.com/robmarkcole/object-detection-app. "
+    "Many thanks to the project."
+)

pages/2_opencv_filters.py

@@ -0,0 +1,62 @@
+"""Video transforms with OpenCV"""
+
+import av
+import cv2
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+_type = st.radio("Select transform type", ("noop", "cartoon", "edges", "rotate"))
+
+
+def callback(frame: av.VideoFrame) -> av.VideoFrame:
+    img = frame.to_ndarray(format="bgr24")
+
+    if _type == "noop":
+        pass
+    elif _type == "cartoon":
+        # prepare color
+        img_color = cv2.pyrDown(cv2.pyrDown(img))
+        for _ in range(6):
+            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
+        img_color = cv2.pyrUp(cv2.pyrUp(img_color))
+
+        # prepare edges
+        img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+        img_edges = cv2.adaptiveThreshold(
+            cv2.medianBlur(img_edges, 7),
+            255,
+            cv2.ADAPTIVE_THRESH_MEAN_C,
+            cv2.THRESH_BINARY,
+            9,
+            2,
+        )
+        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
+
+        # combine color and edges
+        img = cv2.bitwise_and(img_color, img_edges)
+    elif _type == "edges":
+        # perform edge detection
+        img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
+    elif _type == "rotate":
+        # rotate image
+        rows, cols, _ = img.shape
+        M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
+        img = cv2.warpAffine(img, M, (cols, rows))
+
+    return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+
+webrtc_streamer(
+    key="opencv-filter",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    video_frame_callback=callback,
+    media_stream_constraints={"video": True, "audio": False},
+    async_processing=True,
+)
+
+st.markdown(
+    "This demo is based on "
+    "https://github.com/aiortc/aiortc/blob/2362e6d1f0c730a0f8c387bbea76546775ad2fe8/examples/server/server.py#L34. "  # noqa: E501
+    "Many thanks to the project."
+)

pages/3_audio_filter.py

@@ -0,0 +1,38 @@
+import av
+import numpy as np
+import pydub
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+gain = st.slider("Gain", -10.0, +20.0, 1.0, 0.05)
+
+
+def process_audio(frame: av.AudioFrame) -> av.AudioFrame:
+    raw_samples = frame.to_ndarray()
+    sound = pydub.AudioSegment(
+        data=raw_samples.tobytes(),
+        sample_width=frame.format.bytes,
+        frame_rate=frame.sample_rate,
+        channels=len(frame.layout.channels),
+    )
+
+    sound = sound.apply_gain(gain)
+
+    # Ref: https://github.com/jiaaro/pydub/blob/master/API.markdown#audiosegmentget_array_of_samples  # noqa
+    channel_sounds = sound.split_to_mono()
+    channel_samples = [s.get_array_of_samples() for s in channel_sounds]
+    new_samples: np.ndarray = np.array(channel_samples).T
+    new_samples = new_samples.reshape(raw_samples.shape)
+
+    new_frame = av.AudioFrame.from_ndarray(new_samples, layout=frame.layout.name)
+    new_frame.sample_rate = frame.sample_rate
+    return new_frame
+
+
+webrtc_streamer(
+    key="audio-filter",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    audio_frame_callback=process_audio,
+    async_processing=True,
+)

pages/4_delayed_echo.py

@@ -0,0 +1,41 @@
+import asyncio
+import logging
+from typing import List
+
+import av
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+logger = logging.getLogger(__name__)
+
+
+delay = st.slider("Delay", 0.0, 5.0, 1.0, 0.05)
+
+
+async def queued_video_frames_callback(
+    frames: List[av.VideoFrame],
+) -> List[av.VideoFrame]:
+    logger.debug("Delay: %f", delay)
+    # A standalone `await ...` is interpreted as an expression and
+    # the Streamlit magic's target, which leads implicit calls of `st.write`.
+    # To prevent it, fix it as `_ = await ...`, a statement.
+    # See https://discuss.streamlit.io/t/issue-with-asyncio-run-in-streamlit/7745/15
+    await asyncio.sleep(delay)
+    return frames
+
+
+async def queued_audio_frames_callback(
+    frames: List[av.AudioFrame],
+) -> List[av.AudioFrame]:
+    await asyncio.sleep(delay)
+    return frames
+
+
+webrtc_streamer(
+    key="delay",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    queued_video_frames_callback=queued_video_frames_callback,
+    queued_audio_frames_callback=queued_audio_frames_callback,
+    async_processing=True,
+)

pages/5_fork_multi_outputs.py

@@ -0,0 +1,105 @@
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
+
+from typing import cast
+
+import av
+import cv2
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+st.markdown(
+    """
+Fork one input to multiple outputs with different video filters.
+"""
+)
+
+VideoFilterType = Literal["noop", "cartoon", "edges", "rotate"]
+
+
+def make_video_frame_callback(_type: VideoFilterType):
+    def callback(frame: av.VideoFrame) -> av.VideoFrame:
+        img = frame.to_ndarray(format="bgr24")
+
+        if _type == "noop":
+            pass
+        elif _type == "cartoon":
+            # prepare color
+            img_color = cv2.pyrDown(cv2.pyrDown(img))
+            for _ in range(6):
+                img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
+            img_color = cv2.pyrUp(cv2.pyrUp(img_color))
+
+            # prepare edges
+            img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+            img_edges = cv2.adaptiveThreshold(
+                cv2.medianBlur(img_edges, 7),
+                255,
+                cv2.ADAPTIVE_THRESH_MEAN_C,
+                cv2.THRESH_BINARY,
+                9,
+                2,
+            )
+            img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
+
+            # combine color and edges
+            img = cv2.bitwise_and(img_color, img_edges)
+        elif _type == "edges":
+            # perform edge detection
+            img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
+        elif _type == "rotate":
+            # rotate image
+            rows, cols, _ = img.shape
+            M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
+            img = cv2.warpAffine(img, M, (cols, rows))
+
+        return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+    return callback
+
+
+COMMON_RTC_CONFIG = {"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]}
+
+st.header("Input")
+ctx = webrtc_streamer(
+    key="loopback",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)
+
+st.header("Forked output 1")
+filter1_type = st.radio(
+    "Select transform type",
+    ("noop", "cartoon", "edges", "rotate"),
+    key="fork-filter1-type",
+)
+callback = make_video_frame_callback(cast(VideoFilterType, filter1_type))
+webrtc_streamer(
+    key="filter1",
+    mode=WebRtcMode.RECVONLY,
+    video_frame_callback=callback,
+    source_video_track=ctx.output_video_track,
+    desired_playing_state=ctx.state.playing,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)
+
+st.header("Forked output 2")
+filter2_type = st.radio(
+    "Select transform type",
+    ("noop", "cartoon", "edges", "rotate"),
+    key="fork-filter2-type",
+)
+callback = make_video_frame_callback(cast(VideoFilterType, filter2_type))
+webrtc_streamer(
+    key="filter2",
+    mode=WebRtcMode.RECVONLY,
+    video_frame_callback=callback,
+    source_video_track=ctx.output_video_track,
+    desired_playing_state=ctx.state.playing,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)

pages/6_mix_multi_inputs.py

@@ -0,0 +1,182 @@
+import math
+from typing import List
+
+try:
+    from typing import Literal, cast
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
+
+import av
+import cv2
+import numpy as np
+import streamlit as st
+from streamlit_webrtc import (
+    WebRtcMode,
+    create_mix_track,
+    create_process_track,
+    webrtc_streamer,
+)
+
+st.markdown(
+    """
+Mix multiple inputs with different video filters into one stream.
+"""
+)
+
+VideoFilterType = Literal["noop", "cartoon", "edges", "rotate"]
+
+
+def make_video_frame_callback(_type: VideoFilterType):
+    def callback(frame: av.VideoFrame) -> av.VideoFrame:
+        img = frame.to_ndarray(format="bgr24")
+
+        if _type == "noop":
+            pass
+        elif _type == "cartoon":
+            # prepare color
+            img_color = cv2.pyrDown(cv2.pyrDown(img))
+            for _ in range(6):
+                img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
+            img_color = cv2.pyrUp(cv2.pyrUp(img_color))
+
+            # prepare edges
+            img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+            img_edges = cv2.adaptiveThreshold(
+                cv2.medianBlur(img_edges, 7),
+                255,
+                cv2.ADAPTIVE_THRESH_MEAN_C,
+                cv2.THRESH_BINARY,
+                9,
+                2,
+            )
+            img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
+
+            # combine color and edges
+            img = cv2.bitwise_and(img_color, img_edges)
+        elif _type == "edges":
+            # perform edge detection
+            img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
+        elif _type == "rotate":
+            # rotate image
+            rows, cols, _ = img.shape
+            M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
+            img = cv2.warpAffine(img, M, (cols, rows))
+
+        return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+    return callback
+
+
+def mixer_callback(frames: List[av.VideoFrame]) -> av.VideoFrame:
+    buf_w = 640
+    buf_h = 480
+    buffer = np.zeros((buf_h, buf_w, 3), dtype=np.uint8)
+
+    n_inputs = len(frames)
+
+    n_cols = math.ceil(math.sqrt(n_inputs))
+    n_rows = math.ceil(n_inputs / n_cols)
+    grid_w = buf_w // n_cols
+    grid_h = buf_h // n_rows
+
+    for i in range(n_inputs):
+        frame = frames[i]
+        if frame is None:
+            continue
+
+        grid_x = (i % n_cols) * grid_w
+        grid_y = (i // n_cols) * grid_h
+
+        img = frame.to_ndarray(format="bgr24")
+        src_h, src_w = img.shape[0:2]
+
+        aspect_ratio = src_w / src_h
+
+        window_w = min(grid_w, int(grid_h * aspect_ratio))
+        window_h = min(grid_h, int(window_w / aspect_ratio))
+
+        window_offset_x = (grid_w - window_w) // 2
+        window_offset_y = (grid_h - window_h) // 2
+
+        window_x0 = grid_x + window_offset_x
+        window_y0 = grid_y + window_offset_y
+        window_x1 = window_x0 + window_w
+        window_y1 = window_y0 + window_h
+
+        buffer[window_y0:window_y1, window_x0:window_x1, :] = cv2.resize(
+            img, (window_w, window_h)
+        )
+
+    new_frame = av.VideoFrame.from_ndarray(buffer, format="bgr24")
+
+    return new_frame
+
+
+COMMON_RTC_CONFIG = {"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]}
+
+st.header("Input 1")
+input1_ctx = webrtc_streamer(
+    key="input1_ctx",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)
+filter1_type = st.radio(
+    "Select transform type",
+    ("noop", "cartoon", "edges", "rotate"),
+    key="mix-filter1-type",
+)
+callback = make_video_frame_callback(cast(VideoFilterType, filter1_type))
+input1_video_process_track = None
+if input1_ctx.output_video_track:
+    input1_video_process_track = create_process_track(
+        input_track=input1_ctx.output_video_track,
+        frame_callback=callback,
+    )
+
+st.header("Input 2")
+input2_ctx = webrtc_streamer(
+    key="input2_ctx",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)
+filter2_type = st.radio(
+    "Select transform type",
+    ("noop", "cartoon", "edges", "rotate"),
+    key="mix-filter2-type",
+)
+callback = make_video_frame_callback(cast(VideoFilterType, filter2_type))
+input2_video_process_track = None
+if input2_ctx.output_video_track:
+    input2_video_process_track = create_process_track(
+        input_track=input2_ctx.output_video_track, frame_callback=callback
+    )
+
+st.header("Input 3 (no filter)")
+input3_ctx = webrtc_streamer(
+    key="input3_ctx",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    media_stream_constraints={"video": True, "audio": False},
+)
+
+st.header("Mixed output")
+mix_track = create_mix_track(kind="video", mixer_callback=mixer_callback, key="mix")
+mix_ctx = webrtc_streamer(
+    key="mix",
+    mode=WebRtcMode.RECVONLY,
+    rtc_configuration=COMMON_RTC_CONFIG,
+    source_video_track=mix_track,
+    desired_playing_state=input1_ctx.state.playing
+    or input2_ctx.state.playing
+    or input3_ctx.state.playing,
+)
+
+if mix_ctx.source_video_track and input1_video_process_track:
+    mix_ctx.source_video_track.add_input_track(input1_video_process_track)
+if mix_ctx.source_video_track and input2_video_process_track:
+    mix_ctx.source_video_track.add_input_track(input2_video_process_track)
+if mix_ctx.source_video_track and input3_ctx.output_video_track:
+    # Input3 is sourced without any filter.
+    mix_ctx.source_video_track.add_input_track(input3_ctx.output_video_track)

pages/7_record.py

@@ -0,0 +1,65 @@
+import uuid
+from pathlib import Path
+
+import av
+import cv2
+import streamlit as st
+from aiortc.contrib.media import MediaRecorder
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+
+def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
+    img = frame.to_ndarray(format="bgr24")
+
+    # perform edge detection
+    img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
+
+    return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+
+RECORD_DIR = Path("./records")
+RECORD_DIR.mkdir(exist_ok=True)
+
+
+def app():
+    if "prefix" not in st.session_state:
+        st.session_state["prefix"] = str(uuid.uuid4())
+    prefix = st.session_state["prefix"]
+    in_file = RECORD_DIR / f"{prefix}_input.flv"
+    out_file = RECORD_DIR / f"{prefix}_output.flv"
+
+    def in_recorder_factory() -> MediaRecorder:
+        return MediaRecorder(
+            str(in_file), format="flv"
+        )  # HLS does not work. See https://github.com/aiortc/aiortc/issues/331
+
+    def out_recorder_factory() -> MediaRecorder:
+        return MediaRecorder(str(out_file), format="flv")
+
+    webrtc_streamer(
+        key="record",
+        mode=WebRtcMode.SENDRECV,
+        rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+        media_stream_constraints={
+            "video": True,
+            "audio": True,
+        },
+        video_frame_callback=video_frame_callback,
+        in_recorder_factory=in_recorder_factory,
+        out_recorder_factory=out_recorder_factory,
+    )
+
+    if in_file.exists():
+        with in_file.open("rb") as f:
+            st.download_button(
+                "Download the recorded video without video filter", f, "input.flv"
+            )
+    if out_file.exists():
+        with out_file.open("rb") as f:
+            st.download_button(
+                "Download the recorded video with video filter", f, "output.flv"
+            )
+
+
+if __name__ == "__main__":
+    app()

pages/8_media_files_streaming.py

@@ -0,0 +1,128 @@
+"""Media streamings"""
+import logging
+from pathlib import Path
+from typing import Dict, Optional, cast
+
+import av
+import cv2
+import streamlit as st
+from aiortc.contrib.media import MediaPlayer
+from streamlit_webrtc import WebRtcMode, WebRtcStreamerContext, webrtc_streamer
+
+from sample_utils.download import download_file
+
+HERE = Path(__file__).parent
+ROOT = HERE.parent
+
+logger = logging.getLogger(__name__)
+
+
+MEDIAFILES: Dict[str, Dict] = {
+    "big_buck_bunny_720p_2mb.mp4 (local)": {
+        "url": "https://sample-videos.com/video123/mp4/720/big_buck_bunny_720p_2mb.mp4",  # noqa: E501
+        "local_file_path": ROOT / "data/big_buck_bunny_720p_2mb.mp4",
+        "type": "video",
+    },
+    "big_buck_bunny_720p_10mb.mp4 (local)": {
+        "url": "https://sample-videos.com/video123/mp4/720/big_buck_bunny_720p_10mb.mp4",  # noqa: E501
+        "local_file_path": ROOT / "data/big_buck_bunny_720p_10mb.mp4",
+        "type": "video",
+    },
+    "file_example_MP3_700KB.mp3 (local)": {
+        "url": "https://file-examples-com.github.io/uploads/2017/11/file_example_MP3_700KB.mp3",  # noqa: E501
+        "local_file_path": ROOT / "data/file_example_MP3_700KB.mp3",
+        "type": "audio",
+    },
+    "file_example_MP3_5MG.mp3 (local)": {
+        "url": "https://file-examples-com.github.io/uploads/2017/11/file_example_MP3_5MG.mp3",  # noqa: E501
+        "local_file_path": ROOT / "data/file_example_MP3_5MG.mp3",
+        "type": "audio",
+    },
+    "rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov": {
+        "url": "rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov",
+        "type": "video",
+    },
+}
+media_file_label = st.radio("Select a media source to stream", tuple(MEDIAFILES.keys()))
+media_file_info = MEDIAFILES[cast(str, media_file_label)]
+if "local_file_path" in media_file_info:
+    download_file(media_file_info["url"], media_file_info["local_file_path"])
+
+
+def create_player():
+    if "local_file_path" in media_file_info:
+        return MediaPlayer(str(media_file_info["local_file_path"]))
+    else:
+        return MediaPlayer(media_file_info["url"])
+
+    # NOTE: To stream the video from webcam, use the code below.
+    # return MediaPlayer(
+    #     "1:none",
+    #     format="avfoundation",
+    #     options={"framerate": "30", "video_size": "1280x720"},
+    # )
+
+
+key = f"media-streaming-{media_file_label}"
+ctx: Optional[WebRtcStreamerContext] = st.session_state.get(key)
+if media_file_info["type"] == "video" and ctx and ctx.state.playing:
+    _type = st.radio("Select transform type", ("noop", "cartoon", "edges", "rotate"))
+else:
+    _type = "noop"
+
+
+def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
+    img = frame.to_ndarray(format="bgr24")
+
+    if _type == "noop":
+        pass
+    elif _type == "cartoon":
+        # prepare color
+        img_color = cv2.pyrDown(cv2.pyrDown(img))
+        for _ in range(6):
+            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
+        img_color = cv2.pyrUp(cv2.pyrUp(img_color))
+
+        # prepare edges
+        img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+        img_edges = cv2.adaptiveThreshold(
+            cv2.medianBlur(img_edges, 7),
+            255,
+            cv2.ADAPTIVE_THRESH_MEAN_C,
+            cv2.THRESH_BINARY,
+            9,
+            2,
+        )
+        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
+
+        # combine color and edges
+        img = cv2.bitwise_and(img_color, img_edges)
+    elif _type == "edges":
+        # perform edge detection
+        img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
+    elif _type == "rotate":
+        # rotate image
+        rows, cols, _ = img.shape
+        M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
+        img = cv2.warpAffine(img, M, (cols, rows))
+
+    return av.VideoFrame.from_ndarray(img, format="bgr24")
+
+
+webrtc_streamer(
+    key=key,
+    mode=WebRtcMode.RECVONLY,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    media_stream_constraints={
+        "video": media_file_info["type"] == "video",
+        "audio": media_file_info["type"] == "audio",
+    },
+    player_factory=create_player,
+    video_frame_callback=video_frame_callback,
+)
+
+st.markdown(
+    "The video filter in this demo is based on "
+    "https://github.com/aiortc/aiortc/blob/2362e6d1f0c730a0f8c387bbea76546775ad2fe8/examples/server/server.py#L34. "  # noqa: E501
+    "Many thanks to the project."
+)

pages/9_sendonly_video.py

@@ -0,0 +1,34 @@
+"""A sample to use WebRTC in sendonly mode to transfer frames
+from the browser to the server and to render frames via `st.image`."""
+
+import logging
+import queue
+
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+logger = logging.getLogger(__name__)
+
+
+webrtc_ctx = webrtc_streamer(
+    key="video-sendonly",
+    mode=WebRtcMode.SENDONLY,
+    rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+    media_stream_constraints={"video": True},
+)
+
+image_place = st.empty()
+
+while True:
+    if webrtc_ctx.video_receiver:
+        try:
+            video_frame = webrtc_ctx.video_receiver.get_frame(timeout=1)
+        except queue.Empty:
+            logger.warning("Queue is empty. Abort.")
+            break
+
+        img_rgb = video_frame.to_ndarray(format="rgb24")
+        image_place.image(img_rgb)
+    else:
+        logger.warning("AudioReciver is not set. Abort.")
+        break

sample_utils/download.py

@@ -0,0 +1,50 @@
+import urllib.request
+from pathlib import Path
+
+import streamlit as st
+
+
+# This code is based on https://github.com/streamlit/demo-self-driving/blob/230245391f2dda0cb464008195a470751c01770b/streamlit_app.py#L48  # noqa: E501
+def download_file(url, download_to: Path, expected_size=None):
+    # Don't download the file twice.
+    # (If possible, verify the download using the file length.)
+    if download_to.exists():
+        if expected_size:
+            if download_to.stat().st_size == expected_size:
+                return
+        else:
+            st.info(f"{url} is already downloaded.")
+            if not st.button("Download again?"):
+                return
+
+    download_to.parent.mkdir(parents=True, exist_ok=True)
+
+    # These are handles to two visual elements to animate.
+    weights_warning, progress_bar = None, None
+    try:
+        weights_warning = st.warning("Downloading %s..." % url)
+        progress_bar = st.progress(0)
+        with open(download_to, "wb") as output_file:
+            with urllib.request.urlopen(url) as response:
+                length = int(response.info()["Content-Length"])
+                counter = 0.0
+                MEGABYTES = 2.0 ** 20.0
+                while True:
+                    data = response.read(8192)
+                    if not data:
+                        break
+                    counter += len(data)
+                    output_file.write(data)
+
+                    # We perform animation by overwriting the elements.
+                    weights_warning.warning(
+                        "Downloading %s... (%6.2f/%6.2f MB)"
+                        % (url, counter / MEGABYTES, length / MEGABYTES)
+                    )
+                    progress_bar.progress(min(counter / length, 1.0))
+    # Finally, we remove these visual elements by calling .empty().
+    finally:
+        if weights_warning is not None:
+            weights_warning.empty()
+        if progress_bar is not None:
+            progress_bar.empty()