Asr Streaming Conformer Librispeech

Decoding from a live stream using ffmpeg (BBC Radio 4):

python3 asr.py http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_radio_fourfm/bbc_radio_fourfm.isml/bbc_radio_fourfm-audio%3d96000.norewind.m3u8 --model-source=speechbrain/asr-streaming-conformer-librispeech --device=cpu -v

Decoding from a file:

python3 asr.py some-english-speech.wav --model-source=speechbrain/asr-streaming-conformer-librispeech --device=cpu -v

from argparse import ArgumentParser
import logging

parser = ArgumentParser()
parser.add_argument("audio_path")
parser.add_argument("--model-source", required=True)
parser.add_argument("--device", default="cpu")
parser.add_argument("--ip", default="127.0.0.1")
parser.add_argument("--port", default=9431)
parser.add_argument("--chunk-size", default=24, type=int)
parser.add_argument("--left-context-chunks", default=4, type=int)
parser.add_argument("--num-threads", default=None, type=int)
parser.add_argument("--verbose", "-v", default=False, action="store_true")
args = parser.parse_args()

if args.verbose:
    logging.getLogger().setLevel(logging.INFO)

logging.info("Loading libraries")

from speechbrain.inference.ASR import StreamingASR
from speechbrain.utils.dynamic_chunk_training import DynChunkTrainConfig
import torch

device = args.device

if args.num_threads is not None:
    torch.set_num_threads(args.num_threads)

logging.info(f"Loading model from \"{args.model_source}\" onto device {device}")

asr = StreamingASR.from_hparams(args.model_source, run_opts={"device": device})
config = DynChunkTrainConfig(args.chunk_size, args.left_context_chunks)

logging.info(f"Starting stream from URI \"{args.audio_path}\"")

for text_chunk in asr.transcribe_file_streaming(args.audio_path, config):
    print(text_chunk, flush=True, end="")

We want to optimize some things around the model before we create a proper HuggingFace space demonstrating live streaming on CPU.

In the mean time, this is a simple hacky demo of live ASR in the browser using Gradio's live microphone streaming feature.

If you run this, please note:

• Modern browsers refuse to stream microphone input over an untrusted connection (plain HTTP), unless it is localhost. If you are running this on a remote server, you could use SSH port forwarding to expose the remote's port on your machine.
• Streaming using Gradio on Firefox seems to cause some issues. Chromium - based browsers seem to behave better.

Run using:

python3 gradio-asr.py --model-source speechbrain/asr-streaming-conformer-librispeech --ip=localhost --device=cpu

from argparse import ArgumentParser
from dataclasses import dataclass
import logging

parser = ArgumentParser()
parser.add_argument("--model-source", required=True)
parser.add_argument("--device", default="cpu")
parser.add_argument("--ip", default="127.0.0.1")
parser.add_argument("--port", default=9431)
parser.add_argument("--chunk-size", default=24, type=int)
parser.add_argument("--left-context-chunks", default=4, type=int)
parser.add_argument("--num-threads", default=None, type=int)
parser.add_argument("--verbose", "-v", default=False, action="store_true")
args = parser.parse_args()

if args.verbose:
    logging.getLogger().setLevel(logging.INFO)

logging.info("Loading libraries")

from speechbrain.inference.ASR import StreamingASR, ASRStreamingContext
from speechbrain.utils.dynamic_chunk_training import DynChunkTrainConfig
import torch
import gradio as gr
import torchaudio
import numpy as np

device = args.device

if args.num_threads is not None:
    torch.set_num_threads(args.num_threads)

logging.info(f"Loading model from \"{args.model_source}\" onto device {device}")

asr = StreamingASR.from_hparams(args.model_source, run_opts={"device": device})
config = DynChunkTrainConfig(args.chunk_size, args.left_context_chunks)

@dataclass
class GradioStreamingContext:
    context: ASRStreamingContext
    chunk_size: int
    waveform_buffer: torch.Tensor
    decoded_text: str

def transcribe(stream, new_chunk):
    sr, y = new_chunk

    y = y.astype(np.float32)
    y = torch.tensor(y, dtype=torch.float32, device=device)
    y /= max(1, torch.max(torch.abs(y)).item())  # norm by max abs() within chunk & avoid NaN
    if len(y.shape) > 1:
        y = torch.mean(y, dim=1)  # downmix to mono

    # HACK: we are making poor use of the resampler across chunk boundaries
    # which may degrade accuracy.
    # NOTE: we should also absolutely avoid recreating a resampler every time
    resampler = torchaudio.transforms.Resample(orig_freq=sr, new_freq=asr.audio_normalizer.sample_rate).to(device)
    y = resampler(y)  # janky resample (probably to 16kHz)


    if stream is None:
        stream = GradioStreamingContext(
            context=asr.make_streaming_context(config),
            chunk_size=asr.get_chunk_size_frames(config),
            waveform_buffer=y,
            decoded_text="",
        )
    else:
        stream.waveform_buffer = torch.concat((stream.waveform_buffer, y))

    while stream.waveform_buffer.size(0) > stream.chunk_size:
        chunk = stream.waveform_buffer[:stream.chunk_size]
        stream.waveform_buffer = stream.waveform_buffer[stream.chunk_size:]

        # fake batch dim
        chunk = chunk.unsqueeze(0)

        # list of transcribed strings, of size 1 because the batch size is 1
        with torch.no_grad():
            transcribed = asr.transcribe_chunk(stream.context, chunk)
        stream.decoded_text += transcribed[0]

    return stream, stream.decoded_text

# NOTE: latency seems relatively high, which may be due to this:
# https://github.com/gradio-app/gradio/issues/6526

demo = gr.Interface(
    transcribe,
    ["state", gr.Audio(sources=["microphone"], streaming=True)],
    ["state", "text"],
    live=True,
)

demo.launch(server_name=args.ip, server_port=args.port)