%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%23061b40;%20}%20.st1%20{%20fill:%20%23306af1;%20}%20.st2%20{%20fill:%20%235ce5cf;%20}%20%3c/style%3e%3c/defs%3e%3cg%3e%3cpath%20class='st0'%20d='M55,10.5h9v3h-9v9h12V7.5h-12v3ZM64,19.5h-6v-3h6v3Z'/%3e%3cpolygon%20class='st0'%20points='69%2016.5%2078%2016.5%2078%2019.5%2069%2019.5%2069%2022.5%2081%2022.5%2081%2013.5%2072%2013.5%2072%2010.5%2081%2010.5%2081%207.5%2069%207.5%2069%2016.5'/%3e%3cpolygon%20class='st0'%20points='95%2010.5%2095%207.5%2083%207.5%2083%2022.5%2095%2022.5%2095%2019.5%2086%2019.5%2086%2016.5%2095%2016.5%2095%2013.5%2086%2013.5%2086%2010.5%2095%2010.5'/%3e%3cpath%20class='st0'%20d='M40,1.5v21h11.6l1.4-1.4v-7.6h0c0,0-1.4-1.5-1.4-1.5l1.4-1.4V2.9l-1.4-1.4h-11.6ZM50,19.5h-7v-6h7v6ZM50,10.5h-7v-6h7v6Z'/%3e%3c/g%3e%3cpath%20class='st1'%20d='M23.1,24L14.7,4.8l-4.9,11.2h3.8l-1.8,4H3.3L12.1,0H2C.9,0,0,.9,0,2v20c0,1.1.9,2,2,2h21.1Z'/%3e%3cpath%20class='st2'%20d='M34,0h-16.8l10.6,24h6.2c1.1,0,2-.9,2-2V2C36,.9,35.1,0,34,0ZM32.5,20h-4V4h4v16Z'/%3e%3c/svg%3e)
EN
Wav2vec2 Large Xlsr Vietnamese
Vietnamese automatic speech recognition model fine-tuned from facebook/wav2vec2-large-xlsr-53
Downloads 22
Release Time : 3/2/2022
Model Overview
This is an optimized automatic speech recognition (ASR) model for Vietnamese, based on the XLSR Wav2Vec2 architecture, fine-tuned using Common Voice, FOSD, and VIVOS datasets.
Model Features
Multi-dataset fine-tuning
Trained using three Vietnamese datasets (Common Voice, FOSD, and VIVOS) to enhance model adaptability
16kHz sampling rate support
Optimized for processing speech input at 16kHz sampling rate
No language model required
Can be used directly without additional language model support
Model Capabilities
Vietnamese speech recognition
Automatic speech-to-text conversion
Use Cases
Speech transcription
Vietnamese speech transcription
Convert Vietnamese speech content into text
Achieves 49.59% WER on Common Voice Vietnamese test set
Voice assistants
Vietnamese voice command recognition
For voice command recognition in Vietnamese voice assistants or smart home devices
🚀 Wav2Vec2-Large-XLSR-53-Vietnamese
This model is a fine - tuned version of facebook/wav2vec2-large-xlsr-53 on Vietnamese. It utilizes datasets such as Common Voice, FOSD and VIVOS. When using this model, ensure that your speech input is sampled at 16kHz.
📦 Installation
No specific installation steps are provided in the original document, so this section is skipped.
💻 Usage Examples
Basic Usage
import torch
import torchaudio
from datasets import load_dataset
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
ENCODER = {
"ia ": "iê ",
"ìa ": "iề ",
"ía ": "iế ",
"ỉa ": "iể ",
"ĩa ": "iễ ",
"ịa ": "iệ ",
"ya ": "yê ",
"ỳa ": "yề ",
"ýa ": "yế ",
"ỷa ": "yể ",
"ỹa ": "yễ ",
"ỵa ": "yệ ",
"ua ": "uô ",
"ùa ": "uồ ",
"úa ": "uố ",
"ủa ": "uổ ",
"ũa ": "uỗ ",
"ụa ": "uộ ",
"ưa ": "ươ ",
"ừa ": "ườ ",
"ứa ": "ướ ",
"ửa ": "ưở ",
"ữa ": "ưỡ ",
"ựa ": "ượ ",
"ke": "ce",
"kè": "cè",
"ké": "cé",
"kẻ": "cẻ",
"kẽ": "cẽ",
"kẹ": "cẹ",
"kê": "cê",
"kề": "cề",
"kế": "cế",
"kể": "cể",
"kễ": "cễ",
"kệ": "cệ",
"ki": "ci",
"kì": "cì",
"kí": "cí",
"kỉ": "cỉ",
"kĩ": "cĩ",
"kị": "cị",
"ky": "cy",
"kỳ": "cỳ",
"ký": "cý",
"kỷ": "cỷ",
"kỹ": "cỹ",
"kỵ": "cỵ",
"ghe": "ge",
"ghè": "gè",
"ghé": "gé",
"ghẻ": "gẻ",
"ghẽ": "gẽ",
"ghẹ": "gẹ",
"ghê": "gê",
"ghề": "gề",
"ghế": "gế",
"ghể": "gể",
"ghễ": "gễ",
"ghệ": "gệ",
"ngh": "\x80",
"uyê": "\x96",
"uyề": "\x97",
"uyế": "\x98",
"uyể": "\x99",
"uyễ": "\x9a",
"uyệ": "\x9b",
"ng": "\x81",
"ch": "\x82",
"gh": "\x83",
"nh": "\x84",
"gi": "\x85",
"ph": "\x86",
"kh": "\x87",
"th": "\x88",
"tr": "\x89",
"uy": "\x8a",
"uỳ": "\x8b",
"uý": "\x8c",
"uỷ": "\x8d",
"uỹ": "\x8e",
"uỵ": "\x8f",
"iê": "\x90",
"iề": "\x91",
"iế": "\x92",
"iể": "\x93",
"iễ": "\x94",
"iệ": "\x95",
"uô": "\x9c",
"uồ": "\x9d",
"uố": "\x9e",
"uổ": "\x9f",
"uỗ": "\xa0",
"uộ": "\xa1",
"ươ": "\xa2",
"ườ": "\xa3",
"ướ": "\xa4",
"ưở": "\xa5",
"ưỡ": "\xa6",
"ượ": "\xa7",
}
def decode_string(x):
for k, v in list(reversed(list(ENCODER.items()))):
x = x.replace(v, k)
return x
test_dataset = load_dataset("common_voice", "vi", split="test[:2%]")
processor = Wav2Vec2Processor.from_pretrained("Nhut/wav2vec2-large-xlsr-vietnamese")
model = Wav2Vec2ForCTC.from_pretrained("Nhut/wav2vec2-large-xlsr-vietnamese")
resampler = torchaudio.transforms.Resample(48_000, 16_000)
# Preprocessing the datasets.
# We need to read the aduio files as arrays
def speech_file_to_array_fn(batch):
speech_array, sampling_rate = torchaudio.load(batch["path"])
batch["speech"] = resampler(speech_array).squeeze().numpy()
return batch
test_dataset = test_dataset.map(speech_file_to_array_fn)
inputs = processor(test_dataset["speech"][:2], sampling_rate=16_000, return_tensors="pt", padding=True)
with torch.no_grad():
logits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits
predicted_ids = torch.argmax(logits, dim=-1)
print("Prediction:", [decode_string(x) for x in processor.batch_decode(predicted_ids)])
print("Reference:", test_dataset["sentence"][:2])
Advanced Usage
import torch
import torchaudio
from datasets import load_dataset, load_metric
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
import re
ENCODER = {
"ia ": "iê ",
"ìa ": "iề ",
"ía ": "iế ",
"ỉa ": "iể ",
"ĩa ": "iễ ",
"ịa ": "iệ ",
"ya ": "yê ",
"ỳa ": "yề ",
"ýa ": "yế ",
"ỷa ": "yể ",
"ỹa ": "yễ ",
"ỵa ": "yệ ",
"ua ": "uô ",
"ùa ": "uồ ",
"úa ": "uố ",
"ủa ": "uổ ",
"ũa ": "uỗ ",
"ụa ": "uộ ",
"ưa ": "ươ ",
"ừa ": "ườ ",
"ứa ": "ướ ",
"ửa ": "ưở ",
"ữa ": "ưỡ ",
"ựa ": "ượ ",
"ke": "ce",
"kè": "cè",
"ké": "cé",
"kẻ": "cẻ",
"kẽ": "cẽ",
"kẹ": "cẹ",
"kê": "cê",
"kề": "cề",
"kế": "cế",
"kể": "cể",
"kễ": "cễ",
"kệ": "cệ",
"ki": "ci",
"kì": "cì",
"kí": "cí",
"kỉ": "cỉ",
"kĩ": "cĩ",
"kị": "cị",
"ky": "cy",
"kỳ": "cỳ",
"ký": "cý",
"kỷ": "cỷ",
"kỹ": "cỹ",
"kỵ": "cỵ",
"ghe": "ge",
"ghè": "gè",
"ghé": "gé",
"ghẻ": "gẻ",
"ghẽ": "gẽ",
"ghẹ": "gẹ",
"ghê": "gê",
"ghề": "gề",
"ghế": "gế",
"ghể": "gể",
"ghễ": "gễ",
"ghệ": "gệ",
"ngh": "\x80",
"uyê": "\x96",
"uyề": "\x97",
"uyế": "\x98",
"uyể": "\x99",
"uyễ": "\x9a",
"uyệ": "\x9b",
"ng": "\x81",
"ch": "\x82",
"gh": "\x83",
"nh": "\x84",
"gi": "\x85",
"ph": "\x86",
"kh": "\x87",
"th": "\x88",
"tr": "\x89",
"uy": "\x8a",
"uỳ": "\x8b",
"uý": "\x8c",
"uỷ": "\x8d",
"uỹ": "\x8e",
"uỵ": "\x8f",
"iê": "\x90",
"iề": "\x91",
"iế": "\x92",
"iể": "\x93",
"iễ": "\x94",
"iệ": "\x95",
"uô": "\x9c",
"uồ": "\x9d",
"uố": "\x9e",
"uổ": "\x9f",
"uỗ": "\xa0",
"uộ": "\xa1",
"ươ": "\xa2",
"ườ": "\xa3",
"ướ": "\xa4",
"ưở": "\xa5",
"ưỡ": "\xa6",
"ượ": "\xa7",
}
def decode_string(x):
for k, v in list(reversed(list(ENCODER.items()))):
x = x.replace(v, k)
return x
test_dataset = load_dataset("common_voice", "vi", split="test")
wer = load_metric("wer")
processor = Wav2Vec2Processor.from_pretrained("Nhut/wav2vec2-large-xlsr-vietnamese")
model = Wav2Vec2ForCTC.from_pretrained("Nhut/wav2vec2-large-xlsr-vietnamese")
model.to("cuda")
chars_to_ignore_regex = '[\\\+\@\ǀ\,\?\.\!\-\;\:\"\“\%\‘\”\�]'
resampler = torchaudio.transforms.Resample(48_000, 16_000)
# Preprocessing the datasets.
# We need to read the aduio files as arrays
def speech_file_to_array_fn(batch):
batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
speech_array, sampling_rate = torchaudio.load(batch["path"])
batch["speech"] = resampler(speech_array).squeeze().numpy()
return batch
test_dataset = test_dataset.map(speech_file_to_array_fn)
# Preprocessing the datasets.
# We need to read the aduio files as arrays
def evaluate(batch):
inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
with torch.no_grad():
logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
pred_ids = torch.argmax(logits, dim=-1)
batch["pred_strings"] = processor.batch_decode(pred_ids)
# decode_string: We replace the encoded letter with the initial letters
batch["pred_strings"] = [decode_string(x) for x in batch["pred_strings"]]
return batch
result = test_dataset.map(evaluate, batched=True, batch_size=8)
print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
📚 Documentation
Model Information
| Property | Details |
|---|---|
| Model Type | Fine - tuned Wav2Vec2 model on Vietnamese |
| Training Data | Common Voice train, validation, FOSD and VIVOS datasets |
| Metrics | Word Error Rate (WER) |
| Tags | audio, automatic - speech - recognition, speech, xlsr - fine - tuning - week |
| License | apache - 2.0 |
Test Result
The model achieves a Word Error Rate (WER) of 49.59% on the Vietnamese test data of Common Voice.
Training
The Common Voice train, validation, FOSD and VIVOS datasets were used for training. The script used for training can be found here.
📄 License
The model is released under the apache - 2.0 license.
Voice Activity Detection
MIT
Voice activity detection model based on pyannote.audio 2.1, used to identify speech activity segments in audio
Speech Recognition
V
pyannote
7.7M
181
Wav2vec2 Large Xlsr 53 Portuguese
Apache-2.0
This is a fine-tuned XLSR-53 large model for Portuguese speech recognition tasks, trained on the Common Voice 6.1 dataset, supporting Portuguese speech-to-text conversion.
Speech Recognition Other
W
jonatasgrosman
4.9M
32
Whisper Large V3
Apache-2.0
Whisper is an advanced automatic speech recognition (ASR) and speech translation model proposed by OpenAI, trained on over 5 million hours of labeled data, with strong cross-dataset and cross-domain generalization capabilities.
Speech Recognition Supports Multiple Languages
W
openai
4.6M
4,321
Whisper Large V3 Turbo
MIT
Whisper is a state-of-the-art automatic speech recognition (ASR) and speech translation model developed by OpenAI, trained on over 5 million hours of labeled data, demonstrating strong generalization capabilities in zero-shot settings.
Speech Recognition
Transformers Supports Multiple Languages
Transformers Supports Multiple LanguagesW
openai
4.0M
2,317
Wav2vec2 Large Xlsr 53 Russian
Apache-2.0
A Russian speech recognition model fine-tuned from facebook/wav2vec2-large-xlsr-53, supporting 16kHz sampled audio input
Speech Recognition Other
W
jonatasgrosman
3.9M
54
Wav2vec2 Large Xlsr 53 Chinese Zh Cn
Apache-2.0
A Chinese speech recognition model fine-tuned based on facebook/wav2vec2-large-xlsr-53, supporting 16kHz sampling rate audio input.
Speech Recognition Chinese
W
jonatasgrosman
3.8M
110
Wav2vec2 Large Xlsr 53 Dutch
Apache-2.0
A Dutch speech recognition model fine-tuned based on facebook/wav2vec2-large-xlsr-53, trained on the Common Voice and CSS10 datasets, supporting 16kHz audio input.
Speech Recognition Other
W
jonatasgrosman
3.0M
12
Wav2vec2 Large Xlsr 53 Japanese
Apache-2.0
Japanese speech recognition model fine-tuned from facebook/wav2vec2-large-xlsr-53, supporting 16kHz sampling rate audio input
Speech Recognition Japanese
W
jonatasgrosman
2.9M
33
Mms 300m 1130 Forced Aligner
A text-to-audio forced alignment tool based on Hugging Face pre-trained models, supporting multiple languages with high memory efficiency
Speech Recognition Transformers Supports Multiple Languages
Transformers Supports Multiple LanguagesM
MahmoudAshraf
2.5M
50
Wav2vec2 Large Xlsr 53 Arabic
Apache-2.0
Arabic speech recognition model fine-tuned from facebook/wav2vec2-large-xlsr-53, trained on Common Voice and Arabic speech corpus
Speech Recognition Arabic
W
jonatasgrosman
2.3M
37
Featured Recommended AI Models