Emova Qwen 2 5 3b Hf

🚀 EMOVA-Qwen-2.5-3B-HF

EMOVA-Qwen-2.5-3B-HF 是一款端到端的全模态大语言模型，无需依赖外部模型，就能实现视觉、听觉和语言交互。它能根据文本、视觉和语音等多模态输入，生成文本和语音响应，并实现生动的情感控制。该模型具备通用的全模态理解和生成能力，在高级视觉语言理解、情感语音对话以及结构化数据理解的语音对话方面表现出色。

🚀 快速开始

本仓库包含以 HuggingFace 格式组织的 EMOVA-Qwen2.5-3B 检查点，因此可以使用 transformers Auto APIs 直接加载。

from transformers import AutoModel, AutoProcessor
from PIL import Image
import torch

### Uncomment if you want to use Ascend NPUs
# import torch_npu
# from torch_npu.contrib import transfer_to_npu

# prepare models and processors
model = AutoModel.from_pretrained(
    "Emova-ollm/emova-qwen-2-5-3b-hf",
    torch_dtype=torch.bfloat16,
    attn_implementation='flash_attention_2', # OR 'sdpa' for Ascend NPUs
    low_cpu_mem_usage=True,
    trust_remote_code=True).eval().cuda()
processor = AutoProcessor.from_pretrained("Emova-ollm/emova-qwen-2-5-3b-hf", trust_remote_code=True)

# only necessary for spoken dialogue
# Note to inference with speech inputs/outputs, **emova_speech_tokenizer** is still a necessary dependency (https://huggingface.co/Emova-ollm/emova_speech_tokenizer_hf#install).
speeck_tokenizer = AutoModel.from_pretrained("Emova-ollm/emova_speech_tokenizer_hf", torch_dtype=torch.float32, trust_remote_code=True).eval().cuda()
processor.set_speech_tokenizer(speeck_tokenizer)

# Example 1: image-text
inputs = dict(
    text=[
        {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]},
        {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What's shown in this image?"}]},
        {"role": "assistant", "content": [{"type": "text", "text": "This image shows a red stop sign."}]},
        {"role": "user", "content": [{"type": "text", "text": "Describe the image in more details."}]},
    ],
    images=Image.open('path/to/image')
)

# Example 2: text-audio
inputs = dict(
    text=[{"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}],
    audios='path/to/audio'
)

# Example 3: image-text-audio
inputs = dict(
    text=[{"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}],
    images=Image.open('path/to/image'),
    audios='path/to/audio'
)

# run processors
has_speech = 'audios' in inputs.keys()
inputs = processor(**inputs, return_tensors="pt")
inputs = inputs.to(model.device)

# prepare generation arguments
gen_kwargs = {"max_new_tokens": 4096, "do_sample": False} # add if necessary
speech_kwargs = {"speaker": "female", "output_wav_prefix": "output"} if has_speech else {}

# run generation
# for speech outputs, we will return the saved wav paths (c.f., output_wav_prefix)
with torch.no_grad():
    outputs = model.generate(**inputs, **gen_kwargs)
    outputs = outputs[:, inputs['input_ids'].shape[1]:]
    print(processor.batch_decode(outputs, skip_special_tokens=True, **speech_kwargs))

✨ 主要特性

• 先进的全模态性能：EMOVA 在 视觉语言 和 语音 基准测试中同时取得了先进的可比结果。表现最佳的 EMOVA-72B 模型甚至超越了包括 GPT-4o 和 Gemini Pro 1.5 在内的商业模型。
• 情感语音对话：采用了 语义 - 声学解耦 的语音分词器和轻量级的 风格控制 模块，实现了无缝的全模态对齐和多样化的语音风格控制。EMOVA 支持 双语（中文和英文） 语音对话，具备 24 种语音风格 控制（即 2 种说话者、3 种音高和 4 种情感）。
• 多样化的配置：开源了 3 种配置，即 EMOVA-3B/7B/72B，以支持不同计算预算下的全模态使用。可以查看 模型库，为计算设备找到最合适的模型！

📦 安装指南

文档未提及安装步骤，故跳过该章节。

💻 使用示例

基础用法

# 以下是使用 EMOVA-Qwen-2.5-3B-HF 进行图像 - 文本交互的示例代码
from transformers import AutoModel, AutoProcessor
from PIL import Image
import torch

### Uncomment if you want to use Ascend NPUs
# import torch_npu
# from torch_npu.contrib import transfer_to_npu

# prepare models and processors
model = AutoModel.from_pretrained(
    "Emova-ollm/emova-qwen-2-5-3b-hf",
    torch_dtype=torch.bfloat16,
    attn_implementation='flash_attention_2', # OR 'sdpa' for Ascend NPUs
    low_cpu_mem_usage=True,
    trust_remote_code=True).eval().cuda()
processor = AutoProcessor.from_pretrained("Emova-ollm/emova-qwen-2-5-3b-hf", trust_remote_code=True)

# only necessary for spoken dialogue
# Note to inference with speech inputs/outputs, **emova_speech_tokenizer** is still a necessary dependency (https://huggingface.co/Emova-ollm/emova_speech_tokenizer_hf#install).
speeck_tokenizer = AutoModel.from_pretrained("Emova-ollm/emova_speech_tokenizer_hf", torch_dtype=torch.float32, trust_remote_code=True).eval().cuda()
processor.set_speech_tokenizer(speeck_tokenizer)

# Example 1: image-text
inputs = dict(
    text=[
        {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]},
        {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What's shown in this image?"}]},
        {"role": "assistant", "content": [{"type": "text", "text": "This image shows a red stop sign."}]},
        {"role": "user", "content": [{"type": "text", "text": "Describe the image in more details."}]},
    ],
    images=Image.open('path/to/image')
)

# run processors
has_speech = 'audios' in inputs.keys()
inputs = processor(**inputs, return_tensors="pt")
inputs = inputs.to(model.device)

# prepare generation arguments
gen_kwargs = {"max_new_tokens": 4096, "do_sample": False} # add if necessary
speech_kwargs = {"speaker": "female", "output_wav_prefix": "output"} if has_speech else {}

# run generation
# for speech outputs, we will return the saved wav paths (c.f., output_wav_prefix)
with torch.no_grad():
    outputs = model.generate(**inputs, **gen_kwargs)
    outputs = outputs[:, inputs['input_ids'].shape[1]:]
    print(processor.batch_decode(outputs, skip_special_tokens=True, **speech_kwargs))

高级用法

# 以下是使用 EMOVA-Qwen-2.5-3B-HF 进行图像 - 文本 - 音频交互的示例代码
from transformers import AutoModel, AutoProcessor
from PIL import Image
import torch

### Uncomment if you want to use Ascend NPUs
# import torch_npu
# from torch_npu.contrib import transfer_to_npu

# prepare models and processors
model = AutoModel.from_pretrained(
    "Emova-ollm/emova-qwen-2-5-3b-hf",
    torch_dtype=torch.bfloat16,
    attn_implementation='flash_attention_2', # OR 'sdpa' for Ascend NPUs
    low_cpu_mem_usage=True,
    trust_remote_code=True).eval().cuda()
processor = AutoProcessor.from_pretrained("Emova-ollm/emova-qwen-2-5-3b-hf", trust_remote_code=True)

# only necessary for spoken dialogue
# Note to inference with speech inputs/outputs, **emova_speech_tokenizer** is still a necessary dependency (https://huggingface.co/Emova-ollm/emova_speech_tokenizer_hf#install).
speeck_tokenizer = AutoModel.from_pretrained("Emova-ollm/emova_speech_tokenizer_hf", torch_dtype=torch.float32, trust_remote_code=True).eval().cuda()
processor.set_speech_tokenizer(speeck_tokenizer)

# Example 3: image-text-audio
inputs = dict(
    text=[{"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}],
    images=Image.open('path/to/image'),
    audios='path/to/audio'
)

# run processors
has_speech = 'audios' in inputs.keys()
inputs = processor(**inputs, return_tensors="pt")
inputs = inputs.to(model.device)

# prepare generation arguments
gen_kwargs = {"max_new_tokens": 4096, "do_sample": False} # add if necessary
speech_kwargs = {"speaker": "female", "output_wav_prefix": "output"} if has_speech else {}

# run generation
# for speech outputs, we will return the saved wav paths (c.f., output_wav_prefix)
with torch.no_grad():
    outputs = model.generate(**inputs, **gen_kwargs)
    outputs = outputs[:, inputs['input_ids'].shape[1]:]
    print(processor.batch_decode(outputs, skip_special_tokens=True, **speech_kwargs))

📚 详细文档

模型信息

性能表现

🔧 技术细节

文档未提供相关技术细节，故跳过该章节。

📄 许可证

本模型采用 Apache-2.0 许可证。

📚 引用

@article{chen2024emova,
  title={Emova: Empowering language models to see, hear and speak with vivid emotions},
  author={Chen, Kai and Gou, Yunhao and Huang, Runhui and Liu, Zhili and Tan, Daxin and Xu, Jing and Wang, Chunwei and Zhu, Yi and Zeng, Yihan and Yang, Kuo and others},
  journal={arXiv preprint arXiv:2409.18042},
  year={2024}
}