DAM 3B Self Contained

🚀 描述一切：详细的局部图像和视频字幕生成

NVIDIA、加州大学伯克利分校、加州大学旧金山分校联合推出的模型，可根据用户指定的图像区域生成详细描述，助力图像和视频的理解与分析。

🚀 快速开始

以下是使用这个自包含模型进行推理的示例代码：

# Copyright 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-License-Identifier: Apache-2.0

import torch
import numpy as np
from PIL import Image
from transformers import SamModel, SamProcessor, AutoModel
import cv2
import requests
from io import BytesIO


def apply_sam(image, input_points=None, input_boxes=None, input_labels=None):
    inputs = sam_processor(image, input_points=input_points, input_boxes=input_boxes,
                           input_labels=input_labels, return_tensors="pt").to(device)

    with torch.no_grad():
        outputs = sam_model(**inputs)

    masks = sam_processor.image_processor.post_process_masks(
        outputs.pred_masks.cpu(),
        inputs["original_sizes"].cpu(),
        inputs["reshaped_input_sizes"].cpu()
    )[0][0]
    scores = outputs.iou_scores[0, 0]

    mask_selection_index = scores.argmax()
    mask_np = masks[mask_selection_index].numpy()
    return mask_np


def add_contour(img, mask, input_points=None, input_boxes=None):
    img = img.copy()
    mask = mask.astype(np.uint8) * 255
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(img, contours, -1, (1.0, 1.0, 1.0), thickness=6)

    if input_points is not None:
        for points in input_points:
            for x, y in points:
                cv2.circle(img, (int(x), int(y)), radius=10, color=(1.0, 0.0, 0.0), thickness=-1)
                cv2.circle(img, (int(x), int(y)), radius=10, color=(1.0, 1.0, 1.0), thickness=2)

    if input_boxes is not None:
        for box_batch in input_boxes:
            for box in box_batch:
                x1, y1, x2, y2 = map(int, box)
                cv2.rectangle(img, (x1, y1), (x2, y2), color=(1.0, 1.0, 1.0), thickness=4)
                cv2.rectangle(img, (x1, y1), (x2, y2), color=(1.0, 0.0, 0.0), thickness=2)

    return img

def print_streaming(text):
    print(text, end="", flush=True)

if __name__ == '__main__':
    # Download the image via HTTP
    image_url = 'https://github.com/NVlabs/describe-anything/blob/main/images/1.jpg?raw=true'
    response = requests.get(image_url)
    img = Image.open(BytesIO(response.content)).convert('RGB')

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    sam_model = SamModel.from_pretrained("facebook/sam-vit-huge").to(device)
    sam_processor = SamProcessor.from_pretrained("facebook/sam-vit-huge")
    image_size = img.size  # (width, height)

    # Initialize DAM model once
    model = AutoModel.from_pretrained(
        'nvidia/DAM-3B-Self-Contained',
        trust_remote_code=True,
        torch_dtype='torch.float16'
    ).to(device)
    dam = model.init_dam(conv_mode='v1', prompt_mode='full+focal_crop')

    # Define two runs: one with points, one with box
    runs = [
        {
            'use_box': False,
            'points': [[1172, 812], [1572, 800]],
            'output_image_path': 'output_visualization_points.png'
        },
        {
            'use_box': True,
            'box': [800, 500, 1800, 1000],
            'output_image_path': 'output_visualization_box.png'
        }
    ]

    for run in runs:
        if run['use_box']:
            # Prepare box input
            coords = run['box']
            input_boxes = [[coords]]
            print(f"Running inference with input_boxes: {input_boxes}")
            mask_np = apply_sam(img, input_boxes=input_boxes)
            vis_points = None
            vis_boxes = input_boxes
        else:
            # Prepare point input
            pts = run['points']
            input_points = [pts]
            input_labels = [[1] * len(pts)]
            print(f"Running inference with input_points: {input_points}")
            mask_np = apply_sam(img, input_points=input_points, input_labels=input_labels)
            vis_points = input_points
            vis_boxes = None

        # Convert mask and describe
        mask = Image.fromarray((mask_np * 255).astype(np.uint8))
        print("Description:")
        for token in dam.get_description(
            img,
            mask,
            '<image>\nDescribe the masked region in detail.',
            streaming=True,
            temperature=0.2,
            top_p=0.5,
            num_beams=1,
            max_new_tokens=512
        ):
            print_streaming(token)
        print()  # newline

        # Save visualization with contour
        img_np = np.asarray(img).astype(float) / 255.0
        img_with_contour_np = add_contour(img_np, mask_np,
                                          input_points=vis_points,
                                          input_boxes=vis_boxes)
        img_with_contour_pil = Image.fromarray((img_with_contour_np * 255.0).astype(np.uint8))
        img_with_contour_pil.save(run['output_image_path'])
        print(f"Output image with contour saved as {run['output_image_path']}")

✨ 主要特性

• 精准区域描述：Describe Anything Model 3B（DAM - 3B）能够接收用户以点、框、涂鸦或掩码形式指定的图像区域，并生成详细的局部图像描述。
• 创新架构设计：该模型采用新颖的焦点提示和通过门控交叉注意力增强的局部视觉骨干网络，将全图像上下文与细粒度的局部细节相结合。
• 适用范围明确：仅用于研究和开发，可进行非商业使用。

📚 详细文档

模型卡片

描述

Describe Anything Model 3B（DAM - 3B）接收用户在图像中以点、框、涂鸦或掩码形式指定的区域作为输入，并生成详细的局部图像描述。DAM使用新颖的焦点提示和通过门控交叉注意力增强的局部视觉骨干网络，将全图像上下文与细粒度的局部细节相结合。该模型仅用于研究和开发，可进行非商业使用。

许可证

NVIDIA非商业许可证

预期用途

此模型旨在展示和促进对描述任何事物模型的理解和使用。它主要应用于研究和非商业目的。

模型架构

该模型基于VILA - 1.5开发，拥有30亿个模型参数。

输入

输出

支持的硬件微架构兼容性：

• NVIDIA安培
• NVIDIA霍珀
• NVIDIA洛维拉尔

首选/支持的操作系统：

• Linux

训练数据集

Describe Anything训练数据集

评估数据集

我们在详细的局部字幕基准测试中评估我们的模型：DLC - Bench

推理

PyTorch

伦理考量

NVIDIA认为可信AI是一项共同责任，我们已经制定了政策和实践，以支持广泛的AI应用开发。当根据我们的服务条款下载或使用时，开发者应与他们的内部模型团队合作，确保该模型满足相关行业和用例的要求，并解决不可预见的产品滥用问题。

请在此报告安全漏洞或NVIDIA AI相关问题。

📄 许可证

本项目采用NVIDIA非商业许可证。

📖 引用

如果您使用我们的工作或本仓库中的实现，或者认为它们有帮助，请考虑引用：

@article{lian2025describe,
  title={Describe Anything: Detailed Localized Image and Video Captioning}, 
  author={Long Lian and Yifan Ding and Yunhao Ge and Sifei Liu and Hanzi Mao and Boyi Li and Marco Pavone and Ming-Yu Liu and Trevor Darrell and Adam Yala and Yin Cui},
  journal={arXiv preprint arXiv:2504.16072},
  year={2025}
}