DAM 3B Self Contained

🚀 描述一切：詳細的局部圖像和視頻字幕生成

NVIDIA、加州大學伯克利分校、加州大學舊金山分校聯合推出的模型，可根據用戶指定的圖像區域生成詳細描述，助力圖像和視頻的理解與分析。

🚀 快速開始

以下是使用這個自包含模型進行推理的示例代碼：

# Copyright 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-License-Identifier: Apache-2.0

import torch
import numpy as np
from PIL import Image
from transformers import SamModel, SamProcessor, AutoModel
import cv2
import requests
from io import BytesIO


def apply_sam(image, input_points=None, input_boxes=None, input_labels=None):
    inputs = sam_processor(image, input_points=input_points, input_boxes=input_boxes,
                           input_labels=input_labels, return_tensors="pt").to(device)

    with torch.no_grad():
        outputs = sam_model(**inputs)

    masks = sam_processor.image_processor.post_process_masks(
        outputs.pred_masks.cpu(),
        inputs["original_sizes"].cpu(),
        inputs["reshaped_input_sizes"].cpu()
    )[0][0]
    scores = outputs.iou_scores[0, 0]

    mask_selection_index = scores.argmax()
    mask_np = masks[mask_selection_index].numpy()
    return mask_np


def add_contour(img, mask, input_points=None, input_boxes=None):
    img = img.copy()
    mask = mask.astype(np.uint8) * 255
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(img, contours, -1, (1.0, 1.0, 1.0), thickness=6)

    if input_points is not None:
        for points in input_points:
            for x, y in points:
                cv2.circle(img, (int(x), int(y)), radius=10, color=(1.0, 0.0, 0.0), thickness=-1)
                cv2.circle(img, (int(x), int(y)), radius=10, color=(1.0, 1.0, 1.0), thickness=2)

    if input_boxes is not None:
        for box_batch in input_boxes:
            for box in box_batch:
                x1, y1, x2, y2 = map(int, box)
                cv2.rectangle(img, (x1, y1), (x2, y2), color=(1.0, 1.0, 1.0), thickness=4)
                cv2.rectangle(img, (x1, y1), (x2, y2), color=(1.0, 0.0, 0.0), thickness=2)

    return img

def print_streaming(text):
    print(text, end="", flush=True)

if __name__ == '__main__':
    # Download the image via HTTP
    image_url = 'https://github.com/NVlabs/describe-anything/blob/main/images/1.jpg?raw=true'
    response = requests.get(image_url)
    img = Image.open(BytesIO(response.content)).convert('RGB')

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    sam_model = SamModel.from_pretrained("facebook/sam-vit-huge").to(device)
    sam_processor = SamProcessor.from_pretrained("facebook/sam-vit-huge")
    image_size = img.size  # (width, height)

    # Initialize DAM model once
    model = AutoModel.from_pretrained(
        'nvidia/DAM-3B-Self-Contained',
        trust_remote_code=True,
        torch_dtype='torch.float16'
    ).to(device)
    dam = model.init_dam(conv_mode='v1', prompt_mode='full+focal_crop')

    # Define two runs: one with points, one with box
    runs = [
        {
            'use_box': False,
            'points': [[1172, 812], [1572, 800]],
            'output_image_path': 'output_visualization_points.png'
        },
        {
            'use_box': True,
            'box': [800, 500, 1800, 1000],
            'output_image_path': 'output_visualization_box.png'
        }
    ]

    for run in runs:
        if run['use_box']:
            # Prepare box input
            coords = run['box']
            input_boxes = [[coords]]
            print(f"Running inference with input_boxes: {input_boxes}")
            mask_np = apply_sam(img, input_boxes=input_boxes)
            vis_points = None
            vis_boxes = input_boxes
        else:
            # Prepare point input
            pts = run['points']
            input_points = [pts]
            input_labels = [[1] * len(pts)]
            print(f"Running inference with input_points: {input_points}")
            mask_np = apply_sam(img, input_points=input_points, input_labels=input_labels)
            vis_points = input_points
            vis_boxes = None

        # Convert mask and describe
        mask = Image.fromarray((mask_np * 255).astype(np.uint8))
        print("Description:")
        for token in dam.get_description(
            img,
            mask,
            '<image>\nDescribe the masked region in detail.',
            streaming=True,
            temperature=0.2,
            top_p=0.5,
            num_beams=1,
            max_new_tokens=512
        ):
            print_streaming(token)
        print()  # newline

        # Save visualization with contour
        img_np = np.asarray(img).astype(float) / 255.0
        img_with_contour_np = add_contour(img_np, mask_np,
                                          input_points=vis_points,
                                          input_boxes=vis_boxes)
        img_with_contour_pil = Image.fromarray((img_with_contour_np * 255.0).astype(np.uint8))
        img_with_contour_pil.save(run['output_image_path'])
        print(f"Output image with contour saved as {run['output_image_path']}")

✨ 主要特性

• 精準區域描述：Describe Anything Model 3B（DAM - 3B）能夠接收用戶以點、框、塗鴉或掩碼形式指定的圖像區域，並生成詳細的局部圖像描述。
• 創新架構設計：該模型採用新穎的焦點提示和通過門控交叉注意力增強的局部視覺骨幹網絡，將全圖像上下文與細粒度的局部細節相結合。
• 適用範圍明確：僅用於研究和開發，可進行非商業使用。

📚 詳細文檔

模型卡片

描述

Describe Anything Model 3B（DAM - 3B）接收用戶在圖像中以點、框、塗鴉或掩碼形式指定的區域作為輸入，並生成詳細的局部圖像描述。DAM使用新穎的焦點提示和通過門控交叉注意力增強的局部視覺骨幹網絡，將全圖像上下文與細粒度的局部細節相結合。該模型僅用於研究和開發，可進行非商業使用。

許可證

NVIDIA非商業許可證

預期用途

此模型旨在展示和促進對描述任何事物模型的理解和使用。它主要應用於研究和非商業目的。

模型架構

該模型基於VILA - 1.5開發，擁有30億個模型參數。

輸入

輸出

支持的硬件微架構兼容性：

• NVIDIA安培
• NVIDIA霍珀
• NVIDIA洛維拉爾

首選/支持的操作系統：

• Linux

訓練數據集

Describe Anything訓練數據集

評估數據集

我們在詳細的局部字幕基準測試中評估我們的模型：DLC - Bench

推理

PyTorch

倫理考量

NVIDIA認為可信AI是一項共同責任，我們已經制定了政策和實踐，以支持廣泛的AI應用開發。當根據我們的服務條款下載或使用時，開發者應與他們的內部模型團隊合作，確保該模型滿足相關行業和用例的要求，並解決不可預見的產品濫用問題。

請在此報告安全漏洞或NVIDIA AI相關問題。

📄 許可證

本項目採用NVIDIA非商業許可證。

📖 引用

如果您使用我們的工作或本倉庫中的實現，或者認為它們有幫助，請考慮引用：

@article{lian2025describe,
  title={Describe Anything: Detailed Localized Image and Video Captioning}, 
  author={Long Lian and Yifan Ding and Yunhao Ge and Sifei Liu and Hanzi Mao and Boyi Li and Marco Pavone and Ming-Yu Liu and Trevor Darrell and Adam Yala and Yin Cui},
  journal={arXiv preprint arXiv:2504.16072},
  year={2025}
}