diff --git a/docs/source/Instruction/Supported-models-and-datasets.md b/docs/source/Instruction/Supported-models-and-datasets.md index 92567980b2..28227485d2 100644 --- a/docs/source/Instruction/Supported-models-and-datasets.md +++ b/docs/source/Instruction/Supported-models-and-datasets.md @@ -1158,6 +1158,16 @@ |[AI-ModelScope/Florence-2-large](https://modelscope.cn/models/AI-ModelScope/Florence-2-large)|florence|florence||-|✘|vision|[microsoft/Florence-2-large](https://huggingface.co/microsoft/Florence-2-large)| |[AI-ModelScope/Florence-2-large-ft](https://modelscope.cn/models/AI-ModelScope/Florence-2-large-ft)|florence|florence||-|✘|vision|[microsoft/Florence-2-large-ft](https://huggingface.co/microsoft/Florence-2-large-ft)| |[AI-ModelScope/Idefics3-8B-Llama3](https://modelscope.cn/models/AI-ModelScope/Idefics3-8B-Llama3)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceM4/Idefics3-8B-Llama3](https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3)| +|[HuggingFaceTB/SmolVLM-256M-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-256M-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-256M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Instruct)| +|[HuggingFaceTB/SmolVLM-500M-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-500M-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-500M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct)| +|[HuggingFaceTB/SmolVLM-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct)| +|[HuggingFaceTB/SmolVLM-256M-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-256M-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-256M-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Base)| +|[HuggingFaceTB/SmolVLM-500M-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-500M-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-500M-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Base)| +|[HuggingFaceTB/SmolVLM-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-Base)| +|[HuggingFaceTB/SmolVLM2-2.2B-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-2.2B-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM2-2.2B-Base](https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Base)| +|[HuggingFaceTB/SmolVLM2-256M-Video-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-256M-Video-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-256M-Video-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-256M-Video-Instruct)| +|[HuggingFaceTB/SmolVLM2-500M-Video-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-500M-Video-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-500M-Video-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct)| +|[HuggingFaceTB/SmolVLM2-2.2B-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-2.2B-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-2.2B-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Instruct)| |[AI-ModelScope/paligemma-3b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-224)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-224](https://huggingface.co/google/paligemma-3b-pt-224)| |[AI-ModelScope/paligemma-3b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-448)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-448](https://huggingface.co/google/paligemma-3b-pt-448)| |[AI-ModelScope/paligemma-3b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-896)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-896](https://huggingface.co/google/paligemma-3b-pt-896)| diff --git a/docs/source_en/Instruction/Supported-models-and-datasets.md b/docs/source_en/Instruction/Supported-models-and-datasets.md index ce52b22fd4..849611610a 100644 --- a/docs/source_en/Instruction/Supported-models-and-datasets.md +++ b/docs/source_en/Instruction/Supported-models-and-datasets.md @@ -1159,6 +1159,16 @@ The table below introduces the models integrated with ms-swift: |[AI-ModelScope/Florence-2-large](https://modelscope.cn/models/AI-ModelScope/Florence-2-large)|florence|florence||-|✘|vision|[microsoft/Florence-2-large](https://huggingface.co/microsoft/Florence-2-large)| |[AI-ModelScope/Florence-2-large-ft](https://modelscope.cn/models/AI-ModelScope/Florence-2-large-ft)|florence|florence||-|✘|vision|[microsoft/Florence-2-large-ft](https://huggingface.co/microsoft/Florence-2-large-ft)| |[AI-ModelScope/Idefics3-8B-Llama3](https://modelscope.cn/models/AI-ModelScope/Idefics3-8B-Llama3)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceM4/Idefics3-8B-Llama3](https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3)| +|[HuggingFaceTB/SmolVLM-256M-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-256M-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-256M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Instruct)| +|[HuggingFaceTB/SmolVLM-500M-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-500M-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-500M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct)| +|[HuggingFaceTB/SmolVLM-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-Instruct)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct)| +|[HuggingFaceTB/SmolVLM-256M-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-256M-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-256M-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Base)| +|[HuggingFaceTB/SmolVLM-500M-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-500M-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-500M-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Base)| +|[HuggingFaceTB/SmolVLM-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM-Base](https://huggingface.co/HuggingFaceTB/SmolVLM-Base)| +|[HuggingFaceTB/SmolVLM2-2.2B-Base](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-2.2B-Base)|idefics3|idefics3||transformers>=4.45|✘|vision|[HuggingFaceTB/SmolVLM2-2.2B-Base](https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Base)| +|[HuggingFaceTB/SmolVLM2-256M-Video-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-256M-Video-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-256M-Video-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-256M-Video-Instruct)| +|[HuggingFaceTB/SmolVLM2-500M-Video-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-500M-Video-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-500M-Video-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct)| +|[HuggingFaceTB/SmolVLM2-2.2B-Instruct](https://modelscope.cn/models/HuggingFaceTB/SmolVLM2-2.2B-Instruct)|smolvlm2|idefics3||transformers>=4.50, num2words|✘|vision, video|[HuggingFaceTB/SmolVLM2-2.2B-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Instruct)| |[AI-ModelScope/paligemma-3b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-224)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-224](https://huggingface.co/google/paligemma-3b-pt-224)| |[AI-ModelScope/paligemma-3b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-448)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-448](https://huggingface.co/google/paligemma-3b-pt-448)| |[AI-ModelScope/paligemma-3b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-896)|paligemma|paligemma||transformers>=4.41|✘|vision|[google/paligemma-3b-pt-896](https://huggingface.co/google/paligemma-3b-pt-896)| diff --git a/swift/model/constant.py b/swift/model/constant.py index f2a8c66fb8..680dbbc24e 100644 --- a/swift/model/constant.py +++ b/swift/model/constant.py @@ -276,6 +276,7 @@ class MLLMModelType: phi4_multimodal = 'phi4_multimodal' florence = 'florence' idefics3 = 'idefics3' + smolvlm2 = 'smolvlm2' paligemma = 'paligemma' molmo = 'molmo' molmo2 = 'molmo2' diff --git a/swift/model/models/mllm.py b/swift/model/models/mllm.py index 0bd9f3eee7..b41761728c 100644 --- a/swift/model/models/mllm.py +++ b/swift/model/models/mllm.py @@ -33,6 +33,15 @@ def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel: ModelGroup([ Model('AI-ModelScope/Idefics3-8B-Llama3', 'HuggingFaceM4/Idefics3-8B-Llama3'), ]), + ModelGroup([ + Model('HuggingFaceTB/SmolVLM-256M-Instruct', 'HuggingFaceTB/SmolVLM-256M-Instruct'), + Model('HuggingFaceTB/SmolVLM-500M-Instruct', 'HuggingFaceTB/SmolVLM-500M-Instruct'), + Model('HuggingFaceTB/SmolVLM-Instruct', 'HuggingFaceTB/SmolVLM-Instruct'), + Model('HuggingFaceTB/SmolVLM-256M-Base', 'HuggingFaceTB/SmolVLM-256M-Base'), + Model('HuggingFaceTB/SmolVLM-500M-Base', 'HuggingFaceTB/SmolVLM-500M-Base'), + Model('HuggingFaceTB/SmolVLM-Base', 'HuggingFaceTB/SmolVLM-Base'), + Model('HuggingFaceTB/SmolVLM2-2.2B-Base', 'HuggingFaceTB/SmolVLM2-2.2B-Base'), + ]), ], Idefics3Loader, template=TemplateType.idefics3, @@ -42,6 +51,24 @@ def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel: requires=['transformers>=4.45'], )) +register_model( + ModelMeta( + MLLMModelType.smolvlm2, + [ + ModelGroup([ + Model('HuggingFaceTB/SmolVLM2-256M-Video-Instruct', 'HuggingFaceTB/SmolVLM2-256M-Video-Instruct'), + Model('HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct'), + Model('HuggingFaceTB/SmolVLM2-2.2B-Instruct', 'HuggingFaceTB/SmolVLM2-2.2B-Instruct'), + ]), + ], + Idefics3Loader, + template=TemplateType.idefics3, + model_arch=ModelArch.idefics3, + architectures=['SmolVLMForConditionalGeneration'], + tags=['vision', 'video'], + requires=['transformers>=4.50', 'num2words'], + )) + class PixtralLoader(ModelLoader): diff --git a/swift/template/templates/idefics3.py b/swift/template/templates/idefics3.py index b730daf775..1770f0f14c 100644 --- a/swift/template/templates/idefics3.py +++ b/swift/template/templates/idefics3.py @@ -1,37 +1,103 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from typing import Any, Dict +import torch +from typing import Any, Dict, List, Literal from ..base import Template from ..constant import MLLMTemplateType from ..register import TemplateMeta, register_template from ..template_inputs import StdTemplateInputs -from ..utils import align_image_inputs +from ..utils import Context +from ..vision_utils import load_video_hf class Idefics3Template(Template): + """Idefics3, SmolVLM and SmolVLM2. Each ``/`