From 3798f87a06228a382a2247597b12566837ed9b3c Mon Sep 17 00:00:00 2001 From: sunfeng Date: Fri, 30 Oct 2026 21:02:17 +0800 Subject: [PATCH 1/2] Add GPTQModel-backed AWQ export Use GPTQModel public loading API, add Qwen2-VL image calibration and document the optional dependencies and known compatibility limits. AI-Assisted-by: OpenAI Codex --- docs/source/Instruction/Export-and-push.md | 26 +- docs/source_en/Instruction/Export-and-push.md | 26 +- requirements/awq.txt | 1 + setup.py | 1 + swift/pipelines/export/quant.py | 178 +++++++++--- tests/export/test_awq_dependency_metadata.py | 24 ++ tests/export/test_awq_gptqmodel.py | 273 ++++++++++++++++++ tests/export/test_quant_gpu_integration.py | 36 +++ 8 files changed, 509 insertions(+), 56 deletions(-) create mode 100644 requirements/awq.txt create mode 100644 tests/export/test_awq_dependency_metadata.py create mode 100644 tests/export/test_awq_gptqmodel.py create mode 100644 tests/export/test_quant_gpu_integration.py diff --git a/docs/source/Instruction/Export-and-push.md b/docs/source/Instruction/Export-and-push.md index 2b0b45e92c..c325784932 100644 --- a/docs/source/Instruction/Export-and-push.md +++ b/docs/source/Instruction/Export-and-push.md @@ -13,16 +13,17 @@ SWIFT支持AWQ、GPTQ、FP8、BNB模型的量化导出。其中使用AWQ、GPTQ | -------- | ------ | -------- | -------- | | FP8 | ✅ | ✅ | ✅ | | GPTQ | ✅ | ✅ | ✅ | -| AWQ | ✅ | ✅ | ✅ | +| AWQ | 部分支持 | ✅ | ✅ | | BNB | ❌ | ✅ | ✅ | 除SWIFT安装外,需要安装以下额外依赖: ```shell -# 使用awq量化: -# autoawq和cuda版本有对应关系,请按照`https://github.com/casper-hansen/AutoAWQ`选择版本 -# 如果出现torch依赖冲突,请额外增加指令`--no-deps` -pip install autoawq -U +# 使用 AWQ 量化(从源码目录安装): +pip install -e '.[awq]' + +# 已发布的 ms-swift 包提供该选项后,也可以使用: +# pip install 'ms-swift[awq]' # 使用gptq量化: pip install gptqmodel optimum -U @@ -34,13 +35,26 @@ pip install gptqmodel optimum -U pip install bitsandbytes -U ``` +`awq` 是独立的可选依赖,不包含在基础安装或 `all` 选项中。安装前请检查环境中的依赖约束,安装后运行 `python -m pip check`。以下是已验证的特定版本,不能推断其他版本也兼容: + +| 组件 | 测试版本或已知约束 | +| --- | --- | +| AWQ 导出测试环境 | Python 3.12、PyTorch 2.11.0+cu130、Transformers 5.14.1、GPTQModel 7.3.2 | +| GPTQModel 7.3.2 | 要求 Python >=3.10;Python 3.12 下要求 NumPy 2.2.6 和 protobuf >=7.34.0 | +| SwanLab 0.9.2 | 要求 protobuf <7,与 GPTQModel 7.3.2 的要求冲突 | +| ms-opencompass 0.1.6 | 要求 NumPy <2,与 GPTQModel 7.3.2 的要求冲突 | + +如果同一环境需要 AWQ、SwanLab 或 ms-opencompass,先确认所选版本的依赖声明存在共同解;上述冲突版本不能通过调整安装顺序解决。若暂时没有共同解,可以将量化、训练和评测放在各自的环境中。不要使用 `--no-deps` 将 `pip check` 中的冲突视为已解决。实际依赖可能随版本和 Python 版本变化,还应验证 PyTorch、CUDA、Transformers 和其他已安装组件。上述测试镜像自身仍有其他依赖冲突,AWQ 导出测试通过不代表整个镜像兼容。 + 我们提供了一系列脚本展现SWIFT的量化导出能力: - 支持[AWQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/awq.sh)/[GPTQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq.sh)/[GPTQ v2](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq_v2.sh)/[BNB](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/bnb.sh)量化导出。 -- 多模态量化: 支持使用GPTQ和AWQ对多模态模型进行量化,其中AWQ支持的多模态模型有限。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm)。 +- 多模态量化: GPTQ支持多模态模型;AWQ目前仅支持Qwen2-VL图像校准,其他模态和模型尚不支持。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm)。 - 更多系列模型的支持: 支持[Bert](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/bert),[Reward Model](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/reward_model)的量化导出。 - 使用SWIFT量化导出的模型支持使用vllm/sglang/lmdeploy进行推理加速;也支持使用QLoRA继续进行SFT/RLHF。 +AWQ 使用 GPTQModel 生成带零点的 GEMM 格式权重。目前支持文本和 Qwen2-VL 图像校准,使用 safetensors 和未量化的模型;LoRA 需要先使用 `--merge_lora true` 合并。AWQ 不支持在导出时添加特殊 token、指定 `--device_map` 或设置自定义模型加载参数。Qwen2-VL 图像校准使用 GPTQModel 自身的 processor 和模型聊天模板,不支持自定义 `--template`、`--system` 或 `--max_pixels`,仅支持 `--truncation_strategy delete`。模型架构须受 GPTQModel 支持,安装时请检查与现有 PyTorch、Transformers 和其他依赖的兼容性。 + 使用 GPTQ v2 量化时,后端可能将对称量化权重保存为 GPTQ v1 格式。导出配置会记录实际保存的格式,以便正确重新加载模型。 ## 推送模型 diff --git a/docs/source_en/Instruction/Export-and-push.md b/docs/source_en/Instruction/Export-and-push.md index 1b8e82725b..d98bc2303b 100644 --- a/docs/source_en/Instruction/Export-and-push.md +++ b/docs/source_en/Instruction/Export-and-push.md @@ -12,16 +12,17 @@ SWIFT supports quantization exports for AWQ, GPTQ, FP8, and BNB models. AWQ and | ---------------------- | ---------- | ---------------------- | ------------------ | | FP8 | ✅ | ✅ | ✅ | | GPTQ | ✅ | ✅ | ✅ | -| AWQ | ✅ | ✅ | ✅ | +| AWQ | Partial | ✅ | ✅ | | BNB | ❌ | ✅ | ✅ | In addition to the SWIFT installation, the following additional dependencies need to be installed: ```shell -# For AWQ quantization: -# The versions of autoawq and CUDA are correlated; please choose the version according to `https://github.com/casper-hansen/AutoAWQ`. -# If there are dependency conflicts with torch, please add the `--no-deps` option. -pip install autoawq -U +# For AWQ quantization (from the source checkout): +pip install -e '.[awq]' + +# Once a published ms-swift release provides this extra: +# pip install 'ms-swift[awq]' # For GPTQ quantization: pip install gptqmodel optimum -U @@ -33,14 +34,27 @@ pip install gptqmodel optimum -U pip install bitsandbytes -U ``` +`awq` is an independent optional extra; it is not included in the base installation or the `all` extra. Check the environment's dependency constraints before installing and run `python -m pip check` afterward. The following observations apply to the specific versions tested, not to all releases: + +| Component | Tested version or known constraint | +| --- | --- | +| AWQ export test environment | Python 3.12, PyTorch 2.11.0+cu130, Transformers 5.14.1, GPTQModel 7.3.2 | +| GPTQModel 7.3.2 | Requires Python >=3.10; on Python 3.12, requires NumPy 2.2.6 and protobuf >=7.34.0 | +| SwanLab 0.9.2 | Requires protobuf <7, conflicting with GPTQModel 7.3.2 | +| ms-opencompass 0.1.6 | Requires NumPy <2, conflicting with GPTQModel 7.3.2 | + +If AWQ, SwanLab, or ms-opencompass are needed in the same environment, first confirm that the chosen versions' declared requirements have a common solution; changing the installation order cannot resolve the conflicts above. If no common solution is available, use separate environments for quantization, training, and evaluation. Do not treat `--no-deps` as a resolution of `pip check` conflicts. Requirements may change by package and Python version; also verify PyTorch, CUDA, Transformers, and other installed components. The test image has other dependency conflicts, so successful AWQ export does not establish compatibility of the entire image. + We provide a series of scripts to demonstrate SWIFT's quantization export capabilities: - Supports [AWQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/awq.sh)/[GPTQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq.sh)/[GPTQ v2](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq_v2.sh)/[BNB](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/bnb.sh) quantization exports. -- Multimodal quantization: Supports quantizing multimodal models using GPTQ and AWQ, with limited multimodal models supported by AWQ. Refer to [here](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm). +- Multimodal quantization: GPTQ supports multimodal models; AWQ currently supports image calibration for Qwen2-VL only. Refer to [here](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm). - Support for more model series: Supports quantization exports for [BERT](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/bert) and [Reward Model](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/reward_model). - Models exported with SWIFT's quantization support inference acceleration using vllm/sglang/lmdeploy; they also support further SFT/RLHF using QLoRA. +AWQ uses GPTQModel to produce GEMM weights with zero points. It supports text and Qwen2-VL image calibration, safetensors, and unquantized source models; merge LoRA adapters first using `--merge_lora true`. Adding special tokens, setting `--device_map`, or passing custom model loading arguments during AWQ export are not supported. Qwen2-VL image calibration uses GPTQModel's processor and model chat template, does not support custom `--template`, `--system`, or `--max_pixels`, and requires `--truncation_strategy delete`. The model architecture must be supported by GPTQModel. Check compatibility with the existing PyTorch, Transformers, and other dependencies when installing it. + GPTQ backends may save symmetric weights in GPTQ v1 format even when GPTQ v2 is used during quantization. The exported configuration records the saved format so the model reloads correctly. ## Push Model diff --git a/requirements/awq.txt b/requirements/awq.txt new file mode 100644 index 0000000000..bc7bd255a0 --- /dev/null +++ b/requirements/awq.txt @@ -0,0 +1 @@ +gptqmodel>=7.3.2 diff --git a/setup.py b/setup.py index d2cb423369..86dd2efd19 100644 --- a/setup.py +++ b/setup.py @@ -124,6 +124,7 @@ def gen_packages_items(): extra_requires['eval'], _ = parse_requirements('requirements/eval.txt') extra_requires['swanlab'], _ = parse_requirements('requirements/swanlab.txt') extra_requires['ray'], _ = parse_requirements('requirements/ray.txt') + extra_requires['awq'], _ = parse_requirements('requirements/awq.txt') all_requires.extend(install_requires) all_requires.extend(extra_requires['eval']) all_requires.extend(extra_requires['swanlab']) diff --git a/swift/pipelines/export/quant.py b/swift/pipelines/export/quant.py index c6f5ca9586..6df4db7730 100644 --- a/swift/pipelines/export/quant.py +++ b/swift/pipelines/export/quant.py @@ -4,7 +4,9 @@ import transformers from collections import defaultdict from contextlib import contextmanager +from io import BytesIO from packaging import version +from PIL import Image from tqdm import tqdm from typing import Dict, List, Optional @@ -22,11 +24,42 @@ class QuantEngine(ProcessorMixin): def __init__(self, args: ExportArguments): self.args = args - kwargs = {} if args.quant_method == 'awq': - from awq import AutoAWQForCausalLM - kwargs['auto_model_cls'] = AutoAWQForCausalLM - self.model, self.template = prepare_model_template(args, **kwargs) + if args.adapters: + raise ValueError('AWQ requires merged adapters. Use --merge_lora true first.') + if not args.safe_serialization: + raise ValueError('GPTQModel AWQ export requires --safe_serialization true.') + if args.quant_bits is None: + raise ValueError('Please set quant_bits for AWQ export.') + if args.new_special_tokens: + raise ValueError('AWQ does not support adding special tokens during export.') + if args.device_map is not None: + raise ValueError('GPTQModel AWQ selects its own device; --device_map is not supported.') + if args.task_type != 'causal_lm' or args.tuner_backend == 'unsloth': + raise ValueError('GPTQModel AWQ supports causal language models without the unsloth backend.') + for name in ('model_kwargs', 'rope_scaling', 'max_memory', 'attn_impl', 'experts_impl', 'max_model_len', + 'init_strategy'): + if getattr(args, name, None): + raise ValueError(f'GPTQModel AWQ does not support --{name} during export.') + if getattr(args.model_info.config, 'quantization_config', None): + raise ValueError('AWQ requires an unquantized model; the input is already quantized.') + try: + from gptqmodel import AWQConfig, GPTQModel + except ImportError as exc: + raise ImportError('AWQ export requires GPTQModel >= 7.3.2: pip install "gptqmodel>=7.3.2"') from exc + _, processor = args.get_model_processor(load_model=False) + self.template = args.get_template(processor) + quant_config = AWQConfig(bits=args.quant_bits, group_size=args.group_size, sym=False) + self.awq_model = GPTQModel.load( + args.model_dir, quantize_config=quant_config, trust_remote_code=True, dtype=args.torch_dtype) + if self.awq_model.quantized: + raise ValueError('AWQ requires an unquantized model; the input is already quantized.') + self.model = self.awq_model.model + self.model.model_meta = processor.model_meta + self.model.model_info = processor.model_info + self.model.model_dir = args.model_dir + else: + self.model, self.template = prepare_model_template(args) self.template.set_mode('train') self.model.config.use_cache = False HfConfigFactory.set_config_attr(self.model.config, 'use_cache', False) @@ -38,10 +71,9 @@ def quantize(self): if args.quant_bits is None and args.quant_method not in {'fp8', 'entropack'}: raise ValueError(f'Please set the quant_bits. args.quant_bits: {args.quant_bits}') if args.quant_method == 'awq': - self.template.model = self.model.model - self.awq_model_quantize() - self.model.save_quantized( - args.output_dir, safetensors=args.safe_serialization, shard_size=args.max_shard_size) + self.template.model = self.model + awq_model = self.awq_model_quantize() + awq_model.save(args.output_dir, max_shard_size=args.max_shard_size) elif args.quant_method in {'gptq', 'gptq_v2'}: self.template.model = self.model gptq_quantizer = self.gptq_model_quantize(v2=(args.quant_method == 'gptq_v2')) @@ -135,47 +167,105 @@ def _get_quant_dataset(self, *args, **kwargs): return res @staticmethod - @contextmanager - def _patch_awq_move_embed(awq_model): - _origin_move_embed = awq_model.move_embed - - def _move_embed(model, device: str): - if hasattr(model, '_hf_hook') and device != 'cpu': - return - _origin_move_embed(model, device) + def _awq_qwen2_vl_conversation(data): + if data.get('videos') or data.get('audios'): + raise ValueError('Qwen2-VL AWQ currently supports image calibration only.') + for key in ('objects', 'tools', 'chat_template_kwargs'): + if data.get(key): + raise ValueError(f'Qwen2-VL AWQ does not support calibration field: {key}.') + messages = data.get('messages') or [] + images = list(data.get('images') or []) + if not messages: + raise ValueError('Qwen2-VL AWQ calibration requires messages.') + if any(not isinstance(message.get('content'), str) for message in messages): + raise ValueError('Qwen2-VL AWQ calibration requires text messages with image placeholders.') + if any(message.get('role') not in {'system', 'user', 'assistant'} for message in messages): + raise ValueError('Qwen2-VL AWQ does not support tool messages in calibration.') + placeholder_count = sum(message['content'].count('') for message in messages) + if placeholder_count > len(images): + raise ValueError('More image placeholders than calibration images.') + missing = len(images) - placeholder_count + first = next((i for i, message in enumerate(messages) if message['role'] != 'system'), 0) + image_idx = 0 + result = [] + for index, message in enumerate(messages): + content = ('' * missing if index == first else '') + message['content'] + parts = content.split('') + blocks = [] + for part_idx, text in enumerate(parts): + if part_idx: + value = images[image_idx] + image_idx += 1 + if isinstance(value, dict): + if value.get('bytes') is not None: + value = Image.open(BytesIO(value['bytes'])).convert('RGB') + elif value.get('path'): + value = value['path'] + else: + raise ValueError('Calibration image requires bytes or a path.') + blocks.append({'type': 'image', 'image': value}) + if text: + blocks.append({'type': 'text', 'text': text}) + result.append({'role': message['role'], 'content': blocks}) + return result - awq_model.move_embed = _move_embed - try: - yield - finally: - awq_model.move_embed = _origin_move_embed - - def awq_model_quantize(self) -> None: - from awq.quantize import quantizer + def _get_awq_qwen2_vl_dataset(self): + args = self.args + dataset = load_dataset( + args.dataset, split_dataset_ratio=0, shuffle=args.dataset_shuffle, + **args.get_dataset_kwargs())[0].shuffle() + samples = [] + for data in dataset: + conversation = self._awq_qwen2_vl_conversation(data) + try: + self.template.encode(data) + except MaxLengthError: + continue + # GPTQModel applies its own processor and chat template for this model. + # Check that tokenization as well, since its length can differ from Swift's. + if args.max_length is not None: + prepared = self.awq_model.prepare_dataset([conversation], batch_size=1) + if any(item['input_ids'].shape[-1] > args.max_length for item in prepared): + continue + samples.append(conversation) + if len(samples) >= args.quant_n_samples: + break + return samples + def awq_model_quantize(self): args = self.args + if getattr(self.model.config, 'quantization_config', None): + raise ValueError('AWQ requires an unquantized model; the input is already quantized.') + if any(str(param.dtype).startswith('torch.float8') for param in self.model.parameters()): + raise ValueError('AWQ requires an unquantized model; FP8 weights are not supported.') + if args.adapters: + raise ValueError('AWQ requires merged adapters. Use --merge_lora true first.') + if not args.safe_serialization: + raise ValueError('GPTQModel AWQ export requires --safe_serialization true.') + logger.info(f'Quantization dataset: {args.dataset}') - _origin_get_calib_dataset = quantizer.get_calib_dataset - quantizer.get_calib_dataset = self._get_quant_dataset - quant_config = { - 'zero_point': True, - 'q_group_size': args.group_size, - 'w_bit': args.quant_bits, - 'version': 'GEMM' - } - if self.model.model_info.is_moe_model: - quant_config['modules_to_not_convert'] = self.args.get_modules_to_not_convert() - logger.info(f'quant_config: {quant_config}') + if self.model.model_meta.is_multimodal: + if self.model.model_info.model_type != 'qwen2_vl': + raise ValueError('GPTQModel AWQ multimodal calibration currently supports only Qwen2-VL images.') + if getattr(args, 'system', None) is not None: + raise ValueError('Qwen2-VL AWQ does not support a custom --system during export.') + if getattr(args, 'template', None) not in {None, 'qwen2_vl'}: + raise ValueError('Qwen2-VL AWQ does not support a custom --template during export.') + if getattr(args, 'max_pixels', None) is not None: + raise ValueError('Qwen2-VL AWQ does not support --max_pixels during export.') + if getattr(args, 'truncation_strategy', 'delete') != 'delete': + raise ValueError('Qwen2-VL AWQ supports only truncation_strategy=delete.') + calibration = self._get_awq_qwen2_vl_dataset() + else: + calibration = [{ + 'input_ids': block.reshape(-1).tolist() + } for block in self._get_quant_dataset() if block.numel()] + if not calibration: + raise ValueError('AWQ calibration dataset contains no usable tokens.') logger.info('Start quantizing the model...') - with self._patch_awq_move_embed(self.model): - self.model.quantize( - self.tokenizer, quant_config=quant_config, n_parallel_calib_samples=args.quant_batch_size) - quantizer.get_calib_dataset = _origin_get_calib_dataset # recover - if self.model.quant_config.modules_to_not_convert: - model_arch = args.model_meta.model_arch - lm_head_key = getattr(model_arch, 'lm_head', None) or 'lm_head' - if lm_head_key not in self.model.quant_config.modules_to_not_convert: - self.model.quant_config.modules_to_not_convert.append(lm_head_key) + batch_size = args.quant_batch_size if args.quant_batch_size is not None else len(calibration) + self.awq_model.quantize(calibration=calibration, batch_size=batch_size) + return self.awq_model @contextmanager def _patch_gptq(self): diff --git a/tests/export/test_awq_dependency_metadata.py b/tests/export/test_awq_dependency_metadata.py new file mode 100644 index 0000000000..a4560c2270 --- /dev/null +++ b/tests/export/test_awq_dependency_metadata.py @@ -0,0 +1,24 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import subprocess +import sys +from importlib.metadata import PathDistribution +from packaging.requirements import Requirement +from pathlib import Path + + +def test_awq_backend_is_an_opt_in_dependency(tmp_path): + root = Path(__file__).resolve().parents[2] + subprocess.run([sys.executable, 'setup.py', 'egg_info', '--egg-base', + str(tmp_path)], + cwd=root, + check=True, + capture_output=True, + text=True) + metadata = PathDistribution(next(tmp_path.glob('*.egg-info'))) + requirements = [Requirement(item) for item in metadata.requires or []] + gptqmodel = [item for item in requirements if item.name.lower() == 'gptqmodel'] + assert len(gptqmodel) == 1 + assert str(gptqmodel[0].specifier) == '>=7.3.2' + assert gptqmodel[0].marker.evaluate({'extra': 'awq'}) + assert not gptqmodel[0].marker.evaluate({'extra': ''}) + assert not gptqmodel[0].marker.evaluate({'extra': 'all'}) diff --git a/tests/export/test_awq_gptqmodel.py b/tests/export/test_awq_gptqmodel.py new file mode 100644 index 0000000000..9328ef1cca --- /dev/null +++ b/tests/export/test_awq_gptqmodel.py @@ -0,0 +1,273 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import sys +import torch +import types +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from swift.pipelines.export import quant + + +class TestAwqGPTQModel(unittest.TestCase): + + def setUp(self): + self.engine = quant.QuantEngine.__new__(quant.QuantEngine) + self.engine.args = SimpleNamespace( + quant_method='awq', + quant_bits=4, + group_size=128, + quant_batch_size=2, + dataset=['calibration.jsonl'], + model_dir='/model', + output_dir='/output', + safe_serialization=True, + max_shard_size='5GB', + adapters=[]) + self.engine.model = torch.nn.Linear(2, 2) + self.engine.model.config = SimpleNamespace(quantization_config=None) + self.engine.model.model_meta = SimpleNamespace(is_multimodal=False, additional_saved_files=[]) + self.engine.model.model_info = SimpleNamespace(is_moe_model=False, config=SimpleNamespace()) + self.engine.template = SimpleNamespace() + self.engine.processor = SimpleNamespace( + model_meta=self.engine.model.model_meta, model_info=self.engine.model.model_info) + self.engine._get_quant_dataset = MagicMock(return_value=[torch.tensor([[1, 2, 3]])]) + self.wrapper = MagicMock() + self.engine.awq_model = self.wrapper + self.definition = MagicMock(return_value=self.wrapper) + self.config = MagicMock(return_value=object()) + self.resolver = MagicMock(return_value=self.definition) + gptq = types.ModuleType('gptqmodel') + gptq.AWQConfig = self.config + auto = types.ModuleType('gptqmodel.models.auto') + auto.check_and_get_model_definition = self.resolver + importer = types.ModuleType('gptqmodel.utils.importer') + importer.auto_select_device = MagicMock(return_value='cuda') + self.modules = {'gptqmodel': gptq, 'gptqmodel.models.auto': auto, 'gptqmodel.utils.importer': importer} + + def test_wraps_existing_model_and_saves_awq(self): + with patch.dict(sys.modules, self.modules), patch.object(quant, 'save_checkpoint') as save_checkpoint: + self.engine.quantize() + self.wrapper.quantize.assert_called_once() + self.assertEqual(self.wrapper.quantize.call_args.kwargs['calibration'], [{'input_ids': [1, 2, 3]}]) + self.assertEqual(self.wrapper.quantize.call_args.kwargs['batch_size'], 2) + self.wrapper.save.assert_called_once_with('/output', max_shard_size='5GB') + save_checkpoint.assert_called_once() + + def test_unlimited_awq_batch_uses_all_calibration_blocks(self): + self.engine.args.quant_batch_size = None # ExportArguments converts --quant_batch_size -1 to None. + self.engine._get_quant_dataset.return_value = [torch.tensor([[1, 2]]), torch.tensor([[3, 4]])] + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_called_once_with( + calibration=[{ + 'input_ids': [1, 2] + }, { + 'input_ids': [3, 4] + }], batch_size=2) + + def test_awq_loads_model_once_through_public_api(self): + args = MagicMock() + args.quant_method = 'awq' + args.quant_bits = 4 + args.group_size = 128 + args.model_dir = '/merged-model' + args.torch_dtype = torch.float16 + args.adapters = [] + args.safe_serialization = True + args.new_special_tokens = [] + args.device_map = None + args.task_type = 'causal_lm' + args.tuner_backend = 'peft' + for key in [ + 'model_kwargs', 'rope_scaling', 'max_memory', 'attn_impl', 'experts_impl', 'max_model_len', + 'init_strategy' + ]: + setattr(args, key, None) + args.model_meta = self.engine.model.model_meta + args.model_info = self.engine.model.model_info + args.get_model_processor.return_value = (None, self.engine.processor) + args.get_template.return_value = MagicMock() + self.engine.model.config.use_cache = True + self.wrapper.model = self.engine.model + self.wrapper.quantized = False + gptq = self.modules['gptqmodel'] + gptq.GPTQModel = SimpleNamespace(load=MagicMock(return_value=self.wrapper)) + with patch.dict(sys.modules, self.modules), patch.object(quant, 'prepare_model_template') as prepare, \ + patch.object(quant.HfConfigFactory, 'set_config_attr'): + engine = quant.QuantEngine(args) + prepare.assert_not_called() + args.get_model_processor.assert_called_once_with(load_model=False) + self.config.assert_called_once_with(bits=4, group_size=128, sym=False) + gptq.GPTQModel.load.assert_called_once_with( + '/merged-model', quantize_config=self.config.return_value, trust_remote_code=True, dtype=torch.float16) + self.assertIs(engine.model, self.engine.model) + + def test_awq_does_not_silently_ignore_model_kwargs(self): + args = MagicMock() + args.quant_method = 'awq' + args.quant_bits = 4 + args.adapters = [] + args.safe_serialization = True + args.new_special_tokens = [] + args.device_map = None + args.task_type = 'causal_lm' + args.tuner_backend = 'peft' + args.model_kwargs = {'custom_setting': True} + with self.assertRaisesRegex(ValueError, 'model_kwargs'): + quant.QuantEngine(args) + + def test_rejects_already_quantized_checkpoint(self): + self.engine.model.config.quantization_config = {'quant_method': 'fp8'} + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'already quantized'): + self.engine.awq_model_quantize() + self.definition.assert_not_called() + + def test_rejects_unmerged_adapter(self): + self.engine.args.adapters = ['/adapter'] + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'merge_lora'): + self.engine.awq_model_quantize() + self.definition.assert_not_called() + + def test_rejects_fp8_weights_without_config(self): + self.engine.model.to(dtype=torch.float8_e4m3fn) + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'FP8'): + self.engine.awq_model_quantize() + self.definition.assert_not_called() + + def test_rejects_empty_calibration(self): + self.engine._get_quant_dataset.return_value = [torch.tensor([[]])] + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'calibration'): + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_not_called() + + def test_rejects_multimodal_calibration(self): + self.engine.model.model_meta.is_multimodal = True + self.engine.model.model_info.model_type = 'unsupported_vl' + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'multimodal'): + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_not_called() + + def test_qwen2_vl_calibration_preserves_images_and_text(self): + data = { + 'messages': [{ + 'role': 'user', + 'content': 'Compare to .' + }], + 'images': [{ + 'path': '/red.png', + 'bytes': None + }, { + 'path': '/blue.png', + 'bytes': None + }] + } + messages = quant.QuantEngine._awq_qwen2_vl_conversation(data) + self.assertEqual(messages, [{ + 'role': + 'user', + 'content': [{ + 'type': 'text', + 'text': 'Compare ' + }, { + 'type': 'image', + 'image': '/red.png' + }, { + 'type': 'text', + 'text': ' to ' + }, { + 'type': 'image', + 'image': '/blue.png' + }, { + 'type': 'text', + 'text': '.' + }] + }]) + self.assertEqual(data['messages'][0]['content'], 'Compare to .') + + def test_qwen2_vl_calibration_rejects_mismatched_images(self): + data = {'messages': [{'role': 'user', 'content': ''}], 'images': ['/only-one.png']} + with self.assertRaisesRegex(ValueError, 'image'): + quant.QuantEngine._awq_qwen2_vl_conversation(data) + + def test_qwen2_vl_calibration_rejects_unhandled_metadata(self): + data = { + 'messages': [{ + 'role': 'user', + 'content': 'Where?' + }], + 'images': ['/red.png'], + 'objects': { + 'bbox': [[0, 0, 1, 1]] + } + } + with self.assertRaisesRegex(ValueError, 'objects'): + quant.QuantEngine._awq_qwen2_vl_conversation(data) + + def test_qwen2_vl_calibration_inserts_missing_image_placeholder(self): + data = {'messages': [{'role': 'user', 'content': 'Describe it.'}], 'images': ['/red.png']} + messages = quant.QuantEngine._awq_qwen2_vl_conversation(data) + self.assertEqual(messages[0]['content'], [{ + 'type': 'image', + 'image': '/red.png' + }, { + 'type': 'text', + 'text': 'Describe it.' + }]) + + def test_qwen2_vl_uses_visual_calibration(self): + self.engine.model.model_meta.is_multimodal = True + self.engine.model.model_info.model_type = 'qwen2_vl' + conversation = [{'role': 'user', 'content': [{'type': 'image', 'image': '/red.png'}]}] + self.engine._get_awq_qwen2_vl_dataset = MagicMock(return_value=[conversation]) + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_called_once_with(calibration=[conversation], batch_size=2) + self.engine._get_quant_dataset.assert_not_called() + + def test_qwen2_vl_rejects_custom_system_instead_of_ignoring_it(self): + self.engine.model.model_meta.is_multimodal = True + self.engine.model.model_info.model_type = 'qwen2_vl' + self.engine.args.system = 'custom system' + with self.assertRaisesRegex(ValueError, 'system'): + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_not_called() + + def test_qwen2_vl_applies_max_length_to_gptqmodel_tokens(self): + self.engine.args.dataset_shuffle = False + self.engine.args.get_dataset_kwargs = lambda: {} + self.engine.args.max_length = 3 + self.engine.args.quant_n_samples = 2 + self.engine.template.encode = MagicMock() + raw = [{ + 'messages': [{ + 'role': 'user', + 'content': 'What?' + }], + 'images': ['/red.png'] + }, { + 'messages': [{ + 'role': 'user', + 'content': 'What?' + }], + 'images': ['/blue.png'] + }] + dataset = MagicMock() + dataset.shuffle.return_value = raw + self.wrapper.prepare_dataset.side_effect = [[{ + 'input_ids': torch.tensor([[1, 2, 3, 4]]) + }], [{ + 'input_ids': torch.tensor([[1, 2]]) + }]] + with patch.object(quant, 'load_dataset', return_value=(dataset, None)): + calibration = self.engine._get_awq_qwen2_vl_dataset() + self.assertEqual(len(calibration), 1) + self.assertEqual(calibration[0][0]['content'][0]['image'], '/blue.png') + + def test_requires_safetensors(self): + self.engine.args.safe_serialization = False + with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'safe_serialization'): + self.engine.awq_model_quantize() + self.wrapper.quantize.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/export/test_quant_gpu_integration.py b/tests/export/test_quant_gpu_integration.py new file mode 100644 index 0000000000..349ba3db21 --- /dev/null +++ b/tests/export/test_quant_gpu_integration.py @@ -0,0 +1,36 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Opt-in GPU export regression with a local Qwen3 model and calibration dataset.""" +import json +import os +import pytest +import subprocess +import sys +import torch +from pathlib import Path + + +@pytest.mark.parametrize('method', ['awq', 'gptq', 'gptq_v2']) +def test_qwen3_quantization_export(tmp_path, method): + model = os.environ.get('SWIFT_TEST_QUANT_MODEL') + dataset = os.environ.get('SWIFT_TEST_QUANT_DATASET') + if not model or not dataset or not torch.cuda.is_available(): + pytest.skip('Set SWIFT_TEST_QUANT_MODEL and SWIFT_TEST_QUANT_DATASET and provide a CUDA GPU') + + output_dir = tmp_path / method + script = Path(__file__).resolve().parents[2] / 'swift' / 'cli' / 'export.py' + command = [ + sys.executable, + str(script), '--model', model, '--model_type', 'qwen3', '--template', 'qwen3', '--dataset', dataset, + '--output_dir', + str(output_dir), '--quant_method', method, '--quant_bits', '4', '--group_size', '128', '--quant_n_samples', + '32', '--quant_batch_size', '1', '--max_length', '256' + ] + subprocess.run(command, check=True, timeout=1800) + config = json.loads((output_dir / 'config.json').read_text())['quantization_config'] + assert config['bits'] == 4 + assert config['group_size'] == 128 + assert config['quant_method'] == ('awq' if method == 'awq' else 'gptq') + if method == 'awq': + assert config['zero_point'] is True + assert config['version'].lower() == 'gemm' + assert list(output_dir.glob('*.safetensors')) From ee8eacd0ec2aa53f3def81de6e85f213b6d186a3 Mon Sep 17 00:00:00 2001 From: sunfeng Date: Mon, 2 Nov 2026 09:34:03 +0800 Subject: [PATCH 2/2] Remove Qwen2-VL-specific AWQ calibration AI-Assisted-by: OpenAI Codex --- docs/source/Instruction/Export-and-push.md | 6 +- docs/source_en/Instruction/Export-and-push.md | 6 +- swift/pipelines/export/quant.py | 87 +------------ tests/export/test_awq_gptqmodel.py | 123 +----------------- 4 files changed, 15 insertions(+), 207 deletions(-) diff --git a/docs/source/Instruction/Export-and-push.md b/docs/source/Instruction/Export-and-push.md index c325784932..6b5f9129da 100644 --- a/docs/source/Instruction/Export-and-push.md +++ b/docs/source/Instruction/Export-and-push.md @@ -13,7 +13,7 @@ SWIFT支持AWQ、GPTQ、FP8、BNB模型的量化导出。其中使用AWQ、GPTQ | -------- | ------ | -------- | -------- | | FP8 | ✅ | ✅ | ✅ | | GPTQ | ✅ | ✅ | ✅ | -| AWQ | 部分支持 | ✅ | ✅ | +| AWQ | ❌ | ✅ | ✅ | | BNB | ❌ | ✅ | ✅ | @@ -48,12 +48,12 @@ pip install bitsandbytes -U 我们提供了一系列脚本展现SWIFT的量化导出能力: - 支持[AWQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/awq.sh)/[GPTQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq.sh)/[GPTQ v2](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq_v2.sh)/[BNB](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/bnb.sh)量化导出。 -- 多模态量化: GPTQ支持多模态模型;AWQ目前仅支持Qwen2-VL图像校准,其他模态和模型尚不支持。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm)。 +- 多模态量化: GPTQ支持多模态模型;AWQ目前不支持多模态校准。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm)。 - 更多系列模型的支持: 支持[Bert](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/bert),[Reward Model](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/reward_model)的量化导出。 - 使用SWIFT量化导出的模型支持使用vllm/sglang/lmdeploy进行推理加速;也支持使用QLoRA继续进行SFT/RLHF。 -AWQ 使用 GPTQModel 生成带零点的 GEMM 格式权重。目前支持文本和 Qwen2-VL 图像校准,使用 safetensors 和未量化的模型;LoRA 需要先使用 `--merge_lora true` 合并。AWQ 不支持在导出时添加特殊 token、指定 `--device_map` 或设置自定义模型加载参数。Qwen2-VL 图像校准使用 GPTQModel 自身的 processor 和模型聊天模板,不支持自定义 `--template`、`--system` 或 `--max_pixels`,仅支持 `--truncation_strategy delete`。模型架构须受 GPTQModel 支持,安装时请检查与现有 PyTorch、Transformers 和其他依赖的兼容性。 +AWQ 使用 GPTQModel 生成带零点的 GEMM 格式权重。目前支持文本校准,使用 safetensors 和未量化的模型;LoRA 需要先使用 `--merge_lora true` 合并。AWQ 不支持在导出时添加特殊 token、指定 `--device_map` 或设置自定义模型加载参数。模型架构须受 GPTQModel 支持,安装时请检查与现有 PyTorch、Transformers 和其他依赖的兼容性。 使用 GPTQ v2 量化时,后端可能将对称量化权重保存为 GPTQ v1 格式。导出配置会记录实际保存的格式,以便正确重新加载模型。 diff --git a/docs/source_en/Instruction/Export-and-push.md b/docs/source_en/Instruction/Export-and-push.md index d98bc2303b..03845847f4 100644 --- a/docs/source_en/Instruction/Export-and-push.md +++ b/docs/source_en/Instruction/Export-and-push.md @@ -12,7 +12,7 @@ SWIFT supports quantization exports for AWQ, GPTQ, FP8, and BNB models. AWQ and | ---------------------- | ---------- | ---------------------- | ------------------ | | FP8 | ✅ | ✅ | ✅ | | GPTQ | ✅ | ✅ | ✅ | -| AWQ | Partial | ✅ | ✅ | +| AWQ | ❌ | ✅ | ✅ | | BNB | ❌ | ✅ | ✅ | In addition to the SWIFT installation, the following additional dependencies need to be installed: @@ -48,12 +48,12 @@ If AWQ, SwanLab, or ms-opencompass are needed in the same environment, first con We provide a series of scripts to demonstrate SWIFT's quantization export capabilities: - Supports [AWQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/awq.sh)/[GPTQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq.sh)/[GPTQ v2](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq_v2.sh)/[BNB](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/bnb.sh) quantization exports. -- Multimodal quantization: GPTQ supports multimodal models; AWQ currently supports image calibration for Qwen2-VL only. Refer to [here](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm). +- Multimodal quantization: GPTQ supports multimodal models; AWQ does not currently support multimodal calibration. Refer to [here](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm). - Support for more model series: Supports quantization exports for [BERT](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/bert) and [Reward Model](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/reward_model). - Models exported with SWIFT's quantization support inference acceleration using vllm/sglang/lmdeploy; they also support further SFT/RLHF using QLoRA. -AWQ uses GPTQModel to produce GEMM weights with zero points. It supports text and Qwen2-VL image calibration, safetensors, and unquantized source models; merge LoRA adapters first using `--merge_lora true`. Adding special tokens, setting `--device_map`, or passing custom model loading arguments during AWQ export are not supported. Qwen2-VL image calibration uses GPTQModel's processor and model chat template, does not support custom `--template`, `--system`, or `--max_pixels`, and requires `--truncation_strategy delete`. The model architecture must be supported by GPTQModel. Check compatibility with the existing PyTorch, Transformers, and other dependencies when installing it. +AWQ uses GPTQModel to produce GEMM weights with zero points. It supports text calibration, safetensors, and unquantized source models; merge LoRA adapters first using `--merge_lora true`. Adding special tokens, setting `--device_map`, or passing custom model loading arguments during AWQ export are not supported. The model architecture must be supported by GPTQModel. Check compatibility with the existing PyTorch, Transformers, and other dependencies when installing it. GPTQ backends may save symmetric weights in GPTQ v1 format even when GPTQ v2 is used during quantization. The exported configuration records the saved format so the model reloads correctly. diff --git a/swift/pipelines/export/quant.py b/swift/pipelines/export/quant.py index 6df4db7730..0e567185ba 100644 --- a/swift/pipelines/export/quant.py +++ b/swift/pipelines/export/quant.py @@ -4,9 +4,7 @@ import transformers from collections import defaultdict from contextlib import contextmanager -from io import BytesIO from packaging import version -from PIL import Image from tqdm import tqdm from typing import Dict, List, Optional @@ -166,72 +164,6 @@ def _get_quant_dataset(self, *args, **kwargs): res.append(torch.tensor(input_ids)[None]) return res - @staticmethod - def _awq_qwen2_vl_conversation(data): - if data.get('videos') or data.get('audios'): - raise ValueError('Qwen2-VL AWQ currently supports image calibration only.') - for key in ('objects', 'tools', 'chat_template_kwargs'): - if data.get(key): - raise ValueError(f'Qwen2-VL AWQ does not support calibration field: {key}.') - messages = data.get('messages') or [] - images = list(data.get('images') or []) - if not messages: - raise ValueError('Qwen2-VL AWQ calibration requires messages.') - if any(not isinstance(message.get('content'), str) for message in messages): - raise ValueError('Qwen2-VL AWQ calibration requires text messages with image placeholders.') - if any(message.get('role') not in {'system', 'user', 'assistant'} for message in messages): - raise ValueError('Qwen2-VL AWQ does not support tool messages in calibration.') - placeholder_count = sum(message['content'].count('') for message in messages) - if placeholder_count > len(images): - raise ValueError('More image placeholders than calibration images.') - missing = len(images) - placeholder_count - first = next((i for i, message in enumerate(messages) if message['role'] != 'system'), 0) - image_idx = 0 - result = [] - for index, message in enumerate(messages): - content = ('' * missing if index == first else '') + message['content'] - parts = content.split('') - blocks = [] - for part_idx, text in enumerate(parts): - if part_idx: - value = images[image_idx] - image_idx += 1 - if isinstance(value, dict): - if value.get('bytes') is not None: - value = Image.open(BytesIO(value['bytes'])).convert('RGB') - elif value.get('path'): - value = value['path'] - else: - raise ValueError('Calibration image requires bytes or a path.') - blocks.append({'type': 'image', 'image': value}) - if text: - blocks.append({'type': 'text', 'text': text}) - result.append({'role': message['role'], 'content': blocks}) - return result - - def _get_awq_qwen2_vl_dataset(self): - args = self.args - dataset = load_dataset( - args.dataset, split_dataset_ratio=0, shuffle=args.dataset_shuffle, - **args.get_dataset_kwargs())[0].shuffle() - samples = [] - for data in dataset: - conversation = self._awq_qwen2_vl_conversation(data) - try: - self.template.encode(data) - except MaxLengthError: - continue - # GPTQModel applies its own processor and chat template for this model. - # Check that tokenization as well, since its length can differ from Swift's. - if args.max_length is not None: - prepared = self.awq_model.prepare_dataset([conversation], batch_size=1) - if any(item['input_ids'].shape[-1] > args.max_length for item in prepared): - continue - samples.append(conversation) - if len(samples) >= args.quant_n_samples: - break - return samples - def awq_model_quantize(self): args = self.args if getattr(self.model.config, 'quantization_config', None): @@ -245,21 +177,10 @@ def awq_model_quantize(self): logger.info(f'Quantization dataset: {args.dataset}') if self.model.model_meta.is_multimodal: - if self.model.model_info.model_type != 'qwen2_vl': - raise ValueError('GPTQModel AWQ multimodal calibration currently supports only Qwen2-VL images.') - if getattr(args, 'system', None) is not None: - raise ValueError('Qwen2-VL AWQ does not support a custom --system during export.') - if getattr(args, 'template', None) not in {None, 'qwen2_vl'}: - raise ValueError('Qwen2-VL AWQ does not support a custom --template during export.') - if getattr(args, 'max_pixels', None) is not None: - raise ValueError('Qwen2-VL AWQ does not support --max_pixels during export.') - if getattr(args, 'truncation_strategy', 'delete') != 'delete': - raise ValueError('Qwen2-VL AWQ supports only truncation_strategy=delete.') - calibration = self._get_awq_qwen2_vl_dataset() - else: - calibration = [{ - 'input_ids': block.reshape(-1).tolist() - } for block in self._get_quant_dataset() if block.numel()] + raise ValueError('GPTQModel AWQ export does not currently support multimodal calibration.') + calibration = [{ + 'input_ids': block.reshape(-1).tolist() + } for block in self._get_quant_dataset() if block.numel()] if not calibration: raise ValueError('AWQ calibration dataset contains no usable tokens.') logger.info('Start quantizing the model...') diff --git a/tests/export/test_awq_gptqmodel.py b/tests/export/test_awq_gptqmodel.py index 9328ef1cca..28b875d1ca 100644 --- a/tests/export/test_awq_gptqmodel.py +++ b/tests/export/test_awq_gptqmodel.py @@ -142,126 +142,13 @@ def test_rejects_empty_calibration(self): def test_rejects_multimodal_calibration(self): self.engine.model.model_meta.is_multimodal = True - self.engine.model.model_info.model_type = 'unsupported_vl' - with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'multimodal'): - self.engine.awq_model_quantize() - self.wrapper.quantize.assert_not_called() - - def test_qwen2_vl_calibration_preserves_images_and_text(self): - data = { - 'messages': [{ - 'role': 'user', - 'content': 'Compare to .' - }], - 'images': [{ - 'path': '/red.png', - 'bytes': None - }, { - 'path': '/blue.png', - 'bytes': None - }] - } - messages = quant.QuantEngine._awq_qwen2_vl_conversation(data) - self.assertEqual(messages, [{ - 'role': - 'user', - 'content': [{ - 'type': 'text', - 'text': 'Compare ' - }, { - 'type': 'image', - 'image': '/red.png' - }, { - 'type': 'text', - 'text': ' to ' - }, { - 'type': 'image', - 'image': '/blue.png' - }, { - 'type': 'text', - 'text': '.' - }] - }]) - self.assertEqual(data['messages'][0]['content'], 'Compare to .') - - def test_qwen2_vl_calibration_rejects_mismatched_images(self): - data = {'messages': [{'role': 'user', 'content': ''}], 'images': ['/only-one.png']} - with self.assertRaisesRegex(ValueError, 'image'): - quant.QuantEngine._awq_qwen2_vl_conversation(data) - - def test_qwen2_vl_calibration_rejects_unhandled_metadata(self): - data = { - 'messages': [{ - 'role': 'user', - 'content': 'Where?' - }], - 'images': ['/red.png'], - 'objects': { - 'bbox': [[0, 0, 1, 1]] - } - } - with self.assertRaisesRegex(ValueError, 'objects'): - quant.QuantEngine._awq_qwen2_vl_conversation(data) - - def test_qwen2_vl_calibration_inserts_missing_image_placeholder(self): - data = {'messages': [{'role': 'user', 'content': 'Describe it.'}], 'images': ['/red.png']} - messages = quant.QuantEngine._awq_qwen2_vl_conversation(data) - self.assertEqual(messages[0]['content'], [{ - 'type': 'image', - 'image': '/red.png' - }, { - 'type': 'text', - 'text': 'Describe it.' - }]) - - def test_qwen2_vl_uses_visual_calibration(self): - self.engine.model.model_meta.is_multimodal = True - self.engine.model.model_info.model_type = 'qwen2_vl' - conversation = [{'role': 'user', 'content': [{'type': 'image', 'image': '/red.png'}]}] - self.engine._get_awq_qwen2_vl_dataset = MagicMock(return_value=[conversation]) - self.engine.awq_model_quantize() - self.wrapper.quantize.assert_called_once_with(calibration=[conversation], batch_size=2) - self.engine._get_quant_dataset.assert_not_called() - - def test_qwen2_vl_rejects_custom_system_instead_of_ignoring_it(self): - self.engine.model.model_meta.is_multimodal = True - self.engine.model.model_info.model_type = 'qwen2_vl' - self.engine.args.system = 'custom system' - with self.assertRaisesRegex(ValueError, 'system'): - self.engine.awq_model_quantize() + for model_type in ('qwen2_vl', 'qwen3_vl', 'unsupported_vl'): + with self.subTest(model_type=model_type): + self.engine.model.model_info.model_type = model_type + with self.assertRaisesRegex(ValueError, 'multimodal'): + self.engine.awq_model_quantize() self.wrapper.quantize.assert_not_called() - def test_qwen2_vl_applies_max_length_to_gptqmodel_tokens(self): - self.engine.args.dataset_shuffle = False - self.engine.args.get_dataset_kwargs = lambda: {} - self.engine.args.max_length = 3 - self.engine.args.quant_n_samples = 2 - self.engine.template.encode = MagicMock() - raw = [{ - 'messages': [{ - 'role': 'user', - 'content': 'What?' - }], - 'images': ['/red.png'] - }, { - 'messages': [{ - 'role': 'user', - 'content': 'What?' - }], - 'images': ['/blue.png'] - }] - dataset = MagicMock() - dataset.shuffle.return_value = raw - self.wrapper.prepare_dataset.side_effect = [[{ - 'input_ids': torch.tensor([[1, 2, 3, 4]]) - }], [{ - 'input_ids': torch.tensor([[1, 2]]) - }]] - with patch.object(quant, 'load_dataset', return_value=(dataset, None)): - calibration = self.engine._get_awq_qwen2_vl_dataset() - self.assertEqual(len(calibration), 1) - self.assertEqual(calibration[0][0]['content'][0]['image'], '/blue.png') - def test_requires_safetensors(self): self.engine.args.safe_serialization = False with patch.dict(sys.modules, self.modules), self.assertRaisesRegex(ValueError, 'safe_serialization'):