diff --git a/modelscope/__init__.py b/modelscope/__init__.py index ac362be1..162673a0 100644 --- a/modelscope/__init__.py +++ b/modelscope/__init__.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from .utils.hub import read_config, create_model_if_not_exist from .utils.logger import get_logger from .utils.constant import Tasks - from .utils.hf_util import AutoConfig, GenerationConfig + from .utils.hf_util import AutoConfig, GenerationConfig, GPTQConfig, BitsAndBytesConfig from .utils.hf_util import (AutoModel, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoModelForSequenceClassification, @@ -74,8 +74,9 @@ else: 'utils.logger': ['get_logger'], 'utils.constant': ['Tasks'], 'utils.hf_util': [ - 'AutoConfig', 'GenerationConfig', 'AutoModel', - 'AutoModelForCausalLM', 'AutoModelForSeq2SeqLM', 'AutoTokenizer', + 'AutoConfig', 'GenerationConfig', 'AutoModel', 'GPTQConfig', + 'BitsAndBytesConfig', 'AutoModelForCausalLM', + 'AutoModelForSeq2SeqLM', 'AutoTokenizer', 'AutoModelForSequenceClassification', 'AutoModelForTokenClassification' ], diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 3830bb52..79ac2c33 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -136,16 +136,18 @@ class EfficientStableDiffusion(TorchModel): 'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10 adapter_config_dict = {} dim_list = [320, 640, 1280] - target_modules_list = [r"(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)", - r"(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)", - r"(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)"] + target_modules_list = [ + r'(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)', + r'(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)', + r'(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)' + ] for dim, target_modules in zip(dim_list, target_modules_list): adapter_config = AdapterConfig( dim=dim, hidden_pos=0, target_modules=target_modules, adapter_length=adapter_length) - adapter_config_dict[f"adapter_{dim}"] = adapter_config + adapter_config_dict[f'adapter_{dim}'] = adapter_config self.unet = Swift.prepare_model(self.unet, adapter_config_dict) elif tuner_name == 'swift-prompt': if not is_swift_available(): diff --git a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py index 6267fb9d..06f87287 100644 --- a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py +++ b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py @@ -158,9 +158,9 @@ class StableDiffusion(TorchModel): config: Optional[dict] = None, save_config_function: Callable = save_configuration, **kwargs): - config['pipeline']['type'] = 'diffusers-stable-diffusion' # Skip copying the original weights for lora and dreambooth method if self.lora_tune or self.dreambooth_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion' pass else: super().save_pretrained(target_folder, save_checkpoint_names, diff --git a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py index 23ad6676..e0fa5070 100644 --- a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py +++ b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py @@ -244,9 +244,9 @@ class StableDiffusionXL(TorchModel): config: Optional[dict] = None, save_config_function: Callable = save_configuration, **kwargs): - config['pipeline']['type'] = 'diffusers-stable-diffusion-xl' # Skip copying the original weights for lora and dreambooth method - if self.lora_tune or self.dreambooth_tune: + if self.lora_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion-xl' pass else: super().save_pretrained(target_folder, save_checkpoint_names, diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index ecd03079..f825412c 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -161,6 +161,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): decoding_ind=self.cmd['decoding_ind'], decoding_mode=self.cmd['decoding_mode'], fake_streaming=self.cmd['fake_streaming'], + model_lang=self.cmd['model_lang'], **kwargs, ) @@ -305,7 +306,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'idx_text': '', 'sampled_ids': 'seq2seq/sampled_ids', 'sampled_lengths': 'seq2seq/sampled_lengths', - 'lang': 'zh-cn', + 'model_lang': outputs['model_lang'], 'code_base': outputs['code_base'], 'mode': outputs['mode'], 'fs': { @@ -357,16 +358,16 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): if outputs.__contains__('mvn_file'): cmd['cmvn_file'] = outputs['mvn_file'] model_config = self.model_cfg['model_config'] - if model_config.__contains__('vad_model') and self.vad_model != '': + if model_config.__contains__('vad_model') and self.vad_model is None: self.vad_model = model_config['vad_model'] if model_config.__contains__('vad_model_revision'): self.vad_model_revision = model_config['vad_model_revision'] - if model_config.__contains__('punc_model') and self.punc_model != '': + if model_config.__contains__('punc_model') and self.punc_model is None: self.punc_model = model_config['punc_model'] if model_config.__contains__('punc_model_revision'): self.punc_model_revision = model_config['punc_model_revision'] if model_config.__contains__( - 'timestamp_model') and self.timestamp_model != '': + 'timestamp_model') and self.timestamp_model is None: self.timestamp_model = model_config['timestamp_model'] if model_config.__contains__('timestamp_model_revision'): self.timestamp_model_revision = model_config[ diff --git a/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py b/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py index 04fd5910..bb48fae5 100644 --- a/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py +++ b/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py @@ -12,7 +12,7 @@ import numpy as np import torch import torch.nn.functional as F from diffusers import LMSDiscreteScheduler, StableDiffusionPipeline -from diffusers.models.cross_attention import CrossAttention +from diffusers.models.attention_processor import Attention from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import \ StableDiffusionPipelineOutput from PIL import Image @@ -245,7 +245,7 @@ class Cones2AttnProcessor: super().__init__() def __call__(self, - attn: CrossAttention, + attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None): diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py index e5345543..a1f60327 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py @@ -17,6 +17,7 @@ from modelscope.pipelines.builder import PIPELINES from modelscope.pipelines.multi_modal.diffusers_wrapped.diffusers_pipeline import \ DiffusersPipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import is_swift_available @PIPELINES.register_module( @@ -38,9 +39,11 @@ class StableDiffusionPipeline(DiffusersPipeline): custom_dir: custom diffusion weight dir for unet. modifier_token: token to use as a modifier for the concept of custom diffusion. use_safetensors: load safetensors weights. + use_swift: Whether to use swift lora dir for unet. """ use_safetensors = kwargs.pop('use_safetensors', False) torch_type = kwargs.pop('torch_type', torch.float32) + use_swift = kwargs.pop('use_swift', False) # check custom diffusion input value if custom_dir is None and modifier_token is not None: raise ValueError( @@ -58,7 +61,17 @@ class StableDiffusionPipeline(DiffusersPipeline): # load lora moudle to unet if lora_dir is not None: assert os.path.exists(lora_dir), f"{lora_dir} isn't exist" - self.pipeline.unet.load_attn_procs(lora_dir) + if use_swift: + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use efficient_tuners.' + ) + from swift import Swift + self.pipeline.unet = Swift.from_pretrained( + self.pipeline.unet, lora_dir) + else: + self.pipeline.unet.load_attn_procs(lora_dir) + # load custom diffusion to unet if custom_dir is not None: assert os.path.exists(custom_dir), f"{custom_dir} isn't exist" diff --git a/modelscope/preprocessors/asr.py b/modelscope/preprocessors/asr.py index 4696c675..4a24ffb2 100644 --- a/modelscope/preprocessors/asr.py +++ b/modelscope/preprocessors/asr.py @@ -96,6 +96,10 @@ class WavToScp(Preprocessor): else: mode = None inputs['mode'] = mode + if 'lang' in inputs['model_config']: + inputs['model_lang'] = inputs['model_config']['lang'] + else: + inputs['model_lang'] = 'zh-cn' if inputs['model_type'] == Frameworks.torch: assert inputs['model_config'].__contains__( diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index 54ad6e97..2f2ff025 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -54,12 +54,14 @@ class DiffusionImageGenerationPreprocessor(Preprocessor): self.preprocessor_std = kwargs.pop('std', [0.5]) self.preprocessor_image_keys = set(kwargs.pop('image_keys', [])) self.center_crop = kwargs.pop('center_crop', True) - + self.transform_input = transforms.Compose([ transforms.Resize( self.preprocessor_resolution, interpolation=transforms.InterpolationMode.BILINEAR), - transforms.CenterCrop(self.preprocessor_resolution) if self.center_crop else transforms.RandomCrop(self.preprocessor_resolution), + transforms.CenterCrop(self.preprocessor_resolution) + if self.center_crop else transforms.RandomCrop( + self.preprocessor_resolution), transforms.ToTensor(), transforms.Normalize(self.preprocessor_mean, self.preprocessor_std), diff --git a/modelscope/trainers/hooks/swift/__init__.py b/modelscope/trainers/hooks/swift/__init__.py index daf16f92..7fa1d057 100644 --- a/modelscope/trainers/hooks/swift/__init__.py +++ b/modelscope/trainers/hooks/swift/__init__.py @@ -1 +1 @@ -from .swift_hook import SwiftHook \ No newline at end of file +from .swift_hook import SwiftHook diff --git a/modelscope/trainers/hooks/swift/swift_hook.py b/modelscope/trainers/hooks/swift/swift_hook.py index 262dd483..b03b8edc 100644 --- a/modelscope/trainers/hooks/swift/swift_hook.py +++ b/modelscope/trainers/hooks/swift/swift_hook.py @@ -9,12 +9,12 @@ from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import \ LoadCheckpointHook from modelscope.trainers.hooks.hook import Hook -from modelscope.utils.import_utils import is_swift_available from modelscope.utils.checkpoint import save_configuration +from modelscope.utils.import_utils import is_swift_available class SwiftCheckpointProcessor(CheckpointProcessor): - + _BIN_FILE_DIR = 'model' SWIFT_SAVE_SUFFIX = '_swift' @@ -37,7 +37,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor): save_configuration(self.output_dir, self.config) for pop_key in [ - 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' + 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' ]: if config.safe_get('train.checkpoint.period.' + pop_key) is not None: @@ -57,9 +57,10 @@ class SwiftCheckpointProcessor(CheckpointProcessor): _swift_output_dir = output_dir + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX model.save_pretrained( save_directory=_swift_output_dir, - safe_serialization=config.safe_get('train.checkpoint.safe_serialization', False), - adapter_name=config.safe_get('train.checkpoint.adapter_name', 'default') - ) + safe_serialization=config.safe_get( + 'train.checkpoint.safe_serialization', False), + adapter_name=config.safe_get( + 'train.checkpoint.adapter_name', 'default')) else: model.save_pretrained( output_dir, @@ -109,7 +110,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor): @HOOKS.register_module(module_name=Hooks.SwiftHook) class SwiftHook(Hook): - + _BIN_FILE_DIR = 'model' def __init__(self): diff --git a/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py index 68d7c689..b38e0e42 100644 --- a/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py +++ b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py @@ -1,4 +1,5 @@ # Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os from typing import Union import torch @@ -7,16 +8,46 @@ from torch import nn from modelscope.metainfo import Trainers from modelscope.models.base import Model, TorchModel from modelscope.trainers.builder import TRAINERS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import CheckpointHook +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor from modelscope.trainers.optimizer.builder import build_optimizer from modelscope.trainers.trainer import EpochBasedTrainer from modelscope.utils.config import ConfigDict +class SwiftDiffusionCheckpointProcessor(CheckpointProcessor): + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None, + save_optimizers=True): + """Save the state dict for swift lora tune model. + """ + trainer.model.unet.save_pretrained(os.path.join(output_dir)) + + @TRAINERS.register_module(module_name=Trainers.stable_diffusion) class StableDiffusionTrainer(EpochBasedTrainer): def __init__(self, *args, **kwargs): + """Stable Diffusion trainers for fine-tuning. + + Args: + use_swift: Whether to use swift. + + """ super().__init__(*args, **kwargs) + use_swift = kwargs.pop('use_swift', False) + + # set swift lora save checkpoint processor + if use_swift: + ckpt_hook = list( + filter(lambda hook: isinstance(hook, CheckpointHook), + self.hooks))[0] + ckpt_hook.set_processor(SwiftDiffusionCheckpointProcessor()) def build_optimizer(self, cfg: ConfigDict, default_args: dict = None): try: diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index a3707918..25f948bc 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -181,8 +181,20 @@ class EpochBasedTrainer(BaseTrainer): compile_options = {} self.model = compile_model(self.model, **compile_options) - if 'work_dir' in kwargs: + if kwargs.get('work_dir', None) is not None: self.work_dir = kwargs['work_dir'] + if 'train' not in self.cfg: + self.cfg['train'] = ConfigDict() + self.cfg['train']['work_dir'] = self.work_dir + if 'checkpoint' in self.cfg['train']: + if 'period' in self.cfg['train']['checkpoint']: + self.cfg['train']['checkpoint']['period'][ + 'save_dir'] = self.work_dir + if 'best' in self.cfg['train']['checkpoint']: + self.cfg['train']['checkpoint']['best'][ + 'save_dir'] = self.work_dir + if 'logging' in self.cfg['train']: + self.cfg['train']['logging']['out_dir'] = self.work_dir else: self.work_dir = self.cfg.train.get('work_dir', './work_dir') diff --git a/modelscope/utils/hf_util.py b/modelscope/utils/hf_util.py index fd367847..6ef98ccf 100644 --- a/modelscope/utils/hf_util.py +++ b/modelscope/utils/hf_util.py @@ -13,6 +13,7 @@ from transformers import \ from transformers import \ AutoModelForTokenClassification as AutoModelForTokenClassificationHF from transformers import AutoTokenizer as AutoTokenizerHF +from transformers import BitsAndBytesConfig as BitsAndBytesConfigHF from transformers import GenerationConfig as GenerationConfigHF from transformers import (PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase) @@ -22,6 +23,11 @@ from transformers.models.auto.tokenization_auto import ( from modelscope import snapshot_download from modelscope.utils.constant import Invoke +try: + from transformers import GPTQConfig as GPTQConfigHF +except ImportError: + GPTQConfigHF = None + def user_agent(invoked_by=None): if invoked_by is None: @@ -85,12 +91,13 @@ def check_hf_code(model_dir: str, auto_class: type, raise FileNotFoundError(f'{config_path} is not found') config_dict = PretrainedConfig.get_config_dict(config_path)[0] auto_class_name = auto_class.__name__ + if auto_class is AutoTokenizerHF: + tokenizer_config = get_tokenizer_config(model_dir) # load from repo if trust_remote_code: has_remote_code = False if auto_class is AutoTokenizerHF: - tokenizer_config_dict = get_tokenizer_config(model_dir) - auto_map = tokenizer_config_dict.get('auto_map', None) + auto_map = tokenizer_config.get('auto_map', None) if auto_map is not None: module_name = auto_map.get(auto_class_name, None) if module_name is not None: @@ -123,6 +130,9 @@ def check_hf_code(model_dir: str, auto_class: type, f'{model_type} not found in HF `CONFIG_MAPPING`{trust_remote_code_info}' ) elif auto_class is AutoTokenizerHF: + tokenizer_class = tokenizer_config.get('tokenizer_class') + if tokenizer_class is not None: + return if model_type not in TOKENIZER_MAPPING_NAMES: raise ValueError( f'{model_type} not found in HF `TOKENIZER_MAPPING_NAMES`{trust_remote_code_info}' @@ -199,3 +209,5 @@ AutoConfig = get_wrapped_class( AutoConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors']) GenerationConfig = get_wrapped_class( GenerationConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors']) +GPTQConfig = GPTQConfigHF +BitsAndBytesConfig = BitsAndBytesConfigHF diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 3d39514a..b4485830 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -9,7 +9,6 @@ import os import pkgutil import shutil import sys -import venv from contextlib import contextmanager from fnmatch import fnmatch from pathlib import Path @@ -1144,6 +1143,7 @@ class EnvsManager(object): cfg = read_config(model_dir) self.plugins = cfg.get('plugins', []) self.allow_remote = cfg.get('allow_remote', False) + import venv self.env_builder = venv.EnvBuilder( system_site_packages=True, clear=False, diff --git a/tests/utils/test_hf_util.py b/tests/utils/test_hf_util.py index 7c10cca6..fcbaf50c 100644 --- a/tests/utils/test_hf_util.py +++ b/tests/utils/test_hf_util.py @@ -25,6 +25,10 @@ class HFUtilTest(unittest.TestCase): self.assertEqual(tokenizer.model_max_length, 4096) self.assertFalse(tokenizer.is_fast) + def test_quantization_import(self): + from modelscope import GPTQConfig, BitsAndBytesConfig + self.assertTrue(BitsAndBytesConfig is not None) + def test_auto_model(self): model = AutoModelForCausalLM.from_pretrained( 'baichuan-inc/baichuan-7B', trust_remote_code=True)