Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-02-24 20:19:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

ok Merge branch 'master' of github.com:modelscope/modelscope into dev/reset_oss_timeout

Browse Source
This commit is contained in:
xingjun.wang
2023-10-09 15:58:02 +08:00
parent 6472426451 43046a719b
commit d77f2d1122
16 changed files with 114 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modelscope/__init__.py
View File

@@ -27,7 +27,7 @@ if TYPE_CHECKING:
from .utils.hub import read_config, create_model_if_not_exist
from .utils.logger import get_logger
from .utils.constant import Tasks
from .utils.hf_util import AutoConfig, GenerationConfig
from .utils.hf_util import AutoConfig, GenerationConfig, GPTQConfig, BitsAndBytesConfig
from .utils.hf_util import (AutoModel, AutoModelForCausalLM,
AutoModelForSeq2SeqLM,
AutoModelForSequenceClassification,
@@ -74,8 +74,9 @@ else:
'utils.logger': ['get_logger'],
'utils.constant': ['Tasks'],
'utils.hf_util': [
'AutoConfig', 'GenerationConfig', 'AutoModel',
'AutoModelForCausalLM', 'AutoModelForSeq2SeqLM', 'AutoTokenizer',
'AutoConfig', 'GenerationConfig', 'AutoModel', 'GPTQConfig',
'BitsAndBytesConfig', 'AutoModelForCausalLM',
'AutoModelForSeq2SeqLM', 'AutoTokenizer',
'AutoModelForSequenceClassification',
'AutoModelForTokenClassification'
],

10
modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py
View File

@@ -136,16 +136,18 @@ class EfficientStableDiffusion(TorchModel):
'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10
adapter_config_dict = {}
dim_list = [320, 640, 1280]
target_modules_list = [r"(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)",
r"(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)",
r"(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)"]
target_modules_list = [
r'(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)',
r'(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)',
r'(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)'
]
for dim, target_modules in zip(dim_list, target_modules_list):
adapter_config = AdapterConfig(
dim=dim,
hidden_pos=0,
target_modules=target_modules,
adapter_length=adapter_length)
adapter_config_dict[f"adapter_{dim}"] = adapter_config
adapter_config_dict[f'adapter_{dim}'] = adapter_config
self.unet = Swift.prepare_model(self.unet, adapter_config_dict)
elif tuner_name == 'swift-prompt':
if not is_swift_available():

2
modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py
View File

@@ -158,9 +158,9 @@ class StableDiffusion(TorchModel):
config: Optional[dict] = None,
save_config_function: Callable = save_configuration,
**kwargs):
config['pipeline']['type'] = 'diffusers-stable-diffusion'
# Skip copying the original weights for lora and dreambooth method
if self.lora_tune or self.dreambooth_tune:
config['pipeline']['type'] = 'diffusers-stable-diffusion'
pass
else:
super().save_pretrained(target_folder, save_checkpoint_names,

4
modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py
View File

@@ -244,9 +244,9 @@ class StableDiffusionXL(TorchModel):
config: Optional[dict] = None,
save_config_function: Callable = save_configuration,
**kwargs):
config['pipeline']['type'] = 'diffusers-stable-diffusion-xl'
# Skip copying the original weights for lora and dreambooth method
if self.lora_tune or self.dreambooth_tune:
if self.lora_tune:
config['pipeline']['type'] = 'diffusers-stable-diffusion-xl'
pass
else:
super().save_pretrained(target_folder, save_checkpoint_names,

9
modelscope/pipelines/audio/asr_inference_pipeline.py
View File

@@ -161,6 +161,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
decoding_ind=self.cmd['decoding_ind'],
decoding_mode=self.cmd['decoding_mode'],
fake_streaming=self.cmd['fake_streaming'],
model_lang=self.cmd['model_lang'],
**kwargs,
)
@@ -305,7 +306,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
'idx_text': '',
'sampled_ids': 'seq2seq/sampled_ids',
'sampled_lengths': 'seq2seq/sampled_lengths',
'lang': 'zh-cn',
'model_lang': outputs['model_lang'],
'code_base': outputs['code_base'],
'mode': outputs['mode'],
'fs': {
@@ -357,16 +358,16 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
if outputs.__contains__('mvn_file'):
cmd['cmvn_file'] = outputs['mvn_file']
model_config = self.model_cfg['model_config']
if model_config.__contains__('vad_model') and self.vad_model != '':
if model_config.__contains__('vad_model') and self.vad_model is None:
self.vad_model = model_config['vad_model']
if model_config.__contains__('vad_model_revision'):
self.vad_model_revision = model_config['vad_model_revision']
if model_config.__contains__('punc_model') and self.punc_model != '':
if model_config.__contains__('punc_model') and self.punc_model is None:
self.punc_model = model_config['punc_model']
if model_config.__contains__('punc_model_revision'):
self.punc_model_revision = model_config['punc_model_revision']
if model_config.__contains__(
'timestamp_model') and self.timestamp_model != '':
'timestamp_model') and self.timestamp_model is None:
self.timestamp_model = model_config['timestamp_model']
if model_config.__contains__('timestamp_model_revision'):
self.timestamp_model_revision = model_config[

4
modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py
View File

@@ -12,7 +12,7 @@ import numpy as np
import torch
import torch.nn.functional as F
from diffusers import LMSDiscreteScheduler, StableDiffusionPipeline
from diffusers.models.cross_attention import CrossAttention
from diffusers.models.attention_processor import Attention
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import \
StableDiffusionPipelineOutput
from PIL import Image
@@ -245,7 +245,7 @@ class Cones2AttnProcessor:
super().__init__()
def __call__(self,
attn: CrossAttention,
attn: Attention,
hidden_states,
encoder_hidden_states=None,
attention_mask=None):

15
modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py
View File

@@ -17,6 +17,7 @@ from modelscope.pipelines.builder import PIPELINES
from modelscope.pipelines.multi_modal.diffusers_wrapped.diffusers_pipeline import \
DiffusersPipeline
from modelscope.utils.constant import Tasks
from modelscope.utils.import_utils import is_swift_available
@PIPELINES.register_module(
@@ -38,9 +39,11 @@ class StableDiffusionPipeline(DiffusersPipeline):
custom_dir: custom diffusion weight dir for unet.
modifier_token: token to use as a modifier for the concept of custom diffusion.
use_safetensors: load safetensors weights.
use_swift: Whether to use swift lora dir for unet.
"""
use_safetensors = kwargs.pop('use_safetensors', False)
torch_type = kwargs.pop('torch_type', torch.float32)
use_swift = kwargs.pop('use_swift', False)
# check custom diffusion input value
if custom_dir is None and modifier_token is not None:
raise ValueError(
@@ -58,7 +61,17 @@ class StableDiffusionPipeline(DiffusersPipeline):
# load lora moudle to unet
if lora_dir is not None:
assert os.path.exists(lora_dir), f"{lora_dir} isn't exist"
self.pipeline.unet.load_attn_procs(lora_dir)
if use_swift:
if not is_swift_available():
raise ValueError(
'Please install swift by `pip install ms-swift` to use efficient_tuners.'
)
from swift import Swift
self.pipeline.unet = Swift.from_pretrained(
self.pipeline.unet, lora_dir)
else:
self.pipeline.unet.load_attn_procs(lora_dir)
# load custom diffusion to unet
if custom_dir is not None:
assert os.path.exists(custom_dir), f"{custom_dir} isn't exist"

4
modelscope/preprocessors/asr.py
View File

@@ -96,6 +96,10 @@ class WavToScp(Preprocessor):
else:
mode = None
inputs['mode'] = mode
if 'lang' in inputs['model_config']:
inputs['model_lang'] = inputs['model_config']['lang']
else:
inputs['model_lang'] = 'zh-cn'
if inputs['model_type'] == Frameworks.torch:
assert inputs['model_config'].__contains__(

6
modelscope/preprocessors/multi_modal.py
View File

@@ -54,12 +54,14 @@ class DiffusionImageGenerationPreprocessor(Preprocessor):
self.preprocessor_std = kwargs.pop('std', [0.5])
self.preprocessor_image_keys = set(kwargs.pop('image_keys', []))
self.center_crop = kwargs.pop('center_crop', True)
self.transform_input = transforms.Compose([
transforms.Resize(
self.preprocessor_resolution,
interpolation=transforms.InterpolationMode.BILINEAR),
transforms.CenterCrop(self.preprocessor_resolution) if self.center_crop else transforms.RandomCrop(self.preprocessor_resolution),
transforms.CenterCrop(self.preprocessor_resolution)
if self.center_crop else transforms.RandomCrop(
self.preprocessor_resolution),
transforms.ToTensor(),
transforms.Normalize(self.preprocessor_mean,
self.preprocessor_std),

2
modelscope/trainers/hooks/swift/__init__.py
View File

@@ -1 +1 @@
from .swift_hook import SwiftHook
from .swift_hook import SwiftHook

15
modelscope/trainers/hooks/swift/swift_hook.py
View File

@@ -9,12 +9,12 @@ from modelscope.trainers.hooks.checkpoint.checkpoint_hook import (
from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import \
LoadCheckpointHook
from modelscope.trainers.hooks.hook import Hook
from modelscope.utils.import_utils import is_swift_available
from modelscope.utils.checkpoint import save_configuration
from modelscope.utils.import_utils import is_swift_available
class SwiftCheckpointProcessor(CheckpointProcessor):
_BIN_FILE_DIR = 'model'
SWIFT_SAVE_SUFFIX = '_swift'
@@ -37,7 +37,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor):
save_configuration(self.output_dir, self.config)
for pop_key in [
'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub'
'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub'
]:
if config.safe_get('train.checkpoint.period.'
+ pop_key) is not None:
@@ -57,9 +57,10 @@ class SwiftCheckpointProcessor(CheckpointProcessor):
_swift_output_dir = output_dir + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX
model.save_pretrained(
save_directory=_swift_output_dir,
safe_serialization=config.safe_get('train.checkpoint.safe_serialization', False),
adapter_name=config.safe_get('train.checkpoint.adapter_name', 'default')
)
safe_serialization=config.safe_get(
'train.checkpoint.safe_serialization', False),
adapter_name=config.safe_get(
'train.checkpoint.adapter_name', 'default'))
else:
model.save_pretrained(
output_dir,
@@ -109,7 +110,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor):
@HOOKS.register_module(module_name=Hooks.SwiftHook)
class SwiftHook(Hook):
_BIN_FILE_DIR = 'model'
def __init__(self):

31
modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py
View File

@@ -1,4 +1,5 @@
# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved.
import os
from typing import Union
import torch
@@ -7,16 +8,46 @@ from torch import nn
from modelscope.metainfo import Trainers
from modelscope.models.base import Model, TorchModel
from modelscope.trainers.builder import TRAINERS
from modelscope.trainers.hooks.checkpoint.checkpoint_hook import CheckpointHook
from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \
CheckpointProcessor
from modelscope.trainers.optimizer.builder import build_optimizer
from modelscope.trainers.trainer import EpochBasedTrainer
from modelscope.utils.config import ConfigDict
class SwiftDiffusionCheckpointProcessor(CheckpointProcessor):
def save_checkpoints(self,
trainer,
checkpoint_path_prefix,
output_dir,
meta=None,
save_optimizers=True):
"""Save the state dict for swift lora tune model.
"""
trainer.model.unet.save_pretrained(os.path.join(output_dir))
@TRAINERS.register_module(module_name=Trainers.stable_diffusion)
class StableDiffusionTrainer(EpochBasedTrainer):
def __init__(self, *args, **kwargs):
"""Stable Diffusion trainers for fine-tuning.
Args:
use_swift: Whether to use swift.
"""
super().__init__(*args, **kwargs)
use_swift = kwargs.pop('use_swift', False)
# set swift lora save checkpoint processor
if use_swift:
ckpt_hook = list(
filter(lambda hook: isinstance(hook, CheckpointHook),
self.hooks))[0]
ckpt_hook.set_processor(SwiftDiffusionCheckpointProcessor())
def build_optimizer(self, cfg: ConfigDict, default_args: dict = None):
try:

14
modelscope/trainers/trainer.py
View File

@@ -181,8 +181,20 @@ class EpochBasedTrainer(BaseTrainer):
compile_options = {}
self.model = compile_model(self.model, **compile_options)
if 'work_dir' in kwargs:
if kwargs.get('work_dir', None) is not None:
self.work_dir = kwargs['work_dir']
if 'train' not in self.cfg:
self.cfg['train'] = ConfigDict()
self.cfg['train']['work_dir'] = self.work_dir
if 'checkpoint' in self.cfg['train']:
if 'period' in self.cfg['train']['checkpoint']:
self.cfg['train']['checkpoint']['period'][
'save_dir'] = self.work_dir
if 'best' in self.cfg['train']['checkpoint']:
self.cfg['train']['checkpoint']['best'][
'save_dir'] = self.work_dir
if 'logging' in self.cfg['train']:
self.cfg['train']['logging']['out_dir'] = self.work_dir
else:
self.work_dir = self.cfg.train.get('work_dir', './work_dir')

16
modelscope/utils/hf_util.py
View File

@@ -13,6 +13,7 @@ from transformers import \
from transformers import \
AutoModelForTokenClassification as AutoModelForTokenClassificationHF
from transformers import AutoTokenizer as AutoTokenizerHF
from transformers import BitsAndBytesConfig as BitsAndBytesConfigHF
from transformers import GenerationConfig as GenerationConfigHF
from transformers import (PretrainedConfig, PreTrainedModel,
PreTrainedTokenizerBase)
@@ -22,6 +23,11 @@ from transformers.models.auto.tokenization_auto import (
from modelscope import snapshot_download
from modelscope.utils.constant import Invoke
try:
from transformers import GPTQConfig as GPTQConfigHF
except ImportError:
GPTQConfigHF = None
def user_agent(invoked_by=None):
if invoked_by is None:
@@ -85,12 +91,13 @@ def check_hf_code(model_dir: str, auto_class: type,
raise FileNotFoundError(f'{config_path} is not found')
config_dict = PretrainedConfig.get_config_dict(config_path)[0]
auto_class_name = auto_class.__name__
if auto_class is AutoTokenizerHF:
tokenizer_config = get_tokenizer_config(model_dir)
# load from repo
if trust_remote_code:
has_remote_code = False
if auto_class is AutoTokenizerHF:
tokenizer_config_dict = get_tokenizer_config(model_dir)
auto_map = tokenizer_config_dict.get('auto_map', None)
auto_map = tokenizer_config.get('auto_map', None)
if auto_map is not None:
module_name = auto_map.get(auto_class_name, None)
if module_name is not None:
@@ -123,6 +130,9 @@ def check_hf_code(model_dir: str, auto_class: type,
f'{model_type} not found in HF `CONFIG_MAPPING`{trust_remote_code_info}'
)
elif auto_class is AutoTokenizerHF:
tokenizer_class = tokenizer_config.get('tokenizer_class')
if tokenizer_class is not None:
return
if model_type not in TOKENIZER_MAPPING_NAMES:
raise ValueError(
f'{model_type} not found in HF `TOKENIZER_MAPPING_NAMES`{trust_remote_code_info}'
@@ -199,3 +209,5 @@ AutoConfig = get_wrapped_class(
AutoConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors'])
GenerationConfig = get_wrapped_class(
GenerationConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors'])
GPTQConfig = GPTQConfigHF
BitsAndBytesConfig = BitsAndBytesConfigHF

2
modelscope/utils/plugins.py
View File

@@ -9,7 +9,6 @@ import os
import pkgutil
import shutil
import sys
import venv
from contextlib import contextmanager
from fnmatch import fnmatch
from pathlib import Path
@@ -1144,6 +1143,7 @@ class EnvsManager(object):
cfg = read_config(model_dir)
self.plugins = cfg.get('plugins', [])
self.allow_remote = cfg.get('allow_remote', False)
import venv
self.env_builder = venv.EnvBuilder(
system_site_packages=True,
clear=False,

4
tests/utils/test_hf_util.py
View File

@@ -25,6 +25,10 @@ class HFUtilTest(unittest.TestCase):
self.assertEqual(tokenizer.model_max_length, 4096)
self.assertFalse(tokenizer.is_fast)
def test_quantization_import(self):
from modelscope import GPTQConfig, BitsAndBytesConfig
self.assertTrue(BitsAndBytesConfig is not None)
def test_auto_model(self):
model = AutoModelForCausalLM.from_pretrained(
'baichuan-inc/baichuan-7B', trust_remote_code=True)