From fca65485c283a152c8d255ed7b2e422b8ea775ca Mon Sep 17 00:00:00 2001 From: aky15 Date: Wed, 13 Sep 2023 14:53:41 +0800 Subject: [PATCH 01/13] Update asr_inference_pipeline.py add support for simulated streaming ASR inference. --- modelscope/pipelines/audio/asr_inference_pipeline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 2379274c..cc3a53d8 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -160,6 +160,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): token_num_relax=self.cmd['token_num_relax'], decoding_ind=self.cmd['decoding_ind'], decoding_mode=self.cmd['decoding_mode'], + simu_streaming=self.cmd['simu_streaming'], **kwargs, ) @@ -310,7 +311,8 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'fs': { 'model_fs': None, 'audio_fs': None - } + }, + 'simu_streaming': False, } frontend_conf = None @@ -333,7 +335,9 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): decoding_ind = root['decoding_ind'] if 'decoding_mode' in root: decoding_mode = root['decoding_mode'] - + if 'simu_streaming' in root: + simu_streaming = root['simu_streaming'] + cmd['beam_size'] = root['beam_size'] cmd['penalty'] = root['penalty'] cmd['maxlenratio'] = root['maxlenratio'] @@ -389,6 +393,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'punc_model_file', 'punc_infer_config', 'param_dict', + 'simu_streaming', ] for user_args in user_args_dict: From e535c6525e0a0d6f4cb3e77674567b86e1fb02b2 Mon Sep 17 00:00:00 2001 From: aky15 Date: Thu, 14 Sep 2023 16:06:10 +0800 Subject: [PATCH 02/13] Update asr_inference_pipeline.py rename simu_streaming to fake_streaming --- modelscope/pipelines/audio/asr_inference_pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index cc3a53d8..c3933288 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -160,7 +160,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): token_num_relax=self.cmd['token_num_relax'], decoding_ind=self.cmd['decoding_ind'], decoding_mode=self.cmd['decoding_mode'], - simu_streaming=self.cmd['simu_streaming'], + fake_streaming=self.cmd['fake_streaming'], **kwargs, ) @@ -312,7 +312,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'model_fs': None, 'audio_fs': None }, - 'simu_streaming': False, + 'fake_streaming': False, } frontend_conf = None @@ -335,8 +335,8 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): decoding_ind = root['decoding_ind'] if 'decoding_mode' in root: decoding_mode = root['decoding_mode'] - if 'simu_streaming' in root: - simu_streaming = root['simu_streaming'] + if 'fake_streaming' in root: + fake_streaming = root['fake_streaming'] cmd['beam_size'] = root['beam_size'] cmd['penalty'] = root['penalty'] @@ -393,7 +393,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'punc_model_file', 'punc_infer_config', 'param_dict', - 'simu_streaming', + 'fake_streaming', ] for user_args in user_args_dict: From 94b3a9eed7da7f623a8759cb7037f64549e040f4 Mon Sep 17 00:00:00 2001 From: aresnow1 <109642806+aresnow1@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:27:03 +0800 Subject: [PATCH 03/13] Add lsf_suffix arg for api.push_model (#545) --- modelscope/hub/api.py | 8 +++++++- modelscope/hub/repository.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index fd658eba..e0a5254d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -243,7 +243,8 @@ class HubApi: tag: Optional[str] = None, revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, original_model_id: Optional[str] = None, - ignore_file_pattern: Optional[Union[List[str], str]] = None): + ignore_file_pattern: Optional[Union[List[str], str]] = None, + lfs_suffix: Optional[Union[str, List[str]]] = None): """Upload model from a given directory to given repository. A valid model directory must contain a configuration.json file. @@ -281,6 +282,7 @@ class HubApi: branch and push to it. original_model_id (str, optional): The base model id which this model is trained from ignore_file_pattern (`Union[List[str], str]`, optional): The file pattern to ignore uploading + lfs_suffix (`List[str]`, optional): File types to use LFS to manage. examples: '*.safetensors'. Raises: InvalidParameter: Parameter invalid. @@ -349,6 +351,10 @@ class HubApi: date = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') commit_message = '[automsg] push model %s to hub at %s' % ( model_id, date) + if lfs_suffix is not None: + lfs_suffix_list = [lfs_suffix] if isinstance(lfs_suffix, str) else lfs_suffix + for suffix in lfs_suffix_list: + repo.add_lfs_type(suffix) repo.push( commit_message=commit_message, local_branch=revision, diff --git a/modelscope/hub/repository.py b/modelscope/hub/repository.py index 3fc6da2b..7cf32116 100644 --- a/modelscope/hub/repository.py +++ b/modelscope/hub/repository.py @@ -105,7 +105,7 @@ class Repository: examples '*.safetensors' """ os.system( - "printf '%s filter=lfs diff=lfs merge=lfs -text\n'>>%s" % + "printf '\n%s filter=lfs diff=lfs merge=lfs -text\n'>>%s" % (file_name_suffix, os.path.join(self.model_dir, '.gitattributes'))) def push(self, From e2025571469cbdedf1a0316d1d8c6cb38e0d7d8c Mon Sep 17 00:00:00 2001 From: wenmeng zhou Date: Mon, 18 Sep 2023 16:28:42 +0800 Subject: [PATCH 04/13] update traverse to copy __init__.py in model hub to dir of modelscope modules (#541) * update traverse * refine asr inference --- .../pipelines/audio/asr_inference_pipeline.py | 6 ++--- modelscope/utils/ast_utils.py | 17 +++++++----- modelscope/utils/plugins.py | 2 +- tests/utils/test_ast.py | 27 +++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index c3933288..ecd03079 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -319,6 +319,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): token_num_relax = None decoding_ind = None decoding_mode = None + fake_streaming = False if os.path.exists(outputs['am_model_config']): config_file = open(outputs['am_model_config'], encoding='utf-8') root = yaml.full_load(config_file) @@ -335,9 +336,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): decoding_ind = root['decoding_ind'] if 'decoding_mode' in root: decoding_mode = root['decoding_mode'] - if 'fake_streaming' in root: - fake_streaming = root['fake_streaming'] - + cmd['beam_size'] = root['beam_size'] cmd['penalty'] = root['penalty'] cmd['maxlenratio'] = root['maxlenratio'] @@ -354,6 +353,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): cmd['token_num_relax'] = token_num_relax cmd['decoding_ind'] = decoding_ind cmd['decoding_mode'] = decoding_mode + cmd['fake_streaming'] = fake_streaming if outputs.__contains__('mvn_file'): cmd['cmvn_file'] = outputs['mvn_file'] model_config = self.model_cfg['model_config'] diff --git a/modelscope/utils/ast_utils.py b/modelscope/utils/ast_utils.py index 5b6ae721..1aca1ce1 100644 --- a/modelscope/utils/ast_utils.py +++ b/modelscope/utils/ast_utils.py @@ -435,24 +435,27 @@ class FilesAstScanning(object): ignored.add(item) return list(set(output) - set(ignored)) - def traversal_files(self, path, check_sub_dir=None): + def traversal_files(self, path, check_sub_dir=None, include_init=False): self.file_dirs = [] if check_sub_dir is None or len(check_sub_dir) == 0: - self._traversal_files(path) + self._traversal_files(path, include_init=include_init) else: for item in check_sub_dir: sub_dir = os.path.join(path, item) if os.path.isdir(sub_dir): - self._traversal_files(sub_dir) + self._traversal_files(sub_dir, include_init=include_init) - def _traversal_files(self, path): + def _traversal_files(self, path, include_init=False): dir_list = os.scandir(path) for item in dir_list: - if item.name.startswith('__') or item.name.endswith( - '.json') or item.name.endswith('.md'): + if item.name == '__init__.py' and not include_init: + continue + elif (item.name.startswith('__') + and item.name != '__init__.py') or item.name.endswith( + '.json') or item.name.endswith('.md'): continue if item.is_dir(): - self._traversal_files(item.path) + self._traversal_files(item.path, include_init=include_init) elif item.is_file() and item.name.endswith('.py'): self.file_dirs.append(item.path) elif item.is_file() and 'requirement' in item.name: diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 1a3bfffe..3d39514a 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -372,7 +372,7 @@ def import_module_from_model_dir(model_dir): """ from pathlib import Path file_scanner = FilesAstScanning() - file_scanner.traversal_files(model_dir) + file_scanner.traversal_files(model_dir, include_init=True) file_dirs = file_scanner.file_dirs requirements = file_scanner.requirement_dirs diff --git a/tests/utils/test_ast.py b/tests/utils/test_ast.py index 544e75b6..e300e0e4 100644 --- a/tests/utils/test_ast.py +++ b/tests/utils/test_ast.py @@ -24,13 +24,31 @@ class AstScaningTest(unittest.TestCase): def setUp(self): print(('Testing %s.%s' % (type(self).__name__, self._testMethodName))) self.tmp_dir = tempfile.TemporaryDirectory().name + self.tmp_dir2 = tempfile.TemporaryDirectory().name self.test_file = os.path.join(self.tmp_dir, 'test.py') if not os.path.exists(self.tmp_dir): os.makedirs(self.tmp_dir) + fnames = ['1.py', '2.py', '3.py', '__init__.py'] + self.folders = ['.', 'a', 'b', 'c'] + dir_path = self.tmp_dir2 + folder_dirs = [ + os.path.join(dir_path, folder) for folder in self.folders + ] + for folder in folder_dirs: + os.makedirs(folder, exist_ok=True) + for fname in fnames: + fpath = os.path.join(folder, fname) + with open(fpath, 'w') as f: + f.write('hello world') + + for folder in folder_dirs: + print(f'folder: {os.listdir(folder)}') + def tearDown(self): super().tearDown() shutil.rmtree(self.tmp_dir) + shutil.rmtree(self.tmp_dir2) def test_ast_scaning_class(self): astScaner = AstScanning() @@ -75,6 +93,15 @@ class AstScaningTest(unittest.TestCase): index_0 = list(requirements.keys())[0] self.assertIsInstance(requirements[index_0], list) + fileScaner.traversal_files(self.tmp_dir2, include_init=False) + self.assertTrue( + os.path.join(self.tmp_dir2, '__init__.py') not in + fileScaner.file_dirs) + + fileScaner.traversal_files(self.tmp_dir2, include_init=True) + self.assertTrue( + os.path.join(self.tmp_dir2, '__init__.py') in fileScaner.file_dirs) + def test_file_mtime_md5_method(self): fileScaner = FilesAstScanning() # create first file From ae039bbe02660e2093d246d5ae3bb15602344be6 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Mon, 18 Sep 2023 18:22:53 +0800 Subject: [PATCH 05/13] fix dependency (#527) --- .../efficient_stable_diffusion.py | 17 ++++++++++++++++- modelscope/utils/error.py | 6 ++++++ modelscope/utils/import_utils.py | 1 + requirements/framework.txt | 1 - .../test_efficient_diffusion_tuning.py | 5 ++++- .../test_efficient_diffusion_tuning_swift.py | 6 +++++- 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 2fcd1df8..cec87bad 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -13,7 +13,6 @@ from diffusers import (AutoencoderKL, DDPMScheduler, DiffusionPipeline, utils) from diffusers.models import attention from diffusers.utils import deprecation_utils -from swift import AdapterConfig, LoRAConfig, PromptConfig, Swift from transformers import CLIPTextModel, CLIPTokenizer from modelscope import snapshot_download @@ -26,6 +25,7 @@ from modelscope.outputs import OutputKeys from modelscope.utils.checkpoint import save_checkpoint, save_configuration from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.import_utils import is_swift_available from .control_sd_lora import ControlLoRATuner utils.deprecate = lambda *arg, **kwargs: None @@ -34,6 +34,9 @@ attention.deprecate = lambda *arg, **kwargs: None __tuner_MAP__ = {'lora': LoRATuner, 'control_lora': ControlLoRATuner} +if is_swift_available(): + from swift import AdapterConfig, LoRAConfig, PromptConfig, Swift + @MODELS.register_module( Tasks.efficient_diffusion_tuning, @@ -110,6 +113,10 @@ class EfficientStableDiffusion(TorchModel): self.tuner_name = tuner_name if tuner_name == 'swift-lora': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) rank = tuner_config[ 'rank'] if tuner_config and 'rank' in tuner_config else 4 lora_config = LoRAConfig( @@ -119,6 +126,10 @@ class EfficientStableDiffusion(TorchModel): use_merged_linear=False) self.unet = Swift.prepare_model(self.unet, lora_config) elif tuner_name == 'swift-adapter': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) adapter_length = tuner_config[ 'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10 adapter_config = AdapterConfig( @@ -128,6 +139,10 @@ class EfficientStableDiffusion(TorchModel): adapter_length=adapter_length) self.unet = Swift.prepare_model(self.unet, adapter_config) elif tuner_name == 'swift-prompt': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) prompt_length = tuner_config[ 'prompt_length'] if tuner_config and 'prompt_length' in tuner_config else 10 prompt_config = PromptConfig( diff --git a/modelscope/utils/error.py b/modelscope/utils/error.py index 8259c7ce..65c92196 100644 --- a/modelscope/utils/error.py +++ b/modelscope/utils/error.py @@ -174,3 +174,9 @@ XFORMERS_IMPORT_ERROR = """ {0} requires the timm library but it was not found in your environment. You can install it with pip: `pip install xformers>=0.0.17` """ + +# docstyle-ignore +SWIFT_IMPORT_ERROR = """ +{0} requires the ms-swift library but it was not found in your environment. You can install it with pip: +`pip install ms-swift -U` +""" diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index 2ce9d55d..1910039a 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -310,6 +310,7 @@ REQUIREMENTS_MAAPING = OrderedDict([ ('open_clip', (is_package_available('open_clip'), OPENCLIP_IMPORT_ERROR)), ('taming', (is_package_available('taming'), TAMING_IMPORT_ERROR)), ('xformers', (is_package_available('xformers'), XFORMERS_IMPORT_ERROR)), + ('swift', (is_package_available('swift'), SWIFT_IMPORT_ERROR)), ]) SYSTEM_PACKAGE = set(['os', 'sys', 'typing']) diff --git a/requirements/framework.txt b/requirements/framework.txt index e9dc08c4..83e69a00 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -4,7 +4,6 @@ datasets>=2.8.0,<=2.13.0 einops filelock>=3.3.0 gast>=0.2.2 -ms-swift numpy oss2 pandas diff --git a/tests/pipelines/test_efficient_diffusion_tuning.py b/tests/pipelines/test_efficient_diffusion_tuning.py index 330aee57..1f224917 100644 --- a/tests/pipelines/test_efficient_diffusion_tuning.py +++ b/tests/pipelines/test_efficient_diffusion_tuning.py @@ -1,8 +1,8 @@ # Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os import unittest from modelscope.models import Model -from modelscope.models.multi_modal import EfficientStableDiffusion from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.test_utils import test_level @@ -11,6 +11,7 @@ from modelscope.utils.test_utils import test_level class EfficientDiffusionTuningTest(unittest.TestCase): def setUp(self) -> None: + os.system('pip install ms-swift -U') self.task = Tasks.efficient_diffusion_tuning @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') @@ -28,6 +29,7 @@ class EfficientDiffusionTuningTest(unittest.TestCase): model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora' model_revision = 'v1.0.2' model = Model.from_pretrained(model_id, model_revision=model_revision) + from modelscope.models.multi_modal import EfficientStableDiffusion self.assertTrue(model.__class__ == EfficientStableDiffusion) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') @@ -52,6 +54,7 @@ class EfficientDiffusionTuningTest(unittest.TestCase): model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora' model_revision = 'v1.0.2' model = Model.from_pretrained(model_id, model_revision=model_revision) + from modelscope.models.multi_modal import EfficientStableDiffusion self.assertTrue(model.__class__ == EfficientStableDiffusion) diff --git a/tests/pipelines/test_efficient_diffusion_tuning_swift.py b/tests/pipelines/test_efficient_diffusion_tuning_swift.py index a2af7dec..d225a538 100644 --- a/tests/pipelines/test_efficient_diffusion_tuning_swift.py +++ b/tests/pipelines/test_efficient_diffusion_tuning_swift.py @@ -1,11 +1,11 @@ # Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os import tempfile import unittest import cv2 from modelscope.models import Model -from modelscope.models.multi_modal import EfficientStableDiffusion from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.test_utils import test_level @@ -14,6 +14,7 @@ from modelscope.utils.test_utils import test_level class EfficientDiffusionTuningTestSwift(unittest.TestCase): def setUp(self) -> None: + os.system('pip install ms-swift -U') self.task = Tasks.efficient_diffusion_tuning @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') @@ -39,6 +40,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora' model_revision = 'v1.0.2' model = Model.from_pretrained(model_id, model_revision=model_revision) + from modelscope.models.multi_modal import EfficientStableDiffusion self.assertTrue(model.__class__ == EfficientStableDiffusion) @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') @@ -64,6 +66,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter' model_revision = 'v1.0.2' model = Model.from_pretrained(model_id, model_revision=model_revision) + from modelscope.models.multi_modal import EfficientStableDiffusion self.assertTrue(model.__class__ == EfficientStableDiffusion) @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') @@ -89,6 +92,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt' model_revision = 'v1.0.2' model = Model.from_pretrained(model_id, model_revision=model_revision) + from modelscope.models.multi_modal import EfficientStableDiffusion self.assertTrue(model.__class__ == EfficientStableDiffusion) From 9de1b1e67428cf3478a76be6188587d54cf9470a Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Tue, 19 Sep 2023 11:42:59 +0800 Subject: [PATCH 06/13] release data to 2099 --- modelscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/version.py b/modelscope/version.py index 7f3298ab..23ef0243 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -2,4 +2,4 @@ __version__ = '1.9.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2023-09-06 00:00:00' +__release_datetime__ = '2099-09-06 00:00:00' From 4cf7b1e7376d5a6a0ee344cbb959eefacf7fd07b Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:05:44 +0800 Subject: [PATCH 07/13] Add third_party key (#546) --- modelscope/trainers/trainer.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index 65c238da..a3707918 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -142,12 +142,8 @@ class EpochBasedTrainer(BaseTrainer): self._samplers = samplers if isinstance(model, str): - third_party = kwargs.get(ThirdParty.KEY) - if third_party is not None: - kwargs.pop(ThirdParty.KEY) - self.model_dir = self.get_or_download_model_dir( - model, model_revision, third_party) + model, model_revision, kwargs.pop(ThirdParty.KEY, None)) if cfg_file is None: cfg_file = os.path.join(self.model_dir, ModelFile.CONFIGURATION) @@ -159,7 +155,10 @@ class EpochBasedTrainer(BaseTrainer): if hasattr(model, 'model_dir'): check_local_model_is_latest( model.model_dir, - user_agent={Invoke.KEY: Invoke.LOCAL_TRAINER}) + user_agent={ + Invoke.KEY: Invoke.LOCAL_TRAINER, + ThirdParty.KEY: kwargs.pop(ThirdParty.KEY, None) + }) super().__init__(cfg_file, arg_parse_fn) self.cfg_modify_fn = cfg_modify_fn From 5ef842e38a0e3a0bc4cdd9863d778cf42c893aff Mon Sep 17 00:00:00 2001 From: Shen Huang Date: Tue, 19 Sep 2023 17:34:41 +0800 Subject: [PATCH 08/13] enable token_cls_pipeline to inference on longer inputs and return entity probabilities (#551) * allow token classification pipelines to predict longer sentences * bugfix * skip adaseq pipeline ut when connection error occurs * return entity probabilities --- .../nlp/task_models/token_classification.py | 3 +- .../nlp/token_classification_pipeline.py | 85 ++++++++++++++++++- .../test_plugin_model.py | 41 +++++---- .../test_named_entity_recognition.py | 19 +++++ 4 files changed, 129 insertions(+), 19 deletions(-) diff --git a/modelscope/models/nlp/task_models/token_classification.py b/modelscope/models/nlp/task_models/token_classification.py index aa84eaf0..8c5142b9 100644 --- a/modelscope/models/nlp/task_models/token_classification.py +++ b/modelscope/models/nlp/task_models/token_classification.py @@ -102,6 +102,7 @@ class ModelForTokenClassificationWithCRF(ModelForTokenClassification): base_model_prefix = 'encoder' def postprocess(self, inputs, **kwargs): + logits = inputs['logits'] predicts = self.head.decode(inputs['logits'], inputs['label_mask']) offset_mapping = inputs['offset_mapping'] mask = inputs['label_mask'] @@ -119,7 +120,7 @@ class ModelForTokenClassificationWithCRF(ModelForTokenClassification): return AttentionTokenClassificationModelOutput( loss=None, - logits=None, + logits=logits, hidden_states=None, attentions=None, label_mask=mask, diff --git a/modelscope/pipelines/nlp/token_classification_pipeline.py b/modelscope/pipelines/nlp/token_classification_pipeline.py index 9fd8e325..0c87e3a0 100644 --- a/modelscope/pipelines/nlp/token_classification_pipeline.py +++ b/modelscope/pipelines/nlp/token_classification_pipeline.py @@ -1,6 +1,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any, Dict, List, Optional, Union +import math +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import torch @@ -8,7 +9,7 @@ import torch from modelscope.metainfo import Pipelines from modelscope.models import Model from modelscope.outputs import OutputKeys -from modelscope.pipelines.base import Pipeline +from modelscope.pipelines.base import Input, Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.preprocessors import Preprocessor from modelscope.utils.constant import ModelFile, Tasks @@ -64,6 +65,7 @@ class TokenClassificationPipeline(Pipeline): sequence_length=sequence_length, **kwargs) self.model.eval() + self.sequence_length = sequence_length assert hasattr(self.preprocessor, 'id2label') self.id2label = self.preprocessor.id2label @@ -131,9 +133,20 @@ class TokenClassificationPipeline(Pipeline): predictions = torch_nested_numpify(torch_nested_detach(predictions)) labels = [self.id2label[x] for x in predictions] + return_prob = postprocess_params.pop('return_prob', True) + if return_prob: + if OutputKeys.LOGITS in inputs: + logits = inputs[OutputKeys.LOGITS] + if len(logits.shape) == 3: + logits = logits[0] + probs = torch_nested_numpify( + torch_nested_detach(logits.softmax(-1))) + else: + return_prob = False + chunks = [] chunk = {} - for label, offsets in zip(labels, offset_mapping): + for i, (label, offsets) in enumerate(zip(labels, offset_mapping)): if label[0] in 'BS': if chunk: chunk['span'] = text[chunk['start']:chunk['end']] @@ -143,6 +156,8 @@ class TokenClassificationPipeline(Pipeline): 'start': offsets[0], 'end': offsets[1] } + if return_prob: + chunk['prob'] = probs[i][predictions[i]] if label[0] in 'I': if not chunk: chunk = { @@ -150,6 +165,8 @@ class TokenClassificationPipeline(Pipeline): 'start': offsets[0], 'end': offsets[1] } + if return_prob: + chunk['prob'] = probs[i][predictions[i]] if label[0] in 'E': if not chunk: chunk = { @@ -157,6 +174,8 @@ class TokenClassificationPipeline(Pipeline): 'start': offsets[0], 'end': offsets[1] } + if return_prob: + chunk['prob'] = probs[i][predictions[i]] if label[0] in 'IES': if chunk: chunk['end'] = offsets[1] @@ -172,3 +191,63 @@ class TokenClassificationPipeline(Pipeline): chunks.append(chunk) return chunks + + def _process_single(self, input: Input, *args, **kwargs) -> Dict[str, Any]: + split_max_length = kwargs.pop('split_max_length', + 0) # default: no split + if split_max_length <= 0: + return super()._process_single(input, *args, **kwargs) + else: + split_texts, index_mapping = self._auto_split([input], + split_max_length) + outputs = [] + for text in split_texts: + outputs.append(super()._process_single(text, *args, **kwargs)) + return self._auto_join(outputs, index_mapping)[0] + + def _process_batch(self, input: List[Input], batch_size: int, *args, + **kwargs) -> List[Dict[str, Any]]: + split_max_length = kwargs.pop('split_max_length', + 0) # default: no split + if split_max_length <= 0: + return super()._process_batch( + input, batch_size=batch_size, *args, **kwargs) + else: + split_texts, index_mapping = self._auto_split( + input, split_max_length) + outputs = super()._process_batch( + split_texts, batch_size=batch_size, *args, **kwargs) + return self._auto_join(outputs, index_mapping) + + def _auto_split(self, input_texts: List[str], split_max_length: int): + split_texts = [] + index_mapping = {} + new_idx = 0 + for raw_idx, text in enumerate(input_texts): + if len(text) < split_max_length: + split_texts.append(text) + index_mapping[new_idx] = (raw_idx, 0) + new_idx += 1 + else: + n_split = math.ceil(len(text) / split_max_length) + for i in range(n_split): + offset = i * split_max_length + split_texts.append(text[offset:offset + split_max_length]) + index_mapping[new_idx] = (raw_idx, offset) + new_idx += 1 + return split_texts, index_mapping + + def _auto_join( + self, outputs: List[Dict[str, Any]], + index_mapping: Dict[int, Tuple[int, int]]) -> List[Dict[str, Any]]: + joined_outputs = [] + for idx, output in enumerate(outputs): + raw_idx, offset = index_mapping[idx] + if raw_idx >= len(joined_outputs): + joined_outputs.append(output) + else: + for chunk in output[OutputKeys.OUTPUT]: + chunk['start'] += offset + chunk['end'] += offset + joined_outputs[raw_idx][OutputKeys.OUTPUT].append(chunk) + return joined_outputs diff --git a/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py b/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py index 71b9e64f..aeb6c9bd 100644 --- a/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py +++ b/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py @@ -23,20 +23,31 @@ class PluginModelTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_span_based_ner_pipeline(self): - pipeline_ins = pipeline( - Tasks.named_entity_recognition, - 'damo/nlp_nested-ner_named-entity-recognition_chinese-base-med') - print( - pipeline_ins( - '1、可测量目标： 1周内胸闷缓解。2、下一步诊疗措施：1.心内科护理常规，一级护理，低盐低脂饮食，留陪客。' - '2.予“阿司匹林肠溶片”抗血小板聚集，“呋塞米、螺内酯”利尿减轻心前负荷，“瑞舒伐他汀”调脂稳定斑块，“厄贝沙坦片片”降血压抗心机重构' - )) + try: + pipeline_ins = pipeline( + Tasks.named_entity_recognition, + 'damo/nlp_nested-ner_named-entity-recognition_chinese-base-med' + ) + print( + pipeline_ins( + '1、可测量目标： 1周内胸闷缓解。2、下一步诊疗措施：1.心内科护理常规，一级护理，低盐低脂饮食，留陪客。' + '2.予“阿司匹林肠溶片”抗血小板聚集，“呋塞米、螺内酯”利尿减轻心前负荷，“瑞舒伐他汀”调脂稳定斑块，“厄贝沙坦片片”降血压抗心机重构' + )) + except RuntimeError: + print( + 'Skip test span_based_ner_pipeline! RuntimeError: Try loading from huggingface and modelscope failed' + ) def test_maoe_pipelines(self): - pipeline_ins = pipeline( - Tasks.named_entity_recognition, - 'damo/nlp_maoe_named-entity-recognition_chinese-base-general') - print( - pipeline_ins( - '刘培强，男，生理年龄40岁（因为在太空中进入休眠状态），实际年龄52岁，领航员国际空间站中的中国航天员，机械工程专家，军人，军衔中校。' - )) + try: + pipeline_ins = pipeline( + Tasks.named_entity_recognition, + 'damo/nlp_maoe_named-entity-recognition_chinese-base-general') + print( + pipeline_ins( + '刘培强，男，生理年龄40岁（因为在太空中进入休眠状态），实际年龄52岁，领航员国际空间站中的中国航天员，机械工程专家，军人，军衔中校。' + )) + except RuntimeError: + print( + 'Skip test maoe_pipeline! RuntimeError: Try loading from huggingface and modelscope failed' + ) diff --git a/tests/pipelines/test_named_entity_recognition.py b/tests/pipelines/test_named_entity_recognition.py index 8b7424f4..4f431b9f 100644 --- a/tests/pipelines/test_named_entity_recognition.py +++ b/tests/pipelines/test_named_entity_recognition.py @@ -459,6 +459,25 @@ class NamedEntityRecognitionTest(unittest.TestCase): pipeline_ins = pipeline(task=Tasks.named_entity_recognition) print(pipeline_ins(input=self.sentence)) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_long_chinese_with_model_name(self): + pipeline_ins = pipeline( + task=Tasks.named_entity_recognition, model=self.chinese_model_id) + print( + pipeline_ins( + input=self.sentence + '. ' * 1000, + split_max_length=300)) # longer than 512 + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_long_chinese_with_model_name_batch(self): + pipeline_ins = pipeline( + task=Tasks.named_entity_recognition, model=self.chinese_model_id) + print( + pipeline_ins( + input=[self.sentence + '. ' * 1000] * 2, + batch_size=2, + split_max_length=300)) # longer than 512 + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_with_all_modelcards(self): for item in self.all_modelcards_info: From cd976a366ac2fce2a403ad04e5696dfc0e875465 Mon Sep 17 00:00:00 2001 From: wenmeng zhou Date: Wed, 20 Sep 2023 20:39:28 +0800 Subject: [PATCH 09/13] add contact info to issue template (#552) --- .github/ISSUE_TEMPLATE/bug_report.md | 15 +++++++++++++++ .github/ISSUE_TEMPLATE/question.md | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 27b307c9..4fdf7351 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -32,3 +32,18 @@ A clear and concise description of what the bug is. * You may add addition that may be helpful for locating the problem, such as * How you installed PyTorch [e.g., pip, conda, source] * Other environment variables that may be related (such as $PATH, $LD_LIBRARY_PATH, $PYTHONPATH, etc.) + + +Please @ corresponding people according to your problem: + +Model related: @wenmengzhou @tastelikefeet + +Model hub related: @liuyhwangyh + +Dataset releated: @wangxingjun778 + +Finetune related: @tastelikefeet @Jintao-Huang + +Pipeline related: @Firmament-cyou @wenmengzhou + +Contribute your model: @zzclynn diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 06435d1a..c7ec7256 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -3,7 +3,7 @@ name: Question about: Describe this issue template's purpose here. title: '' labels: '' -assignees: zzclynn +assignees: zzclynn,wenmengzhou --- @@ -15,3 +15,17 @@ Before asking a question, make sure you have: * Googled your question. * Searched related issues but cannot get the expected help. * The bug has not been fixed in the latest version. + +Please @ corresponding people according to your problem: + +Model related: @wenmengzhou @tastelikefeet + +Model hub related: @liuyhwangyh + +Dataset releated: @wangxingjun778 + +Finetune related: @tastelikefeet @Jintao-Huang + +Pipeline related: @Firmament-cyou @wenmengzhou + +Contribute your model: @zzclynn From 3e6acb7998213390bdf5ed78abe2153868d885c0 Mon Sep 17 00:00:00 2001 From: jiangzeyinzi Date: Thu, 21 Sep 2023 16:02:31 +0800 Subject: [PATCH 10/13] Compatible with Swift on SD Tuner (#554) Co-authored-by: zeyinzi.jzyz --- modelscope/metainfo.py | 1 + .../efficient_stable_diffusion.py | 42 ++++-- modelscope/preprocessors/multi_modal.py | 3 + modelscope/trainers/hooks/__init__.py | 2 + modelscope/trainers/hooks/swift/__init__.py | 1 + modelscope/trainers/hooks/swift/swift_hook.py | 131 ++++++++++++++++++ ...fficient_diffusion_tuning_trainer_swift.py | 2 +- 7 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 modelscope/trainers/hooks/swift/__init__.py create mode 100644 modelscope/trainers/hooks/swift/swift_hook.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 23ffdab1..d2d8115a 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -1233,6 +1233,7 @@ class Hooks(object): DeepspeedHook = 'DeepspeedHook' MegatronHook = 'MegatronHook' DDPHook = 'DDPHook' + SwiftHook = 'SwiftHook' class LR_Schedulers(object): diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index cec87bad..3830bb52 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -86,6 +86,8 @@ class EfficientStableDiffusion(TorchModel): self.pipe.scheduler.config) self.pipe = self.pipe.to(self.device) self.unet = self.pipe.unet + self.text_encoder = self.pipe.text_encoder + self.vae = self.pipe.vae else: # Load scheduler, tokenizer and models. self.noise_scheduler = DDPMScheduler.from_pretrained( @@ -132,12 +134,19 @@ class EfficientStableDiffusion(TorchModel): ) adapter_length = tuner_config[ 'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10 - adapter_config = AdapterConfig( - dim=-1, - hidden_pos=0, - target_modules=r'.*ff\.net\.2$', - adapter_length=adapter_length) - self.unet = Swift.prepare_model(self.unet, adapter_config) + adapter_config_dict = {} + dim_list = [320, 640, 1280] + target_modules_list = [r"(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)", + r"(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)", + r"(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)"] + for dim, target_modules in zip(dim_list, target_modules_list): + adapter_config = AdapterConfig( + dim=dim, + hidden_pos=0, + target_modules=target_modules, + adapter_length=adapter_length) + adapter_config_dict[f"adapter_{dim}"] = adapter_config + self.unet = Swift.prepare_model(self.unet, adapter_config_dict) elif tuner_name == 'swift-prompt': if not is_swift_available(): raise ValueError( @@ -154,7 +163,8 @@ class EfficientStableDiffusion(TorchModel): r'.*[down_blocks|up_blocks|mid_block]\.\d+\.attentions\.\d+\.transformer_blocks\.\d+$', embedding_pos=0, prompt_length=prompt_length, - attach_front=False) + attach_front=False, + extract_embedding=True) self.unet = Swift.prepare_model(self.unet, prompt_config) elif tuner_name in ('lora', 'control_lora'): # if not set the config of control-tuner, we add the lora tuner directly to the original framework, @@ -181,13 +191,13 @@ class EfficientStableDiffusion(TorchModel): else: super().load_state_dict(state_dict=state_dict, strict=strict) - def state_dict(self): + def state_dict(self, *arg, **kwargs): if hasattr(self, 'tuner'): - return self.tuner.state_dict() - elif self.tuner_name.startswith('swift'): - return self.unet.state_dict() + return self.tuner.state_dict(*arg, **kwargs) + elif self.tuner_name.startswith('swift-'): + return self.unet.state_dict(*arg, **kwargs) else: - return super().state_dict() + return super().state_dict(*arg, **kwargs) def tokenize_caption(self, captions): """ Convert caption text to token data. @@ -204,7 +214,7 @@ class EfficientStableDiffusion(TorchModel): return_tensors='pt') return inputs.input_ids - def forward(self, prompt='', cond=None, target=None, **args): + def forward(self, prompt, cond=None, target=None, **args): if self.inference: if 'generator_seed' in args and isinstance(args['generator_seed'], int): @@ -213,11 +223,13 @@ class EfficientStableDiffusion(TorchModel): else: generator = None num_inference_steps = args.get('num_inference_steps', 30) + guidance_scale = args.get('guidance_scale', 7.5) if self.is_control: _ = self.tuner(cond.to(self.device)).control_states images = self.pipe( prompt, num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, generator=generator).images return images else: @@ -243,8 +255,8 @@ class EfficientStableDiffusion(TorchModel): input_ids = self.tokenize_caption(prompt).to(self.device) # Get the text embedding for conditioning - with torch.no_grad(): - encoder_hidden_states = self.text_encoder(input_ids)[0] + # with torch.no_grad(): + encoder_hidden_states = self.text_encoder(input_ids)[0] # Inject control states to unet if self.is_control: diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index d180289b..54ad6e97 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -53,10 +53,13 @@ class DiffusionImageGenerationPreprocessor(Preprocessor): self.preprocessor_mean = kwargs.pop('mean', [0.5]) self.preprocessor_std = kwargs.pop('std', [0.5]) self.preprocessor_image_keys = set(kwargs.pop('image_keys', [])) + self.center_crop = kwargs.pop('center_crop', True) + self.transform_input = transforms.Compose([ transforms.Resize( self.preprocessor_resolution, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.CenterCrop(self.preprocessor_resolution) if self.center_crop else transforms.RandomCrop(self.preprocessor_resolution), transforms.ToTensor(), transforms.Normalize(self.preprocessor_mean, self.preprocessor_std), diff --git a/modelscope/trainers/hooks/__init__.py b/modelscope/trainers/hooks/__init__.py index 072105be..a51c50e8 100644 --- a/modelscope/trainers/hooks/__init__.py +++ b/modelscope/trainers/hooks/__init__.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from .distributed.ddp_hook import DDPHook from .distributed.deepspeed_hook import DeepspeedHook from .distributed.megatron_hook import MegatronHook + from .swift.swift_hook import SwiftHook else: _import_structure = { @@ -40,6 +41,7 @@ else: 'distributed.ddp_hook': ['DDPHook'], 'distributed.deepspeed_hook': ['DeepspeedHook'], 'distributed.megatron_hook': ['MegatronHook'], + 'swift.swift_hook': ['SwiftHook'], 'priority': ['Priority', 'get_priority'] } diff --git a/modelscope/trainers/hooks/swift/__init__.py b/modelscope/trainers/hooks/swift/__init__.py new file mode 100644 index 00000000..daf16f92 --- /dev/null +++ b/modelscope/trainers/hooks/swift/__init__.py @@ -0,0 +1 @@ +from .swift_hook import SwiftHook \ No newline at end of file diff --git a/modelscope/trainers/hooks/swift/swift_hook.py b/modelscope/trainers/hooks/swift/swift_hook.py new file mode 100644 index 00000000..262dd483 --- /dev/null +++ b/modelscope/trainers/hooks/swift/swift_hook.py @@ -0,0 +1,131 @@ +import os +import shutil + +from modelscope.metainfo import Hooks +from modelscope.trainers import EpochBasedTrainer +from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( + BestCkptSaverHook, CheckpointHook, CheckpointProcessor) +from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import \ + LoadCheckpointHook +from modelscope.trainers.hooks.hook import Hook +from modelscope.utils.import_utils import is_swift_available +from modelscope.utils.checkpoint import save_configuration + + +class SwiftCheckpointProcessor(CheckpointProcessor): + + _BIN_FILE_DIR = 'model' + SWIFT_SAVE_SUFFIX = '_swift' + + @staticmethod + def copy_files_and_dump_config(trainer, output_dir, config, bin_file): + """Copy useful files to target output folder and dumps the target configuration.json. + """ + model = trainer.unwrap_module(trainer.model) + + class SaveConfig: + + def __init__(self, output_dir, config): + self.output_dir = output_dir + self.config = config + + def __call__(self, _output_dir, _config): + self.config = _config + + def save_config(self): + save_configuration(self.output_dir, self.config) + + for pop_key in [ + 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' + ]: + if config.safe_get('train.checkpoint.period.' + + pop_key) is not None: + config.safe_get('train.checkpoint.period').pop(pop_key) + if config.safe_get('train.checkpoint.best.' + pop_key) is not None: + config.safe_get('train.checkpoint.best').pop(pop_key) + + save_config_fn = SaveConfig(output_dir, config) + + if hasattr(model, 'save_pretrained'): + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use SwiftHook.' + ) + from swift import SwiftModel + if isinstance(model, SwiftModel): + _swift_output_dir = output_dir + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX + model.save_pretrained( + save_directory=_swift_output_dir, + safe_serialization=config.safe_get('train.checkpoint.safe_serialization', False), + adapter_name=config.safe_get('train.checkpoint.adapter_name', 'default') + ) + else: + model.save_pretrained( + output_dir, + bin_file, + save_function=lambda *args, **kwargs: None, + config=save_config_fn.config, + save_config_function=save_config_fn) + + if trainer.train_preprocessor is not None: + trainer.train_preprocessor.save_pretrained( + output_dir, + save_config_fn.config, + save_config_function=save_config_fn) + if trainer.eval_preprocessor is not None: + trainer.eval_preprocessor.save_pretrained( + output_dir, + save_config_fn.config, + save_config_function=save_config_fn) + save_config_fn.save_config() + + def link_dir(self, source_dir, output_dir): + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + shutil.copytree(source_dir, output_dir) + + def save_swift_model_state(self, model, filename): + model.save_pretrained(filename) + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None, + save_optimizers=True): + model = trainer.unwrap_module(trainer.model) + _model_file, _train_state_file = self._get_state_file_name( + checkpoint_path_prefix) + _swift_save_dir = checkpoint_path_prefix + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX + _swift_output_dir = output_dir + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX + self.save_trainer_state(trainer, model, _train_state_file, meta, + save_optimizers) + self.save_model_state(model, _model_file) + self.link(model, _model_file, output_dir) + self.save_swift_model_state(model, _swift_save_dir) + self.link_dir(_swift_save_dir, _swift_output_dir) + + +@HOOKS.register_module(module_name=Hooks.SwiftHook) +class SwiftHook(Hook): + + _BIN_FILE_DIR = 'model' + + def __init__(self): + pass + + def register_processor(self, trainer: EpochBasedTrainer): + processor = SwiftCheckpointProcessor() + ckpt_hook = trainer.get_hook(CheckpointHook) + if len(ckpt_hook) > 0 and not isinstance(ckpt_hook[0].processor, + SwiftCheckpointProcessor): + ckpt_hook[0].set_processor(processor) + best_ckpt_hook = trainer.get_hook(BestCkptSaverHook) + if len(best_ckpt_hook) > 0 and not isinstance( + best_ckpt_hook[0].processor, SwiftCheckpointProcessor): + best_ckpt_hook[0].set_processor(processor) + load_ckpt_hook = trainer.get_hook(LoadCheckpointHook) + if len(load_ckpt_hook) > 0 and not isinstance( + load_ckpt_hook[0].processor, SwiftCheckpointProcessor): + load_ckpt_hook[0].set_processor(processor) diff --git a/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py b/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py index c661b8ee..c05e504c 100644 --- a/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py +++ b/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py @@ -22,7 +22,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase): split='train', subset_name='Anime').remap_columns({'Image:FILE': 'target:FILE'}) - self.max_epochs = 30 + self.max_epochs = 1 self.lr = 0.0001 self.tmp_dir = tempfile.TemporaryDirectory().name From 881488a830319aa6d5a309ca70f444833fa3af30 Mon Sep 17 00:00:00 2001 From: Yabin Li Date: Mon, 25 Sep 2023 14:02:54 +0800 Subject: [PATCH 11/13] fix bug: support local asr models (#556) * fix bug: support local asr models * update asr_inference_pipeline --- modelscope/pipelines/audio/asr_inference_pipeline.py | 9 +++++---- modelscope/preprocessors/asr.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index ecd03079..f825412c 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -161,6 +161,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): decoding_ind=self.cmd['decoding_ind'], decoding_mode=self.cmd['decoding_mode'], fake_streaming=self.cmd['fake_streaming'], + model_lang=self.cmd['model_lang'], **kwargs, ) @@ -305,7 +306,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): 'idx_text': '', 'sampled_ids': 'seq2seq/sampled_ids', 'sampled_lengths': 'seq2seq/sampled_lengths', - 'lang': 'zh-cn', + 'model_lang': outputs['model_lang'], 'code_base': outputs['code_base'], 'mode': outputs['mode'], 'fs': { @@ -357,16 +358,16 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): if outputs.__contains__('mvn_file'): cmd['cmvn_file'] = outputs['mvn_file'] model_config = self.model_cfg['model_config'] - if model_config.__contains__('vad_model') and self.vad_model != '': + if model_config.__contains__('vad_model') and self.vad_model is None: self.vad_model = model_config['vad_model'] if model_config.__contains__('vad_model_revision'): self.vad_model_revision = model_config['vad_model_revision'] - if model_config.__contains__('punc_model') and self.punc_model != '': + if model_config.__contains__('punc_model') and self.punc_model is None: self.punc_model = model_config['punc_model'] if model_config.__contains__('punc_model_revision'): self.punc_model_revision = model_config['punc_model_revision'] if model_config.__contains__( - 'timestamp_model') and self.timestamp_model != '': + 'timestamp_model') and self.timestamp_model is None: self.timestamp_model = model_config['timestamp_model'] if model_config.__contains__('timestamp_model_revision'): self.timestamp_model_revision = model_config[ diff --git a/modelscope/preprocessors/asr.py b/modelscope/preprocessors/asr.py index 4696c675..4a24ffb2 100644 --- a/modelscope/preprocessors/asr.py +++ b/modelscope/preprocessors/asr.py @@ -96,6 +96,10 @@ class WavToScp(Preprocessor): else: mode = None inputs['mode'] = mode + if 'lang' in inputs['model_config']: + inputs['model_lang'] = inputs['model_config']['lang'] + else: + inputs['model_lang'] = 'zh-cn' if inputs['model_type'] == Frameworks.torch: assert inputs['model_config'].__contains__( From 70fe158d1302a71f8f6051232767acc25ea05b92 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Mon, 25 Sep 2023 19:19:55 +0800 Subject: [PATCH 12/13] add quantization import to library (#562) * add quantization import to library * pre-commit passed --- modelscope/__init__.py | 7 ++++--- .../efficient_stable_diffusion.py | 10 ++++++---- modelscope/preprocessors/multi_modal.py | 6 ++++-- modelscope/trainers/hooks/swift/__init__.py | 2 +- modelscope/trainers/hooks/swift/swift_hook.py | 15 ++++++++------- modelscope/utils/hf_util.py | 8 ++++++++ tests/utils/test_hf_util.py | 4 ++++ 7 files changed, 35 insertions(+), 17 deletions(-) diff --git a/modelscope/__init__.py b/modelscope/__init__.py index ac362be1..162673a0 100644 --- a/modelscope/__init__.py +++ b/modelscope/__init__.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from .utils.hub import read_config, create_model_if_not_exist from .utils.logger import get_logger from .utils.constant import Tasks - from .utils.hf_util import AutoConfig, GenerationConfig + from .utils.hf_util import AutoConfig, GenerationConfig, GPTQConfig, BitsAndBytesConfig from .utils.hf_util import (AutoModel, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoModelForSequenceClassification, @@ -74,8 +74,9 @@ else: 'utils.logger': ['get_logger'], 'utils.constant': ['Tasks'], 'utils.hf_util': [ - 'AutoConfig', 'GenerationConfig', 'AutoModel', - 'AutoModelForCausalLM', 'AutoModelForSeq2SeqLM', 'AutoTokenizer', + 'AutoConfig', 'GenerationConfig', 'AutoModel', 'GPTQConfig', + 'BitsAndBytesConfig', 'AutoModelForCausalLM', + 'AutoModelForSeq2SeqLM', 'AutoTokenizer', 'AutoModelForSequenceClassification', 'AutoModelForTokenClassification' ], diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 3830bb52..79ac2c33 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -136,16 +136,18 @@ class EfficientStableDiffusion(TorchModel): 'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10 adapter_config_dict = {} dim_list = [320, 640, 1280] - target_modules_list = [r"(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)", - r"(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)", - r"(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)"] + target_modules_list = [ + r'(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)', + r'(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)', + r'(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)' + ] for dim, target_modules in zip(dim_list, target_modules_list): adapter_config = AdapterConfig( dim=dim, hidden_pos=0, target_modules=target_modules, adapter_length=adapter_length) - adapter_config_dict[f"adapter_{dim}"] = adapter_config + adapter_config_dict[f'adapter_{dim}'] = adapter_config self.unet = Swift.prepare_model(self.unet, adapter_config_dict) elif tuner_name == 'swift-prompt': if not is_swift_available(): diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index 54ad6e97..2f2ff025 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -54,12 +54,14 @@ class DiffusionImageGenerationPreprocessor(Preprocessor): self.preprocessor_std = kwargs.pop('std', [0.5]) self.preprocessor_image_keys = set(kwargs.pop('image_keys', [])) self.center_crop = kwargs.pop('center_crop', True) - + self.transform_input = transforms.Compose([ transforms.Resize( self.preprocessor_resolution, interpolation=transforms.InterpolationMode.BILINEAR), - transforms.CenterCrop(self.preprocessor_resolution) if self.center_crop else transforms.RandomCrop(self.preprocessor_resolution), + transforms.CenterCrop(self.preprocessor_resolution) + if self.center_crop else transforms.RandomCrop( + self.preprocessor_resolution), transforms.ToTensor(), transforms.Normalize(self.preprocessor_mean, self.preprocessor_std), diff --git a/modelscope/trainers/hooks/swift/__init__.py b/modelscope/trainers/hooks/swift/__init__.py index daf16f92..7fa1d057 100644 --- a/modelscope/trainers/hooks/swift/__init__.py +++ b/modelscope/trainers/hooks/swift/__init__.py @@ -1 +1 @@ -from .swift_hook import SwiftHook \ No newline at end of file +from .swift_hook import SwiftHook diff --git a/modelscope/trainers/hooks/swift/swift_hook.py b/modelscope/trainers/hooks/swift/swift_hook.py index 262dd483..b03b8edc 100644 --- a/modelscope/trainers/hooks/swift/swift_hook.py +++ b/modelscope/trainers/hooks/swift/swift_hook.py @@ -9,12 +9,12 @@ from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import \ LoadCheckpointHook from modelscope.trainers.hooks.hook import Hook -from modelscope.utils.import_utils import is_swift_available from modelscope.utils.checkpoint import save_configuration +from modelscope.utils.import_utils import is_swift_available class SwiftCheckpointProcessor(CheckpointProcessor): - + _BIN_FILE_DIR = 'model' SWIFT_SAVE_SUFFIX = '_swift' @@ -37,7 +37,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor): save_configuration(self.output_dir, self.config) for pop_key in [ - 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' + 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' ]: if config.safe_get('train.checkpoint.period.' + pop_key) is not None: @@ -57,9 +57,10 @@ class SwiftCheckpointProcessor(CheckpointProcessor): _swift_output_dir = output_dir + SwiftCheckpointProcessor.SWIFT_SAVE_SUFFIX model.save_pretrained( save_directory=_swift_output_dir, - safe_serialization=config.safe_get('train.checkpoint.safe_serialization', False), - adapter_name=config.safe_get('train.checkpoint.adapter_name', 'default') - ) + safe_serialization=config.safe_get( + 'train.checkpoint.safe_serialization', False), + adapter_name=config.safe_get( + 'train.checkpoint.adapter_name', 'default')) else: model.save_pretrained( output_dir, @@ -109,7 +110,7 @@ class SwiftCheckpointProcessor(CheckpointProcessor): @HOOKS.register_module(module_name=Hooks.SwiftHook) class SwiftHook(Hook): - + _BIN_FILE_DIR = 'model' def __init__(self): diff --git a/modelscope/utils/hf_util.py b/modelscope/utils/hf_util.py index fd367847..3abcce6d 100644 --- a/modelscope/utils/hf_util.py +++ b/modelscope/utils/hf_util.py @@ -13,6 +13,7 @@ from transformers import \ from transformers import \ AutoModelForTokenClassification as AutoModelForTokenClassificationHF from transformers import AutoTokenizer as AutoTokenizerHF +from transformers import BitsAndBytesConfig as BitsAndBytesConfigHF from transformers import GenerationConfig as GenerationConfigHF from transformers import (PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase) @@ -22,6 +23,11 @@ from transformers.models.auto.tokenization_auto import ( from modelscope import snapshot_download from modelscope.utils.constant import Invoke +try: + from transformers import GPTQConfig as GPTQConfigHF +except ImportError: + GPTQConfigHF = None + def user_agent(invoked_by=None): if invoked_by is None: @@ -199,3 +205,5 @@ AutoConfig = get_wrapped_class( AutoConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors']) GenerationConfig = get_wrapped_class( GenerationConfigHF, ignore_file_pattern=[r'\w+\.bin', r'\w+\.safetensors']) +GPTQConfig = GPTQConfigHF +BitsAndBytesConfig = BitsAndBytesConfigHF diff --git a/tests/utils/test_hf_util.py b/tests/utils/test_hf_util.py index 7c10cca6..fcbaf50c 100644 --- a/tests/utils/test_hf_util.py +++ b/tests/utils/test_hf_util.py @@ -25,6 +25,10 @@ class HFUtilTest(unittest.TestCase): self.assertEqual(tokenizer.model_max_length, 4096) self.assertFalse(tokenizer.is_fast) + def test_quantization_import(self): + from modelscope import GPTQConfig, BitsAndBytesConfig + self.assertTrue(BitsAndBytesConfig is not None) + def test_auto_model(self): model = AutoModelForCausalLM.from_pretrained( 'baichuan-inc/baichuan-7B', trust_remote_code=True) From ef97e3b0fe6753af1e333e8cccda35b34208ac07 Mon Sep 17 00:00:00 2001 From: Wang Qiang <37444407+XDUWQ@users.noreply.github.com> Date: Mon, 25 Sep 2023 19:24:54 +0800 Subject: [PATCH 13/13] support swift trainer and pipeline (#547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * support swift trainer and pipeline * support swift lora pipeline * stable diffusion xl trainer * tests sdxl * fix diffusers attention * swift support * support swift sd --------- Co-authored-by: 翊靖 --- .../stable_diffusion/stable_diffusion.py | 2 +- .../stable_diffusion/stable_diffusion_xl.py | 4 +-- .../cones2_inference_pipeline.py | 4 +-- .../stable_diffusion_pipeline.py | 15 ++++++++- .../stable_diffusion_trainer.py | 31 +++++++++++++++++++ 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py index 6267fb9d..06f87287 100644 --- a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py +++ b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py @@ -158,9 +158,9 @@ class StableDiffusion(TorchModel): config: Optional[dict] = None, save_config_function: Callable = save_configuration, **kwargs): - config['pipeline']['type'] = 'diffusers-stable-diffusion' # Skip copying the original weights for lora and dreambooth method if self.lora_tune or self.dreambooth_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion' pass else: super().save_pretrained(target_folder, save_checkpoint_names, diff --git a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py index 23ad6676..e0fa5070 100644 --- a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py +++ b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion_xl.py @@ -244,9 +244,9 @@ class StableDiffusionXL(TorchModel): config: Optional[dict] = None, save_config_function: Callable = save_configuration, **kwargs): - config['pipeline']['type'] = 'diffusers-stable-diffusion-xl' # Skip copying the original weights for lora and dreambooth method - if self.lora_tune or self.dreambooth_tune: + if self.lora_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion-xl' pass else: super().save_pretrained(target_folder, save_checkpoint_names, diff --git a/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py b/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py index 04fd5910..bb48fae5 100644 --- a/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py +++ b/modelscope/pipelines/multi_modal/cone2_pipeline/cones2_inference_pipeline.py @@ -12,7 +12,7 @@ import numpy as np import torch import torch.nn.functional as F from diffusers import LMSDiscreteScheduler, StableDiffusionPipeline -from diffusers.models.cross_attention import CrossAttention +from diffusers.models.attention_processor import Attention from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import \ StableDiffusionPipelineOutput from PIL import Image @@ -245,7 +245,7 @@ class Cones2AttnProcessor: super().__init__() def __call__(self, - attn: CrossAttention, + attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None): diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py index e5345543..a1f60327 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py @@ -17,6 +17,7 @@ from modelscope.pipelines.builder import PIPELINES from modelscope.pipelines.multi_modal.diffusers_wrapped.diffusers_pipeline import \ DiffusersPipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import is_swift_available @PIPELINES.register_module( @@ -38,9 +39,11 @@ class StableDiffusionPipeline(DiffusersPipeline): custom_dir: custom diffusion weight dir for unet. modifier_token: token to use as a modifier for the concept of custom diffusion. use_safetensors: load safetensors weights. + use_swift: Whether to use swift lora dir for unet. """ use_safetensors = kwargs.pop('use_safetensors', False) torch_type = kwargs.pop('torch_type', torch.float32) + use_swift = kwargs.pop('use_swift', False) # check custom diffusion input value if custom_dir is None and modifier_token is not None: raise ValueError( @@ -58,7 +61,17 @@ class StableDiffusionPipeline(DiffusersPipeline): # load lora moudle to unet if lora_dir is not None: assert os.path.exists(lora_dir), f"{lora_dir} isn't exist" - self.pipeline.unet.load_attn_procs(lora_dir) + if use_swift: + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use efficient_tuners.' + ) + from swift import Swift + self.pipeline.unet = Swift.from_pretrained( + self.pipeline.unet, lora_dir) + else: + self.pipeline.unet.load_attn_procs(lora_dir) + # load custom diffusion to unet if custom_dir is not None: assert os.path.exists(custom_dir), f"{custom_dir} isn't exist" diff --git a/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py index 68d7c689..b38e0e42 100644 --- a/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py +++ b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py @@ -1,4 +1,5 @@ # Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os from typing import Union import torch @@ -7,16 +8,46 @@ from torch import nn from modelscope.metainfo import Trainers from modelscope.models.base import Model, TorchModel from modelscope.trainers.builder import TRAINERS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import CheckpointHook +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor from modelscope.trainers.optimizer.builder import build_optimizer from modelscope.trainers.trainer import EpochBasedTrainer from modelscope.utils.config import ConfigDict +class SwiftDiffusionCheckpointProcessor(CheckpointProcessor): + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None, + save_optimizers=True): + """Save the state dict for swift lora tune model. + """ + trainer.model.unet.save_pretrained(os.path.join(output_dir)) + + @TRAINERS.register_module(module_name=Trainers.stable_diffusion) class StableDiffusionTrainer(EpochBasedTrainer): def __init__(self, *args, **kwargs): + """Stable Diffusion trainers for fine-tuning. + + Args: + use_swift: Whether to use swift. + + """ super().__init__(*args, **kwargs) + use_swift = kwargs.pop('use_swift', False) + + # set swift lora save checkpoint processor + if use_swift: + ckpt_hook = list( + filter(lambda hook: isinstance(hook, CheckpointHook), + self.hooks))[0] + ckpt_hook.set_processor(SwiftDiffusionCheckpointProcessor()) def build_optimizer(self, cfg: ConfigDict, default_args: dict = None): try: