diff --git a/README.md b/README.md index 3bed2b4a..68f03744 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ Audio: * [speech_charctc_kws_phone-xiaoyun](https://modelscope.cn/models/damo/speech_charctc_kws_phone-xiaoyun) * [u2pp_conformer-asr-cn-16k-online](https://modelscope.cn/models/wenet/u2pp_conformer-asr-cn-16k-online) + +* [speech_fsmn_vad_zh-cn-16k-common-pytorch](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) + +* [punc_ct-transformer_zh-cn-common-vocab272727-pytorch](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) * [speech_frcrn_ans_cirm_16k](https://modelscope.cn/models/damo/speech_frcrn_ans_cirm_16k) diff --git a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py new file mode 100644 index 00000000..bd05097d --- /dev/null +++ b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass, field + +from modelscope.msdatasets import MsDataset +from modelscope.trainers import EpochBasedTrainer, build_trainer +from modelscope.trainers.training_args import TrainingArgs + + +@dataclass +class StableDiffusionArguments(TrainingArgs): + + def __call__(self, config): + config = super().__call__(config) + config.train.lr_scheduler.T_max = self.max_epochs + config.model.inference = False + return config + + +args = StableDiffusionArguments.from_cli(task='efficient-diffusion-tuning') +print(args) + +dataset = MsDataset.load(args.dataset_name, namespace=args.namespace) +train_dataset = dataset['train'] +validation_dataset = dataset['validation'] + +kwargs = dict( + model=args.model, + work_dir=args.work_dir, + train_dataset=train_dataset, + eval_dataset=validation_dataset, + cfg_modify_fn=args) + +trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) +trainer.train() diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh new file mode 100644 index 00000000..c8bfa26c --- /dev/null +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -0,0 +1,11 @@ +PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \ + --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ + --work_dir './tmp/stable_diffusion_tuning' \ + --namespace 'damo' \ + --dataset_name 'buptwq/lora-stable-diffusion-finetune-dog' \ + --max_epochs 150 \ + --save_ckpt_strategy 'by_epoch' \ + --logging_interval 100 \ + --train.dataloader.workers_per_gpu 0 \ + --evaluation.dataloader.workers_per_gpu 0 \ + --train.optimizer.lr 1e-4 diff --git a/modelscope/models/cv/ocr_detection/preprocessor.py b/modelscope/models/cv/ocr_detection/preprocessor.py index 4b95b3c9..d3009af8 100644 --- a/modelscope/models/cv/ocr_detection/preprocessor.py +++ b/modelscope/models/cv/ocr_detection/preprocessor.py @@ -37,7 +37,7 @@ class OCRDetectionPreprocessor(Preprocessor): inputs: - A string containing an HTTP link pointing to an image - A string containing a local path to an image - - An image loaded in PIL or opencv directly + - An image loaded in PIL(PIL.Image.Image) or opencv(np.ndarray) directly, 3 channels RGB Returns: outputs: the preprocessed image """ @@ -45,6 +45,8 @@ class OCRDetectionPreprocessor(Preprocessor): img = np.array(load_image(inputs)) elif isinstance(inputs, PIL.Image.Image): img = np.array(inputs) + elif isinstance(inputs, np.ndarray): + img = inputs else: raise TypeError( f'inputs should be either str, PIL.Image, np.array, but got {type(inputs)}' diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 6387bb3a..b5a4cba7 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -265,6 +265,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): if self.preprocessor is None: self.preprocessor = WavToScp() + # pipeline() from pipelines/builder.py passes 'device' but 'ngpu' needed here + device = extra_args.get('device') + if device == 'cpu': + extra_args['ngpu'] = 0 + elif device == 'gpu': + extra_args['ngpu'] = 1 outputs = self.preprocessor.config_checking(self.model_cfg) # generate asr inference command cmd = { diff --git a/modelscope/pipelines/base.py b/modelscope/pipelines/base.py index 192ca3de..ac244421 100644 --- a/modelscope/pipelines/base.py +++ b/modelscope/pipelines/base.py @@ -295,7 +295,16 @@ class Pipeline(ABC): out = {} for k, element in batched_out.items(): if element is not None: - out[k] = element[batch_idx] + if isinstance(element, (tuple, list)): + if isinstance(element[0], torch.Tensor): + out[k] = type(element)( + e[batch_idx:batch_idx + 1] + for e in element) + else: + # Compatible with traditional pipelines + out[k] = element[batch_idx] + else: + out[k] = element[batch_idx:batch_idx + 1] out = self.postprocess(out, **postprocess_params) self._check_output(out) output_list.append(out) diff --git a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py index f53f186c..fa7b23b8 100644 --- a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py @@ -40,7 +40,8 @@ class DialogIntentPredictionPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DialogIntentPredictionPreprocessor( self.model.model_dir, **kwargs) diff --git a/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py b/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py index 207b4f81..75f12a67 100644 --- a/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py @@ -42,7 +42,9 @@ class DialogStateTrackingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if preprocessor is None: self.preprocessor = DialogStateTrackingPreprocessor( diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py index 5fc1a193..8c773dfe 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py @@ -46,7 +46,8 @@ class DocumentGroundedDialogGeneratePipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DocumentGroundedDialogGeneratePreprocessor( diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py index d72366e9..8fdef380 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py @@ -64,7 +64,8 @@ class DocumentGroundedDialogRerankPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - seed=seed) + seed=seed, + **kwarg) self.model = model self.preprocessor = preprocessor self.device = device diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py index e3461b09..c3fb1a32 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py @@ -55,7 +55,8 @@ class DocumentGroundedDialogRetrievalPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DocumentGroundedDialogRetrievalPreprocessor( diff --git a/modelscope/pipelines/nlp/document_segmentation_pipeline.py b/modelscope/pipelines/nlp/document_segmentation_pipeline.py index 6e2121c3..6e195ed0 100644 --- a/modelscope/pipelines/nlp/document_segmentation_pipeline.py +++ b/modelscope/pipelines/nlp/document_segmentation_pipeline.py @@ -48,8 +48,14 @@ class DocumentSegmentationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) + kwargs = kwargs + if 'compile' in kwargs.keys(): + kwargs.pop('compile') + if 'compile_options' in kwargs.keys(): + kwargs.pop('compile_options') self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg if preprocessor is None: diff --git a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py index 1581690e..c01f28fc 100644 --- a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py +++ b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py @@ -41,7 +41,14 @@ class ExtractiveSummarizationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) + + kwargs = kwargs + if 'compile' in kwargs.keys(): + kwargs.pop('compile') + if 'compile_options' in kwargs.keys(): + kwargs.pop('compile_options') self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg diff --git a/modelscope/pipelines/nlp/feature_extraction_pipeline.py b/modelscope/pipelines/nlp/feature_extraction_pipeline.py index 6131fa61..0f6979ba 100644 --- a/modelscope/pipelines/nlp/feature_extraction_pipeline.py +++ b/modelscope/pipelines/nlp/feature_extraction_pipeline.py @@ -53,7 +53,8 @@ class FeatureExtractionPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py index 702f303a..da802868 100644 --- a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py +++ b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py @@ -191,8 +191,8 @@ class FidDialoguePipeline(Pipeline): def postprocess(self, inputs: TokenGeneratorOutput, **postprocess_params) -> Dict[str, Any]: - if torch.cuda.is_available(): - hypotheses = inputs.sequences.detach().cpu().tolist() + # if torch.cuda.is_available(): + hypotheses = inputs.sequences.detach().cpu().tolist() response = self.preprocessor_tokenizer.decode( hypotheses[0], skip_special_tokens=self.is_t5) diff --git a/modelscope/pipelines/nlp/fill_mask_pipeline.py b/modelscope/pipelines/nlp/fill_mask_pipeline.py index dc12efa7..6bc7622f 100644 --- a/modelscope/pipelines/nlp/fill_mask_pipeline.py +++ b/modelscope/pipelines/nlp/fill_mask_pipeline.py @@ -62,7 +62,8 @@ class FillMaskPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py index ba174bae..2cf30037 100644 --- a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py +++ b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py @@ -55,7 +55,8 @@ class NamedEntityRecognitionPipeline(TokenClassificationPipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py index 7aaa073b..4e01397d 100644 --- a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py +++ b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py @@ -42,7 +42,8 @@ class SentenceEmbeddingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/siamese_uie_pipeline.py b/modelscope/pipelines/nlp/siamese_uie_pipeline.py index 21582900..cdbd9119 100644 --- a/modelscope/pipelines/nlp/siamese_uie_pipeline.py +++ b/modelscope/pipelines/nlp/siamese_uie_pipeline.py @@ -67,7 +67,8 @@ class SiameseUiePipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/table_question_answering_pipeline.py b/modelscope/pipelines/nlp/table_question_answering_pipeline.py index 365c6c6c..0472ecb8 100644 --- a/modelscope/pipelines/nlp/table_question_answering_pipeline.py +++ b/modelscope/pipelines/nlp/table_question_answering_pipeline.py @@ -51,7 +51,8 @@ class TableQuestionAnsweringPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 61f3a421..2b851dc4 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -58,7 +58,8 @@ class TextGenerationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_ranking_pipeline.py b/modelscope/pipelines/nlp/text_ranking_pipeline.py index 1b313acc..a42baaa2 100644 --- a/modelscope/pipelines/nlp/text_ranking_pipeline.py +++ b/modelscope/pipelines/nlp/text_ranking_pipeline.py @@ -43,7 +43,8 @@ class TextRankingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py index 1f8ba79a..8a339517 100644 --- a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py +++ b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py @@ -42,7 +42,11 @@ class TranslationEvaluationPipeline(Pipeline): `"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the source/reference/source+reference can be presented during evaluation. """ - super().__init__(model=model, preprocessor=preprocessor) + super().__init__( + model=model, + preprocessor=preprocessor, + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) self.eval_mode = eval_mode self.checking_eval_mode() diff --git a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py index fc55dc84..76fcd7a8 100644 --- a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py +++ b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py @@ -26,7 +26,8 @@ class UserSatisfactionEstimationPipeline(Pipeline): preprocessor: DialogueClassificationUsePreprocessor = None, config_file: str = None, device: str = 'gpu', - auto_collate=True): + auto_collate=True, + **kwargs): """The inference pipeline for the user satisfaction estimation task. Args: @@ -49,7 +50,8 @@ class UserSatisfactionEstimationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if hasattr(self.preprocessor, 'id2label'): self.id2label = self.preprocessor.id2label diff --git a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py index 5bc611fb..9cd27adc 100644 --- a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py +++ b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py @@ -66,7 +66,8 @@ class ZeroShotClassificationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) self.entailment_id = 0 self.contradiction_id = 2 diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index a8eac217..f4e4e138 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -565,6 +565,9 @@ class TrainingArgs: 'cfg_node': 'evaluation.metrics' }) + namespace: str = field( + default=None, metadata={'help': 'The namespace of dataset'}) + @classmethod def from_cli(cls, parser_args=None, **extra_kwargs): """Construct a TrainingArg class by the parameters of CLI. diff --git a/modelscope/utils/error.py b/modelscope/utils/error.py index d3000130..841662c0 100644 --- a/modelscope/utils/error.py +++ b/modelscope/utils/error.py @@ -162,3 +162,9 @@ You can install it with pip on linux or mac: Or you can checkout the instructions on the installation page: https://github.com/mlfoundations/open_clip and follow the ones that match your environment. """ + +# docstyle-ignore +TAMING_IMPORT_ERROR = """ +{0} requires the timm library but it was not found in your environment. You can install it with pip: +`pip install taming-transformers-rom1504` +""" diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index ea123ed7..3e8be2e1 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -305,6 +305,7 @@ REQUIREMENTS_MAAPING = OrderedDict([ TEXT2SQL_LGESQL_IMPORT_ERROR)), ('mpi4py', (is_package_available('mpi4py'), MPI4PY_IMPORT_ERROR)), ('open_clip', (is_package_available('open_clip'), OPENCLIP_IMPORT_ERROR)), + ('taming', (is_package_available('taming'), TAMING_IMPORT_ERROR)), ]) SYSTEM_PACKAGE = set(['os', 'sys', 'typing']) diff --git a/tests/pipelines/test_base.py b/tests/pipelines/test_base.py index 32bf8877..ec58811f 100644 --- a/tests/pipelines/test_base.py +++ b/tests/pipelines/test_base.py @@ -70,13 +70,12 @@ class CustomPipelineTest(unittest.TestCase): preprocessor=None, **kwargs): super().__init__(config_file, model, preprocessor, **kwargs) + self._postprocess_inputs = None def _batch(self, sample_list): sample_batch = {'img': [], 'url': []} for sample in sample_list: - resized_img = torch.from_numpy( - np.array(sample['img'].resize((640, 640)))) - sample_batch['img'].append(torch.unsqueeze(resized_img, 0)) + sample_batch['img'].append(sample['img']) sample_batch['url'].append(sample['url']) sample_batch['img'] = torch.concat(sample_batch['img']) @@ -89,7 +88,11 @@ class CustomPipelineTest(unittest.TestCase): """ if not isinstance(input, Image.Image): from modelscope.preprocessors import load_image - data_dict = {'img': load_image(input), 'url': input} + image = load_image(input) + resized_img = torch.from_numpy( + np.array(image.resize((640, 640)))) + unsqueezed_img = torch.unsqueeze(resized_img, 0) + data_dict = {'img': unsqueezed_img, 'url': input} else: data_dict = {'img': input} return data_dict @@ -101,19 +104,30 @@ class CustomPipelineTest(unittest.TestCase): return inputs def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + if self._postprocess_inputs is None: + self._postprocess_inputs = inputs + else: + self._check_postprocess_input(inputs) inputs['url'] += 'dummy_end' return inputs + def _check_postprocess_input(self, current_input: Dict[str, Any]): + for key in current_input: + if isinstance(current_input[key], torch.Tensor): + assert len(current_input[key].shape) == len( + self._postprocess_inputs[key].shape) + self.assertTrue(dummy_module in PIPELINES.modules[dummy_task]) add_default_pipeline_info(dummy_task, dummy_module, overwrite=True) pipe = pipeline( task=dummy_task, pipeline_name=dummy_module, model=self.model_dir) img_url = 'data/test/images/dogs.jpg' + pipe(img_url) output = pipe([img_url for _ in range(9)], batch_size=2) for out in output: self.assertEqual(out['url'], img_url + 'dummy_end') - self.assertEqual(out['img'].shape, (640, 640, 3)) + self.assertEqual(out['img'].shape, (1, 640, 640, 3)) pipe_nocollate = pipeline( task=dummy_task, @@ -125,7 +139,7 @@ class CustomPipelineTest(unittest.TestCase): output = pipe_nocollate([img_url for _ in range(9)], batch_size=2) for out in output: self.assertEqual(out['url'], img_url + 'dummy_end') - self.assertEqual(out['img'].shape, (640, 640, 3)) + self.assertEqual(out['img'].shape, (1, 640, 640, 3)) def test_custom(self): dummy_task = 'dummy-task'