From 9bfc4a9d83c4beaf8378d0a186261ffc1cd9f960 Mon Sep 17 00:00:00 2001 From: Wang Qiang <37444407+XDUWQ@users.noreply.github.com> Date: Fri, 28 Apr 2023 10:18:42 +0800 Subject: [PATCH 01/10] Add finetuning stable diffusion example (#285) --- .../finetune_stable_diffusion.py | 33 +++++++++++++++++++ .../pytorch/stable_diffusion/run_train.sh | 11 +++++++ modelscope/trainers/training_args.py | 3 ++ 3 files changed, 47 insertions(+) create mode 100644 examples/pytorch/stable_diffusion/finetune_stable_diffusion.py create mode 100644 examples/pytorch/stable_diffusion/run_train.sh diff --git a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py new file mode 100644 index 00000000..bd05097d --- /dev/null +++ b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass, field + +from modelscope.msdatasets import MsDataset +from modelscope.trainers import EpochBasedTrainer, build_trainer +from modelscope.trainers.training_args import TrainingArgs + + +@dataclass +class StableDiffusionArguments(TrainingArgs): + + def __call__(self, config): + config = super().__call__(config) + config.train.lr_scheduler.T_max = self.max_epochs + config.model.inference = False + return config + + +args = StableDiffusionArguments.from_cli(task='efficient-diffusion-tuning') +print(args) + +dataset = MsDataset.load(args.dataset_name, namespace=args.namespace) +train_dataset = dataset['train'] +validation_dataset = dataset['validation'] + +kwargs = dict( + model=args.model, + work_dir=args.work_dir, + train_dataset=train_dataset, + eval_dataset=validation_dataset, + cfg_modify_fn=args) + +trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) +trainer.train() diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh new file mode 100644 index 00000000..e3b78691 --- /dev/null +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -0,0 +1,11 @@ +PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \ + --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ + --work_dir './tmp/stable_diffusion_tuning' \ + --namespace 'damo' \ + --dataset_name 'controlnet_dataset_condition_fill50k' \ + --max_epochs 1 \ + --save_ckpt_strategy 'by_epoch' \ + --logging_interval 100 \ + --train.dataloader.workers_per_gpu 0 \ + --evaluation.dataloader.workers_per_gpu 0 \ + --train.optimizer.lr 1e-5 diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index a8eac217..f4e4e138 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -565,6 +565,9 @@ class TrainingArgs: 'cfg_node': 'evaluation.metrics' }) + namespace: str = field( + default=None, metadata={'help': 'The namespace of dataset'}) + @classmethod def from_cli(cls, parser_args=None, **extra_kwargs): """Construct a TrainingArg class by the parameters of CLI. From 1f2bb82ea6f6c0f72918ecaef11e2d79a3db98cb Mon Sep 17 00:00:00 2001 From: leway Date: Sun, 7 May 2023 14:51:47 +0800 Subject: [PATCH 02/10] Update fid_dialogue_pipeline.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复在cpu下推理时的错误，这里不应该检查cuda是否可用。这里的逻辑似乎有错误 --- modelscope/pipelines/nlp/fid_dialogue_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py index 702f303a..da802868 100644 --- a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py +++ b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py @@ -191,8 +191,8 @@ class FidDialoguePipeline(Pipeline): def postprocess(self, inputs: TokenGeneratorOutput, **postprocess_params) -> Dict[str, Any]: - if torch.cuda.is_available(): - hypotheses = inputs.sequences.detach().cpu().tolist() + # if torch.cuda.is_available(): + hypotheses = inputs.sequences.detach().cpu().tolist() response = self.preprocessor_tokenizer.decode( hypotheses[0], skip_special_tokens=self.is_t5) From cb35b9c04e2ca97e353f2e434d9fb8e2e64ecf1b Mon Sep 17 00:00:00 2001 From: xfanplus Date: Wed, 10 May 2023 10:34:42 +0800 Subject: [PATCH 03/10] fix type checking of inputs for np.array (#271) * fix type checking of inputs for np.array inputs type of np.ndarray is not checked correctly. * add docstr for /preprocessor.py add docstr about np.ndarray, shape and order --- modelscope/models/cv/ocr_detection/preprocessor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modelscope/models/cv/ocr_detection/preprocessor.py b/modelscope/models/cv/ocr_detection/preprocessor.py index 4b95b3c9..d3009af8 100644 --- a/modelscope/models/cv/ocr_detection/preprocessor.py +++ b/modelscope/models/cv/ocr_detection/preprocessor.py @@ -37,7 +37,7 @@ class OCRDetectionPreprocessor(Preprocessor): inputs: - A string containing an HTTP link pointing to an image - A string containing a local path to an image - - An image loaded in PIL or opencv directly + - An image loaded in PIL(PIL.Image.Image) or opencv(np.ndarray) directly, 3 channels RGB Returns: outputs: the preprocessed image """ @@ -45,6 +45,8 @@ class OCRDetectionPreprocessor(Preprocessor): img = np.array(load_image(inputs)) elif isinstance(inputs, PIL.Image.Image): img = np.array(inputs) + elif isinstance(inputs, np.ndarray): + img = inputs else: raise TypeError( f'inputs should be either str, PIL.Image, np.array, but got {type(inputs)}' From 626b60e8c25bc6498eab058c1a44f92c9d046321 Mon Sep 17 00:00:00 2001 From: Li Tong <327578505@qq.com> Date: Wed, 10 May 2023 10:49:54 +0800 Subject: [PATCH 04/10] fix issue: Batch and single are not uniform #215 (#258) * fix issue: Batch and single are not uniform #215, https://github.com/modelscope/modelscope/issues/215 * remove extra code * fix bug * update test --------- Co-authored-by: ezeli.lt Co-authored-by: hemu --- modelscope/pipelines/base.py | 11 ++++++++++- tests/pipelines/test_base.py | 26 ++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/modelscope/pipelines/base.py b/modelscope/pipelines/base.py index 192ca3de..ac244421 100644 --- a/modelscope/pipelines/base.py +++ b/modelscope/pipelines/base.py @@ -295,7 +295,16 @@ class Pipeline(ABC): out = {} for k, element in batched_out.items(): if element is not None: - out[k] = element[batch_idx] + if isinstance(element, (tuple, list)): + if isinstance(element[0], torch.Tensor): + out[k] = type(element)( + e[batch_idx:batch_idx + 1] + for e in element) + else: + # Compatible with traditional pipelines + out[k] = element[batch_idx] + else: + out[k] = element[batch_idx:batch_idx + 1] out = self.postprocess(out, **postprocess_params) self._check_output(out) output_list.append(out) diff --git a/tests/pipelines/test_base.py b/tests/pipelines/test_base.py index 32bf8877..ec58811f 100644 --- a/tests/pipelines/test_base.py +++ b/tests/pipelines/test_base.py @@ -70,13 +70,12 @@ class CustomPipelineTest(unittest.TestCase): preprocessor=None, **kwargs): super().__init__(config_file, model, preprocessor, **kwargs) + self._postprocess_inputs = None def _batch(self, sample_list): sample_batch = {'img': [], 'url': []} for sample in sample_list: - resized_img = torch.from_numpy( - np.array(sample['img'].resize((640, 640)))) - sample_batch['img'].append(torch.unsqueeze(resized_img, 0)) + sample_batch['img'].append(sample['img']) sample_batch['url'].append(sample['url']) sample_batch['img'] = torch.concat(sample_batch['img']) @@ -89,7 +88,11 @@ class CustomPipelineTest(unittest.TestCase): """ if not isinstance(input, Image.Image): from modelscope.preprocessors import load_image - data_dict = {'img': load_image(input), 'url': input} + image = load_image(input) + resized_img = torch.from_numpy( + np.array(image.resize((640, 640)))) + unsqueezed_img = torch.unsqueeze(resized_img, 0) + data_dict = {'img': unsqueezed_img, 'url': input} else: data_dict = {'img': input} return data_dict @@ -101,19 +104,30 @@ class CustomPipelineTest(unittest.TestCase): return inputs def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + if self._postprocess_inputs is None: + self._postprocess_inputs = inputs + else: + self._check_postprocess_input(inputs) inputs['url'] += 'dummy_end' return inputs + def _check_postprocess_input(self, current_input: Dict[str, Any]): + for key in current_input: + if isinstance(current_input[key], torch.Tensor): + assert len(current_input[key].shape) == len( + self._postprocess_inputs[key].shape) + self.assertTrue(dummy_module in PIPELINES.modules[dummy_task]) add_default_pipeline_info(dummy_task, dummy_module, overwrite=True) pipe = pipeline( task=dummy_task, pipeline_name=dummy_module, model=self.model_dir) img_url = 'data/test/images/dogs.jpg' + pipe(img_url) output = pipe([img_url for _ in range(9)], batch_size=2) for out in output: self.assertEqual(out['url'], img_url + 'dummy_end') - self.assertEqual(out['img'].shape, (640, 640, 3)) + self.assertEqual(out['img'].shape, (1, 640, 640, 3)) pipe_nocollate = pipeline( task=dummy_task, @@ -125,7 +139,7 @@ class CustomPipelineTest(unittest.TestCase): output = pipe_nocollate([img_url for _ in range(9)], batch_size=2) for out in output: self.assertEqual(out['url'], img_url + 'dummy_end') - self.assertEqual(out['img'].shape, (640, 640, 3)) + self.assertEqual(out['img'].shape, (1, 640, 640, 3)) def test_custom(self): dummy_task = 'dummy-task' From 1c5222d6c1b5490d8df4ad8ebc5104138f9f81c6 Mon Sep 17 00:00:00 2001 From: slin000111 Date: Thu, 11 May 2023 10:27:45 +0800 Subject: [PATCH 05/10] add argument compile to nlp pipelines --- .../pipelines/nlp/dialog_intent_prediction_pipeline.py | 3 ++- .../pipelines/nlp/dialog_state_tracking_pipeline.py | 4 +++- .../nlp/document_grounded_dialog_generate_pipeline.py | 3 ++- .../nlp/document_grounded_dialog_rerank_pipeline.py | 3 ++- .../nlp/document_grounded_dialog_retrieval_pipeline.py | 3 ++- .../pipelines/nlp/document_segmentation_pipeline.py | 8 +++++++- .../pipelines/nlp/extractive_summarization_pipeline.py | 9 ++++++++- modelscope/pipelines/nlp/feature_extraction_pipeline.py | 3 ++- modelscope/pipelines/nlp/fill_mask_pipeline.py | 3 ++- .../pipelines/nlp/named_entity_recognition_pipeline.py | 3 ++- modelscope/pipelines/nlp/sentence_embedding_pipeline.py | 3 ++- modelscope/pipelines/nlp/siamese_uie_pipeline.py | 3 ++- .../pipelines/nlp/table_question_answering_pipeline.py | 3 ++- modelscope/pipelines/nlp/text_generation_pipeline.py | 3 ++- modelscope/pipelines/nlp/text_ranking_pipeline.py | 3 ++- .../pipelines/nlp/translation_evaluation_pipeline.py | 2 +- .../nlp/user_satisfaction_estimation_pipeline.py | 6 ++++-- .../pipelines/nlp/zero_shot_classification_pipeline.py | 3 ++- 18 files changed, 49 insertions(+), 19 deletions(-) diff --git a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py index f53f186c..fa7b23b8 100644 --- a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py @@ -40,7 +40,8 @@ class DialogIntentPredictionPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DialogIntentPredictionPreprocessor( self.model.model_dir, **kwargs) diff --git a/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py b/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py index 207b4f81..75f12a67 100644 --- a/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py @@ -42,7 +42,9 @@ class DialogStateTrackingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if preprocessor is None: self.preprocessor = DialogStateTrackingPreprocessor( diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py index 5fc1a193..8c773dfe 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py @@ -46,7 +46,8 @@ class DocumentGroundedDialogGeneratePipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DocumentGroundedDialogGeneratePreprocessor( diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py index d72366e9..8fdef380 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py @@ -64,7 +64,8 @@ class DocumentGroundedDialogRerankPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - seed=seed) + seed=seed, + **kwarg) self.model = model self.preprocessor = preprocessor self.device = device diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py index e3461b09..c3fb1a32 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py @@ -55,7 +55,8 @@ class DocumentGroundedDialogRetrievalPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if preprocessor is None: self.preprocessor = DocumentGroundedDialogRetrievalPreprocessor( diff --git a/modelscope/pipelines/nlp/document_segmentation_pipeline.py b/modelscope/pipelines/nlp/document_segmentation_pipeline.py index 6e2121c3..6e195ed0 100644 --- a/modelscope/pipelines/nlp/document_segmentation_pipeline.py +++ b/modelscope/pipelines/nlp/document_segmentation_pipeline.py @@ -48,8 +48,14 @@ class DocumentSegmentationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) + kwargs = kwargs + if 'compile' in kwargs.keys(): + kwargs.pop('compile') + if 'compile_options' in kwargs.keys(): + kwargs.pop('compile_options') self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg if preprocessor is None: diff --git a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py index 1581690e..c01f28fc 100644 --- a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py +++ b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py @@ -41,7 +41,14 @@ class ExtractiveSummarizationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) + + kwargs = kwargs + if 'compile' in kwargs.keys(): + kwargs.pop('compile') + if 'compile_options' in kwargs.keys(): + kwargs.pop('compile_options') self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg diff --git a/modelscope/pipelines/nlp/feature_extraction_pipeline.py b/modelscope/pipelines/nlp/feature_extraction_pipeline.py index 6131fa61..0f6979ba 100644 --- a/modelscope/pipelines/nlp/feature_extraction_pipeline.py +++ b/modelscope/pipelines/nlp/feature_extraction_pipeline.py @@ -53,7 +53,8 @@ class FeatureExtractionPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/fill_mask_pipeline.py b/modelscope/pipelines/nlp/fill_mask_pipeline.py index dc12efa7..6bc7622f 100644 --- a/modelscope/pipelines/nlp/fill_mask_pipeline.py +++ b/modelscope/pipelines/nlp/fill_mask_pipeline.py @@ -62,7 +62,8 @@ class FillMaskPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py index ba174bae..2cf30037 100644 --- a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py +++ b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py @@ -55,7 +55,8 @@ class NamedEntityRecognitionPipeline(TokenClassificationPipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py index 7aaa073b..4e01397d 100644 --- a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py +++ b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py @@ -42,7 +42,8 @@ class SentenceEmbeddingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/siamese_uie_pipeline.py b/modelscope/pipelines/nlp/siamese_uie_pipeline.py index 21582900..cdbd9119 100644 --- a/modelscope/pipelines/nlp/siamese_uie_pipeline.py +++ b/modelscope/pipelines/nlp/siamese_uie_pipeline.py @@ -67,7 +67,8 @@ class SiameseUiePipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/table_question_answering_pipeline.py b/modelscope/pipelines/nlp/table_question_answering_pipeline.py index 365c6c6c..0472ecb8 100644 --- a/modelscope/pipelines/nlp/table_question_answering_pipeline.py +++ b/modelscope/pipelines/nlp/table_question_answering_pipeline.py @@ -51,7 +51,8 @@ class TableQuestionAnsweringPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 61f3a421..2b851dc4 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -58,7 +58,8 @@ class TextGenerationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_ranking_pipeline.py b/modelscope/pipelines/nlp/text_ranking_pipeline.py index 1b313acc..a42baaa2 100644 --- a/modelscope/pipelines/nlp/text_ranking_pipeline.py +++ b/modelscope/pipelines/nlp/text_ranking_pipeline.py @@ -43,7 +43,8 @@ class TextRankingPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py index 1f8ba79a..a9e929d1 100644 --- a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py +++ b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py @@ -42,7 +42,7 @@ class TranslationEvaluationPipeline(Pipeline): `"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the source/reference/source+reference can be presented during evaluation. """ - super().__init__(model=model, preprocessor=preprocessor) + super().__init__(model=model, preprocessor=preprocessor, **kwargs) self.eval_mode = eval_mode self.checking_eval_mode() diff --git a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py index fc55dc84..76fcd7a8 100644 --- a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py +++ b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py @@ -26,7 +26,8 @@ class UserSatisfactionEstimationPipeline(Pipeline): preprocessor: DialogueClassificationUsePreprocessor = None, config_file: str = None, device: str = 'gpu', - auto_collate=True): + auto_collate=True, + **kwargs): """The inference pipeline for the user satisfaction estimation task. Args: @@ -49,7 +50,8 @@ class UserSatisfactionEstimationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) if hasattr(self.preprocessor, 'id2label'): self.id2label = self.preprocessor.id2label diff --git a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py index 5bc611fb..9cd27adc 100644 --- a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py +++ b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py @@ -66,7 +66,8 @@ class ZeroShotClassificationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + **kwargs) self.entailment_id = 0 self.contradiction_id = 2 From d76303d324e747196587578e5ea24d81a6798ae3 Mon Sep 17 00:00:00 2001 From: slin000111 Date: Thu, 11 May 2023 15:11:22 +0800 Subject: [PATCH 06/10] translation evaluation pipeline --- modelscope/pipelines/nlp/translation_evaluation_pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py index a9e929d1..8a339517 100644 --- a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py +++ b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py @@ -42,7 +42,11 @@ class TranslationEvaluationPipeline(Pipeline): `"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the source/reference/source+reference can be presented during evaluation. """ - super().__init__(model=model, preprocessor=preprocessor, **kwargs) + super().__init__( + model=model, + preprocessor=preprocessor, + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) self.eval_mode = eval_mode self.checking_eval_mode() From 63777b995d090200b99ec15736280ac6de3b1433 Mon Sep 17 00:00:00 2001 From: Wang Qiang <37444407+XDUWQ@users.noreply.github.com> Date: Thu, 11 May 2023 15:23:22 +0800 Subject: [PATCH 07/10] Fix bug: install taming error prompt (#296) --- modelscope/utils/error.py | 6 ++++++ modelscope/utils/import_utils.py | 1 + 2 files changed, 7 insertions(+) diff --git a/modelscope/utils/error.py b/modelscope/utils/error.py index d3000130..841662c0 100644 --- a/modelscope/utils/error.py +++ b/modelscope/utils/error.py @@ -162,3 +162,9 @@ You can install it with pip on linux or mac: Or you can checkout the instructions on the installation page: https://github.com/mlfoundations/open_clip and follow the ones that match your environment. """ + +# docstyle-ignore +TAMING_IMPORT_ERROR = """ +{0} requires the timm library but it was not found in your environment. You can install it with pip: +`pip install taming-transformers-rom1504` +""" diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index ea123ed7..3e8be2e1 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -305,6 +305,7 @@ REQUIREMENTS_MAAPING = OrderedDict([ TEXT2SQL_LGESQL_IMPORT_ERROR)), ('mpi4py', (is_package_available('mpi4py'), MPI4PY_IMPORT_ERROR)), ('open_clip', (is_package_available('open_clip'), OPENCLIP_IMPORT_ERROR)), + ('taming', (is_package_available('taming'), TAMING_IMPORT_ERROR)), ]) SYSTEM_PACKAGE = set(['os', 'sys', 'typing']) From d88ec545001c24b373f68bc6856c30ec2d2f0ad5 Mon Sep 17 00:00:00 2001 From: veelion Date: Fri, 12 May 2023 18:15:18 +0800 Subject: [PATCH 08/10] fix: device arg not work, rename device to ngpu (#272) --- modelscope/pipelines/audio/asr_inference_pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 6387bb3a..b5a4cba7 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -265,6 +265,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): if self.preprocessor is None: self.preprocessor = WavToScp() + # pipeline() from pipelines/builder.py passes 'device' but 'ngpu' needed here + device = extra_args.get('device') + if device == 'cpu': + extra_args['ngpu'] = 0 + elif device == 'gpu': + extra_args['ngpu'] = 1 outputs = self.preprocessor.config_checking(self.model_cfg) # generate asr inference command cmd = { From 9ba53ad3baec1e5ef4ff370d47a4cbd00b133118 Mon Sep 17 00:00:00 2001 From: Wang Qiang <37444407+XDUWQ@users.noreply.github.com> Date: Fri, 12 May 2023 18:15:53 +0800 Subject: [PATCH 09/10] Correcting the lora stable diffusion example script (#300) --- examples/pytorch/stable_diffusion/run_train.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh index e3b78691..c8bfa26c 100644 --- a/examples/pytorch/stable_diffusion/run_train.sh +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -2,10 +2,10 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusio --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ --work_dir './tmp/stable_diffusion_tuning' \ --namespace 'damo' \ - --dataset_name 'controlnet_dataset_condition_fill50k' \ - --max_epochs 1 \ + --dataset_name 'buptwq/lora-stable-diffusion-finetune-dog' \ + --max_epochs 150 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 100 \ --train.dataloader.workers_per_gpu 0 \ --evaluation.dataloader.workers_per_gpu 0 \ - --train.optimizer.lr 1e-5 + --train.optimizer.lr 1e-4 From 34e6f76c80e0ad988183e113372998eb274eacea Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Fri, 12 May 2023 18:46:24 +0800 Subject: [PATCH 10/10] add vad model and punc model in README.md add vad model and punc model --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3bed2b4a..68f03744 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ Audio: * [speech_charctc_kws_phone-xiaoyun](https://modelscope.cn/models/damo/speech_charctc_kws_phone-xiaoyun) * [u2pp_conformer-asr-cn-16k-online](https://modelscope.cn/models/wenet/u2pp_conformer-asr-cn-16k-online) + +* [speech_fsmn_vad_zh-cn-16k-common-pytorch](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) + +* [punc_ct-transformer_zh-cn-common-vocab272727-pytorch](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) * [speech_frcrn_ans_cirm_16k](https://modelscope.cn/models/damo/speech_frcrn_ans_cirm_16k)