Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-07-10 04:22:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into llama_finetune

Browse Source
This commit is contained in:
suluyana
2023-05-15 10:21:22 +08:00
parent dd77941188 52aea36c12
commit ac6620b384
29 changed files with 152 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -108,6 +108,10 @@ Audio:
* [speech_charctc_kws_phone-xiaoyun](https://modelscope.cn/models/damo/speech_charctc_kws_phone-xiaoyun)
* [u2pp_conformer-asr-cn-16k-online](https://modelscope.cn/models/wenet/u2pp_conformer-asr-cn-16k-online)
* [speech_fsmn_vad_zh-cn-16k-common-pytorch](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary)
* [punc_ct-transformer_zh-cn-common-vocab272727-pytorch](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary)
* [speech_frcrn_ans_cirm_16k](https://modelscope.cn/models/damo/speech_frcrn_ans_cirm_16k)

33
examples/pytorch/stable_diffusion/finetune_stable_diffusion.py Normal file
View File

@@ -0,0 +1,33 @@
from dataclasses import dataclass, field
from modelscope.msdatasets import MsDataset
from modelscope.trainers import EpochBasedTrainer, build_trainer
from modelscope.trainers.training_args import TrainingArgs
@dataclass
class StableDiffusionArguments(TrainingArgs):
def __call__(self, config):
config = super().__call__(config)
config.train.lr_scheduler.T_max = self.max_epochs
config.model.inference = False
return config
args = StableDiffusionArguments.from_cli(task='efficient-diffusion-tuning')
print(args)
dataset = MsDataset.load(args.dataset_name, namespace=args.namespace)
train_dataset = dataset['train']
validation_dataset = dataset['validation']
kwargs = dict(
model=args.model,
work_dir=args.work_dir,
train_dataset=train_dataset,
eval_dataset=validation_dataset,
cfg_modify_fn=args)
trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs)
trainer.train()

11
examples/pytorch/stable_diffusion/run_train.sh Normal file
View File

@@ -0,0 +1,11 @@
PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \
--model 'damo/multi-modal_efficient-diffusion-tuning-lora' \
--work_dir './tmp/stable_diffusion_tuning' \
--namespace 'damo' \
--dataset_name 'buptwq/lora-stable-diffusion-finetune-dog' \
--max_epochs 150 \
--save_ckpt_strategy 'by_epoch' \
--logging_interval 100 \
--train.dataloader.workers_per_gpu 0 \
--evaluation.dataloader.workers_per_gpu 0 \
--train.optimizer.lr 1e-4

4
modelscope/models/cv/ocr_detection/preprocessor.py
View File

@@ -37,7 +37,7 @@ class OCRDetectionPreprocessor(Preprocessor):
inputs:
- A string containing an HTTP link pointing to an image
- A string containing a local path to an image
- An image loaded in PIL or opencv directly
- An image loaded in PIL(PIL.Image.Image) or opencv(np.ndarray) directly, 3 channels RGB
Returns:
outputs: the preprocessed image
"""
@@ -45,6 +45,8 @@ class OCRDetectionPreprocessor(Preprocessor):
img = np.array(load_image(inputs))
elif isinstance(inputs, PIL.Image.Image):
img = np.array(inputs)
elif isinstance(inputs, np.ndarray):
img = inputs
else:
raise TypeError(
f'inputs should be either str, PIL.Image, np.array, but got {type(inputs)}'

6
modelscope/pipelines/audio/asr_inference_pipeline.py
View File

@@ -265,6 +265,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
if self.preprocessor is None:
self.preprocessor = WavToScp()
# pipeline() from pipelines/builder.py passes 'device' but 'ngpu' needed here
device = extra_args.get('device')
if device == 'cpu':
extra_args['ngpu'] = 0
elif device == 'gpu':
extra_args['ngpu'] = 1
outputs = self.preprocessor.config_checking(self.model_cfg)
# generate asr inference command
cmd = {

11
modelscope/pipelines/base.py
View File

@@ -295,7 +295,16 @@ class Pipeline(ABC):
out = {}
for k, element in batched_out.items():
if element is not None:
out[k] = element[batch_idx]
if isinstance(element, (tuple, list)):
if isinstance(element[0], torch.Tensor):
out[k] = type(element)(
e[batch_idx:batch_idx + 1]
for e in element)
else:
# Compatible with traditional pipelines
out[k] = element[batch_idx]
else:
out[k] = element[batch_idx:batch_idx + 1]
out = self.postprocess(out, **postprocess_params)
self._check_output(out)
output_list.append(out)

3
modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py
View File

@@ -40,7 +40,8 @@ class DialogIntentPredictionPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
if preprocessor is None:
self.preprocessor = DialogIntentPredictionPreprocessor(
self.model.model_dir, **kwargs)

4
modelscope/pipelines/nlp/dialog_state_tracking_pipeline.py
View File

@@ -42,7 +42,9 @@ class DialogStateTrackingPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
compile=kwargs.pop('compile', False),
compile_options=kwargs.pop('compile_options', {}))
if preprocessor is None:
self.preprocessor = DialogStateTrackingPreprocessor(

3
modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py
View File

@@ -46,7 +46,8 @@ class DocumentGroundedDialogGeneratePipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
if preprocessor is None:
self.preprocessor = DocumentGroundedDialogGeneratePreprocessor(

3
modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py
View File

@@ -64,7 +64,8 @@ class DocumentGroundedDialogRerankPipeline(Pipeline):
config_file=config_file,
device=device,
auto_collate=auto_collate,
seed=seed)
seed=seed,
**kwarg)
self.model = model
self.preprocessor = preprocessor
self.device = device

3
modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py
View File

@@ -55,7 +55,8 @@ class DocumentGroundedDialogRetrievalPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
if preprocessor is None:
self.preprocessor = DocumentGroundedDialogRetrievalPreprocessor(

8
modelscope/pipelines/nlp/document_segmentation_pipeline.py
View File

@@ -48,8 +48,14 @@ class DocumentSegmentationPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
kwargs = kwargs
if 'compile' in kwargs.keys():
kwargs.pop('compile')
if 'compile_options' in kwargs.keys():
kwargs.pop('compile_options')
self.model_dir = self.model.model_dir
self.model_cfg = self.model.model_cfg
if preprocessor is None:

9
modelscope/pipelines/nlp/extractive_summarization_pipeline.py
View File

@@ -41,7 +41,14 @@ class ExtractiveSummarizationPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
kwargs = kwargs
if 'compile' in kwargs.keys():
kwargs.pop('compile')
if 'compile_options' in kwargs.keys():
kwargs.pop('compile_options')
self.model_dir = self.model.model_dir
self.model_cfg = self.model.model_cfg

3
modelscope/pipelines/nlp/feature_extraction_pipeline.py
View File

@@ -53,7 +53,8 @@ class FeatureExtractionPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

4
modelscope/pipelines/nlp/fid_dialogue_pipeline.py
View File

@@ -191,8 +191,8 @@ class FidDialoguePipeline(Pipeline):
def postprocess(self, inputs: TokenGeneratorOutput,
**postprocess_params) -> Dict[str, Any]:
if torch.cuda.is_available():
hypotheses = inputs.sequences.detach().cpu().tolist()
# if torch.cuda.is_available():
hypotheses = inputs.sequences.detach().cpu().tolist()
response = self.preprocessor_tokenizer.decode(
hypotheses[0], skip_special_tokens=self.is_t5)

3
modelscope/pipelines/nlp/fill_mask_pipeline.py
View File

@@ -62,7 +62,8 @@ class FillMaskPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/named_entity_recognition_pipeline.py
View File

@@ -55,7 +55,8 @@ class NamedEntityRecognitionPipeline(TokenClassificationPipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/sentence_embedding_pipeline.py
View File

@@ -42,7 +42,8 @@ class SentenceEmbeddingPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/siamese_uie_pipeline.py
View File

@@ -67,7 +67,8 @@ class SiameseUiePipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/table_question_answering_pipeline.py
View File

@@ -51,7 +51,8 @@ class TableQuestionAnsweringPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/text_generation_pipeline.py
View File

@@ -58,7 +58,8 @@ class TextGenerationPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

3
modelscope/pipelines/nlp/text_ranking_pipeline.py
View File

@@ -43,7 +43,8 @@ class TextRankingPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
assert isinstance(self.model, Model), \
f'please check whether model config exists in {ModelFile.CONFIGURATION}'

6
modelscope/pipelines/nlp/translation_evaluation_pipeline.py
View File

@@ -42,7 +42,11 @@ class TranslationEvaluationPipeline(Pipeline):
`"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the
source/reference/source+reference can be presented during evaluation.
"""
super().__init__(model=model, preprocessor=preprocessor)
super().__init__(
model=model,
preprocessor=preprocessor,
compile=kwargs.pop('compile', False),
compile_options=kwargs.pop('compile_options', {}))
self.eval_mode = eval_mode
self.checking_eval_mode()

6
modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py
View File

@@ -26,7 +26,8 @@ class UserSatisfactionEstimationPipeline(Pipeline):
preprocessor: DialogueClassificationUsePreprocessor = None,
config_file: str = None,
device: str = 'gpu',
auto_collate=True):
auto_collate=True,
**kwargs):
"""The inference pipeline for the user satisfaction estimation task.
Args:
@@ -49,7 +50,8 @@ class UserSatisfactionEstimationPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
if hasattr(self.preprocessor, 'id2label'):
self.id2label = self.preprocessor.id2label

3
modelscope/pipelines/nlp/zero_shot_classification_pipeline.py
View File

@@ -66,7 +66,8 @@ class ZeroShotClassificationPipeline(Pipeline):
preprocessor=preprocessor,
config_file=config_file,
device=device,
auto_collate=auto_collate)
auto_collate=auto_collate,
**kwargs)
self.entailment_id = 0
self.contradiction_id = 2

3
modelscope/trainers/training_args.py
View File

@@ -565,6 +565,9 @@ class TrainingArgs:
'cfg_node': 'evaluation.metrics'
})
namespace: str = field(
default=None, metadata={'help': 'The namespace of dataset'})
@classmethod
def from_cli(cls, parser_args=None, **extra_kwargs):
"""Construct a TrainingArg class by the parameters of CLI.

6
modelscope/utils/error.py
View File

@@ -162,3 +162,9 @@ You can install it with pip on linux or mac:
Or you can checkout the instructions on the
installation page: https://github.com/mlfoundations/open_clip and follow the ones that match your environment.
"""
# docstyle-ignore
TAMING_IMPORT_ERROR = """
{0} requires the timm library but it was not found in your environment. You can install it with pip:
`pip install taming-transformers-rom1504`
"""

1
modelscope/utils/import_utils.py
View File

@@ -305,6 +305,7 @@ REQUIREMENTS_MAAPING = OrderedDict([
TEXT2SQL_LGESQL_IMPORT_ERROR)),
('mpi4py', (is_package_available('mpi4py'), MPI4PY_IMPORT_ERROR)),
('open_clip', (is_package_available('open_clip'), OPENCLIP_IMPORT_ERROR)),
('taming', (is_package_available('taming'), TAMING_IMPORT_ERROR)),
])
SYSTEM_PACKAGE = set(['os', 'sys', 'typing'])

26
tests/pipelines/test_base.py
View File

@@ -70,13 +70,12 @@ class CustomPipelineTest(unittest.TestCase):
preprocessor=None,
**kwargs):
super().__init__(config_file, model, preprocessor, **kwargs)
self._postprocess_inputs = None
def _batch(self, sample_list):
sample_batch = {'img': [], 'url': []}
for sample in sample_list:
resized_img = torch.from_numpy(
np.array(sample['img'].resize((640, 640))))
sample_batch['img'].append(torch.unsqueeze(resized_img, 0))
sample_batch['img'].append(sample['img'])
sample_batch['url'].append(sample['url'])
sample_batch['img'] = torch.concat(sample_batch['img'])
@@ -89,7 +88,11 @@ class CustomPipelineTest(unittest.TestCase):
"""
if not isinstance(input, Image.Image):
from modelscope.preprocessors import load_image
data_dict = {'img': load_image(input), 'url': input}
image = load_image(input)
resized_img = torch.from_numpy(
np.array(image.resize((640, 640))))
unsqueezed_img = torch.unsqueeze(resized_img, 0)
data_dict = {'img': unsqueezed_img, 'url': input}
else:
data_dict = {'img': input}
return data_dict
@@ -101,19 +104,30 @@ class CustomPipelineTest(unittest.TestCase):
return inputs
def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
if self._postprocess_inputs is None:
self._postprocess_inputs = inputs
else:
self._check_postprocess_input(inputs)
inputs['url'] += 'dummy_end'
return inputs
def _check_postprocess_input(self, current_input: Dict[str, Any]):
for key in current_input:
if isinstance(current_input[key], torch.Tensor):
assert len(current_input[key].shape) == len(
self._postprocess_inputs[key].shape)
self.assertTrue(dummy_module in PIPELINES.modules[dummy_task])
add_default_pipeline_info(dummy_task, dummy_module, overwrite=True)
pipe = pipeline(
task=dummy_task, pipeline_name=dummy_module, model=self.model_dir)
img_url = 'data/test/images/dogs.jpg'
pipe(img_url)
output = pipe([img_url for _ in range(9)], batch_size=2)
for out in output:
self.assertEqual(out['url'], img_url + 'dummy_end')
self.assertEqual(out['img'].shape, (640, 640, 3))
self.assertEqual(out['img'].shape, (1, 640, 640, 3))
pipe_nocollate = pipeline(
task=dummy_task,
@@ -125,7 +139,7 @@ class CustomPipelineTest(unittest.TestCase):
output = pipe_nocollate([img_url for _ in range(9)], batch_size=2)
for out in output:
self.assertEqual(out['url'], img_url + 'dummy_end')
self.assertEqual(out['img'].shape, (640, 640, 3))
self.assertEqual(out['img'].shape, (1, 640, 640, 3))
def test_custom(self):
dummy_task = 'dummy-task'