fix transformer example and fix some bugs

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12626375

examples/pytorch/image_classification/finetune_image_classification.py

@@ -1,13 +1,12 @@
 import os
 from dataclasses import dataclass, field
 
+from modelscope import MsDataset, TrainingArgs
 from modelscope.metainfo import Trainers
-from modelscope.msdatasets.ms_dataset import MsDataset
-from modelscope.trainers.args import TrainingArgs
 from modelscope.trainers.builder import build_trainer
 
 
-@dataclass
+@dataclass(init=False)
 class ImageClassificationTrainingArgs(TrainingArgs):
     num_classes: int = field(
         default=None,
@@ -46,26 +45,35 @@ def create_dataset(name, split):
         dataset_name, namespace=namespace, subset_name='default', split=split)
 
 
-def train():
-    args = ImageClassificationTrainingArgs.from_cli(
-        model='damo/cv_vit-base_image-classification_ImageNet-labels',
-        max_epochs=1,
-        lr=1e-4,
-        optimizer='AdamW',
-        warmup_iters=1,
-        topk=(1, ))
-    if args.dataset_name is not None:
-        train_dataset = create_dataset(args.dataset_name, split='train')
-        val_dataset = create_dataset(args.dataset_name, split='validation')
+training_args = ImageClassificationTrainingArgs(
+    model='damo/cv_vit-base_image-classification_ImageNet-labels',
+    max_epochs=1,
+    lr=1e-4,
+    optimizer='AdamW',
+    warmup_iters=1,
+    topk=(1, )).parse_cli()
+config, args = training_args.to_config()
+
+
+def cfg_modify_fn(cfg):
+    if args.use_model_config:
+        cfg.merge_from_dict(config)
     else:
-        train_dataset = create_dataset(args.train_dataset_name, split='train')
-        val_dataset = create_dataset(args.val_dataset_name, split='validation')
+        cfg = config
+    return cfg
+
+
+def train():
+    train_dataset = create_dataset(
+        training_args.train_dataset_name, split=training_args.train_split)
+    val_dataset = create_dataset(
+        training_args.val_dataset_name, split=training_args.val_split)
 
     kwargs = dict(
         model=args.model,  # model id
         train_dataset=train_dataset,  # training dataset
         eval_dataset=val_dataset,  # validation dataset
-        cfg_modify_fn=args  # callback to modify configuration
+        cfg_modify_fn=cfg_modify_fn  # callback to modify configuration
     )
 
     # in distributed training, specify pytorch launcher

examples/pytorch/image_classification/run_train.sh

@@ -2,4 +2,7 @@ PYTHONPATH=. python -m torch.distributed.launch --nproc_per_node=2 \
     examples/pytorch/image_classification/finetune_image_classification.py \
     --num_classes 2 \
     --train_dataset_name 'tany0699/cats_and_dogs' \
-    --val_dataset_name 'tany0699/cats_and_dogs'
+    --val_dataset_name 'tany0699/cats_and_dogs' \
+    --train_split train \
+    --val_split validation \
+    --use_model_config true \

examples/pytorch/stable_diffusion/finetune_stable_diffusion.py

@@ -4,30 +4,32 @@ from modelscope.msdatasets import MsDataset
 from modelscope.trainers import EpochBasedTrainer, build_trainer
 from modelscope.trainers.training_args import TrainingArgs
 
-
-@dataclass
-class StableDiffusionArguments(TrainingArgs):
-
-    def __call__(self, config):
-        config = super().__call__(config)
-        config.train.lr_scheduler.T_max = self.max_epochs
-        config.model.inference = False
-        return config
-
-
-args = StableDiffusionArguments.from_cli(task='efficient-diffusion-tuning')
+training_args = TrainingArgs(task='efficient-diffusion-tuning').parse_cli()
+config, args = training_args.to_config()
 print(args)
 
-dataset = MsDataset.load(args.dataset_name, namespace=args.namespace)
+dataset = MsDataset.load(
+    args.train_dataset_name, namespace=args.train_dataset_namespace)
 train_dataset = dataset['train']
 validation_dataset = dataset['validation']
 
+
+def cfg_modify_fn(cfg):
+    if args.use_model_config:
+        cfg.merge_from_dict(config)
+    else:
+        cfg = config
+    cfg.train.lr_scheduler.T_max = training_args.max_epochs
+    cfg.model.inference = False
+    return cfg
+
+
 kwargs = dict(
-    model=args.model,
-    work_dir=args.work_dir,
+    model=training_args.model,
+    work_dir=training_args.work_dir,
     train_dataset=train_dataset,
     eval_dataset=validation_dataset,
-    cfg_modify_fn=args)
+    cfg_modify_fn=cfg_modify_fn)
 
 trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs)
 trainer.train()

examples/pytorch/stable_diffusion/run_train.sh

@@ -1,11 +1,12 @@
 PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \
     --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \
     --work_dir './tmp/stable_diffusion_tuning' \
-    --namespace 'damo' \
-    --dataset_name 'controlnet_dataset_condition_fill50k' \
+    --train_dataset_namespace 'damo' \
+    --train_dataset_name 'controlnet_dataset_condition_fill50k' \
     --max_epochs 1 \
     --save_ckpt_strategy 'by_epoch' \
     --logging_interval 100 \
     --train.dataloader.workers_per_gpu 0 \
     --evaluation.dataloader.workers_per_gpu 0 \
-    --train.optimizer.lr 1e-5
+    --train.optimizer.lr 1e-5 \
+    --use_model_config true

examples/pytorch/transformers/configuration.json

@@ -1 +0,0 @@
-{"framework":"pytorch","train":{"work_dir":"/tmp","max_epochs":10,"dataloader":{"batch_size_per_gpu":16,"workers_per_gpu":0},"optimizer":{"type":"SGD","lr":0.001},"lr_scheduler":{"type":"StepLR","step_size":2},"hooks":[{"type":"CheckpointHook","interval":1}]},"evaluation":{"dataloader":{"batch_size_per_gpu":16,"workers_per_gpu":0,"shuffle":false}}}

examples/pytorch/transformers/finetune_transformers_model.py

@@ -5,11 +5,11 @@ from datasets import load_dataset
 from transformers import (BertForSequenceClassification, BertTokenizerFast,
                           default_data_collator)
 
+from modelscope import TrainingArgs
 from modelscope.trainers import EpochBasedTrainer, build_trainer
-from modelscope.trainers.default_config import DEFAULT_CONFIG, TrainingArgs
 
 
-@dataclass
+@dataclass(init=False)
 class TransformersArguments(TrainingArgs):
 
     num_labels: int = field(
@@ -17,13 +17,27 @@ class TransformersArguments(TrainingArgs):
             'help': 'The number of labels',
         })
 
+    sentence: str = field(
+        default=None, metadata={
+            'help': 'The sentence key',
+        })
 
-args = TransformersArguments.from_cli(
-    task='text-classification', eval_metrics='seq-cls-metric')
+    label: str = field(
+        default=None, metadata={
+            'help': 'The label key',
+        })
 
-print(args)
 
-dataset = load_dataset(args.dataset_name, args.subset_name)
+training_args = TransformersArguments(
+    task='text-classification', eval_metrics='seq-cls-metric').parse_cli()
+config, args = training_args.to_config()
+
+print(config, args)
+
+train_dataset = load_dataset(
+    args.train_dataset_name, args.train_subset_name, split=args.train_split)
+val_dataset = load_dataset(
+    args.val_dataset_name, args.val_subset_name, split=args.val_split)
 
 model = BertForSequenceClassification.from_pretrained(
     args.model, num_labels=args.num_labels)
@@ -31,26 +45,30 @@ tokenizer = BertTokenizerFast.from_pretrained(args.model)
 
 
 def tokenize_sentence(row):
-    return tokenizer(row['sentence'], padding='max_length', max_length=128)
+    return tokenizer(
+        row[training_args.sentence], padding='max_length', max_length=128)
 
 
 # Extra columns, Rename columns
-dataset = dataset.map(tokenize_sentence).remove_columns(['sentence',
-                                                         'idx']).rename_column(
-                                                             'label', 'labels')
+train_dataset = train_dataset.map(tokenize_sentence)
+val_dataset = val_dataset.map(tokenize_sentence)
+if training_args.label != 'labels':
+    train_dataset = train_dataset.rename_columns(
+        {training_args.label: 'labels'})
+    val_dataset = val_dataset.rename_columns({training_args.label: 'labels'})
 
 cfg_file = os.path.join(args.work_dir or './', 'configuration.json')
-DEFAULT_CONFIG.dump(cfg_file)
+config.dump(cfg_file)
 
 kwargs = dict(
     model=model,
     cfg_file=cfg_file,
     # data_collator
     data_collator=default_data_collator,
-    train_dataset=dataset['train'],
-    eval_dataset=dataset['validation'],
-    seed=args.seed,
-    cfg_modify_fn=args)
+    train_dataset=train_dataset,
+    eval_dataset=val_dataset,
+    remove_unused_data=True,
+    seed=args.seed)
 
 os.environ['LOCAL_RANK'] = str(args.local_rank)
 trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs)

examples/pytorch/transformers/run_train.sh

@@ -1,5 +1,14 @@
 PYTHONPATH=. python examples/pytorch/transformers/finetune_transformers_model.py \
     --model bert-base-uncased \
     --num_labels 15 \
-    --dataset_name clue \
-    --subset_name tnews
+    --train_dataset_name clue \
+    --train_subset_name tnews \
+    --train_split train \
+    --val_dataset_name clue \
+    --val_subset_name tnews \
+    --train_split train \
+    --val_split validation \
+    --sentence sentence \
+    --label label \
+    --eval_strategy by_step \
+    --eval_interval 100

modelscope/trainers/hooks/checkpoint/checkpoint_hook.py

@@ -348,7 +348,7 @@ class BestCkptSaverHook(CheckpointHook):
             self._do_save(trainer, 'by_step')
 
     def _should_save(self, trainer):
-        return self._is_best_metric(trainer.metric_values) and self.save_best
+        return self.save_best and self._is_best_metric(trainer.metric_values)
 
     def _is_best_metric(self, metric_values):
         if metric_values is None:

modelscope/trainers/training_args.py

@@ -51,6 +51,16 @@ class DatasetArgs:
             'help': 'The subset name used for evaluating, can be None',
         })
 
+    train_split: str = field(
+        default=None, metadata={
+            'help': 'The split of train dataset',
+        })
+
+    val_split: str = field(
+        default=None, metadata={
+            'help': 'The split of val dataset',
+        })
+
     train_dataset_namespace: str = field(
         default=None,
         metadata={
@@ -431,7 +441,10 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs):
         """
         parser = CliArgumentParser(self)
         args, unknown = parser.parse_known_args(parser_args)
-        unknown = [item for item in unknown if item not in ('\\', '\n')]
+        unknown = [
+            item for item in unknown
+            if item not in ('\\', '\n') and '--local-rank=' not in item
+        ]
         _unknown = {}
         for i in range(0, len(unknown), 2):
             _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1])
@@ -460,8 +473,11 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs):
             cfg_setter = f.metadata.get('cfg_setter') or (lambda x: x)
             if cfg_node is not None:
                 if f.name in self.manual_args or not ignore_default_config:
-                    cfg.merge_from_dict(
-                        {cfg_node: cfg_setter(getattr(self, f.name))})
+                    if isinstance(cfg_node, str):
+                        cfg_node = [cfg_node]
+                    for _node in cfg_node:
+                        cfg.merge_from_dict(
+                            {_node: cfg_setter(getattr(self, f.name))})
             else:
                 args_dict[f.name] = getattr(self, f.name)